solution: kotlin code conventions
This commit is contained in:
8
.gitignore
vendored
8
.gitignore
vendored
@@ -1,7 +1,11 @@
|
|||||||
.gradle/
|
.gradle/
|
||||||
build/
|
build/
|
||||||
out/
|
out/
|
||||||
*.iml
|
|
||||||
./dshackle.yaml
|
./dshackle.yaml
|
||||||
./upstream.yaml
|
./upstream.yaml
|
||||||
testsetup/
|
testsetup/
|
||||||
|
.idea/
|
||||||
|
.idea_modules/
|
||||||
|
*.iml
|
||||||
|
*.ipr
|
||||||
|
*.iws
|
||||||
@@ -27,6 +27,7 @@ plugins {
|
|||||||
id 'io.spring.dependency-management' version '1.0.6.RELEASE'
|
id 'io.spring.dependency-management' version '1.0.6.RELEASE'
|
||||||
id 'com.palantir.git-version' version '0.12.2'
|
id 'com.palantir.git-version' version '0.12.2'
|
||||||
id "com.google.protobuf" version "0.8.12"
|
id "com.google.protobuf" version "0.8.12"
|
||||||
|
id "org.jlleitschuh.gradle.ktlint" version "10.2.0"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
kotlin.code.style=official
|
||||||
|
|
||||||
# Languages
|
# Languages
|
||||||
groovyVersion=2.5.14
|
groovyVersion=2.5.14
|
||||||
kotlinVersion=1.5.30
|
kotlinVersion=1.5.30
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ package io.emeraldpay.dshackle
|
|||||||
|
|
||||||
import io.emeraldpay.api.proto.Common
|
import io.emeraldpay.api.proto.Common
|
||||||
import io.emeraldpay.grpc.Chain
|
import io.emeraldpay.grpc.Chain
|
||||||
import java.util.*
|
import java.util.EnumMap
|
||||||
import java.util.concurrent.locks.ReentrantLock
|
import java.util.concurrent.locks.ReentrantLock
|
||||||
import kotlin.concurrent.withLock
|
import kotlin.concurrent.withLock
|
||||||
|
|
||||||
@@ -25,7 +25,7 @@ import kotlin.concurrent.withLock
|
|||||||
* Keeps a lazily created value associated with a Chain
|
* Keeps a lazily created value associated with a Chain
|
||||||
*/
|
*/
|
||||||
class ChainValue<V>(
|
class ChainValue<V>(
|
||||||
private val factory: (chain: Chain) -> V
|
private val factory: (chain: Chain) -> V
|
||||||
) {
|
) {
|
||||||
|
|
||||||
private val values = EnumMap<Chain, V>(Chain::class.java)
|
private val values = EnumMap<Chain, V>(Chain::class.java)
|
||||||
@@ -52,4 +52,4 @@ class ChainValue<V>(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,26 +16,26 @@
|
|||||||
*/
|
*/
|
||||||
package io.emeraldpay.dshackle
|
package io.emeraldpay.dshackle
|
||||||
|
|
||||||
import com.fasterxml.jackson.core.Version
|
import io.emeraldpay.dshackle.config.CacheConfig
|
||||||
import com.fasterxml.jackson.databind.DeserializationFeature
|
import io.emeraldpay.dshackle.config.MainConfig
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper
|
import io.emeraldpay.dshackle.config.MainConfigReader
|
||||||
import com.fasterxml.jackson.databind.module.SimpleModule
|
import io.emeraldpay.dshackle.config.MonitoringConfig
|
||||||
import io.emeraldpay.dshackle.config.*
|
import io.emeraldpay.dshackle.config.TokensConfig
|
||||||
|
import io.emeraldpay.dshackle.config.UpstreamsConfig
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
import org.springframework.beans.factory.annotation.Qualifier
|
import org.springframework.beans.factory.annotation.Qualifier
|
||||||
import org.springframework.boot.ExitCodeGenerator
|
import org.springframework.boot.ExitCodeGenerator
|
||||||
import org.springframework.boot.SpringApplication
|
import org.springframework.boot.SpringApplication
|
||||||
import org.springframework.context.ApplicationContext
|
import org.springframework.context.ApplicationContext
|
||||||
import org.springframework.context.annotation.*
|
import org.springframework.context.annotation.Bean
|
||||||
|
import org.springframework.context.annotation.Configuration
|
||||||
import org.springframework.core.env.Environment
|
import org.springframework.core.env.Environment
|
||||||
import org.springframework.scheduling.annotation.EnableAsync
|
import org.springframework.scheduling.annotation.EnableAsync
|
||||||
import org.springframework.scheduling.annotation.EnableScheduling
|
import org.springframework.scheduling.annotation.EnableScheduling
|
||||||
import reactor.core.scheduler.Scheduler
|
import reactor.core.scheduler.Scheduler
|
||||||
import reactor.core.scheduler.Schedulers
|
import reactor.core.scheduler.Schedulers
|
||||||
import java.io.File
|
import java.io.File
|
||||||
import java.text.SimpleDateFormat
|
|
||||||
import java.util.*
|
|
||||||
import java.util.concurrent.Executors
|
import java.util.concurrent.Executors
|
||||||
import kotlin.system.exitProcess
|
import kotlin.system.exitProcess
|
||||||
|
|
||||||
@@ -43,8 +43,8 @@ import kotlin.system.exitProcess
|
|||||||
@EnableScheduling
|
@EnableScheduling
|
||||||
@EnableAsync
|
@EnableAsync
|
||||||
open class Config(
|
open class Config(
|
||||||
@Autowired private val env: Environment,
|
@Autowired private val env: Environment,
|
||||||
@Autowired private val ctx: ApplicationContext
|
@Autowired private val ctx: ApplicationContext
|
||||||
) {
|
) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -68,14 +68,15 @@ open class Config(
|
|||||||
if (!FileResolver.isAccessible(target)) {
|
if (!FileResolver.isAccessible(target)) {
|
||||||
target = File(LOCAL_CONFIG)
|
target = File(LOCAL_CONFIG)
|
||||||
if (!FileResolver.isAccessible(target)) {
|
if (!FileResolver.isAccessible(target)) {
|
||||||
throw IllegalStateException("Configuration is not found neither at ${DEFAULT_CONFIG} nor ${LOCAL_CONFIG}")
|
throw IllegalStateException("Configuration is not found neither at $DEFAULT_CONFIG nor $LOCAL_CONFIG")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
target = target.normalize()
|
target = target.normalize()
|
||||||
return target
|
return target
|
||||||
}
|
}
|
||||||
|
|
||||||
@Bean @Qualifier("upstreamScheduler")
|
@Bean
|
||||||
|
@Qualifier("upstreamScheduler")
|
||||||
open fun upstreamScheduler(): Scheduler {
|
open fun upstreamScheduler(): Scheduler {
|
||||||
return Schedulers.fromExecutorService(Executors.newFixedThreadPool(16))
|
return Schedulers.fromExecutorService(Executors.newFixedThreadPool(16))
|
||||||
}
|
}
|
||||||
@@ -91,7 +92,7 @@ open class Config(
|
|||||||
}
|
}
|
||||||
val reader = MainConfigReader(fileResolver)
|
val reader = MainConfigReader(fileResolver)
|
||||||
return reader.read(f.inputStream())
|
return reader.read(f.inputStream())
|
||||||
?: throw IllegalStateException("Config is not available at ${f.absolutePath}")
|
?: throw IllegalStateException("Config is not available at ${f.absolutePath}")
|
||||||
}
|
}
|
||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
@@ -119,5 +120,4 @@ open class Config(
|
|||||||
open fun monitoringConfig(@Autowired mainConfig: MainConfig): MonitoringConfig {
|
open fun monitoringConfig(@Autowired mainConfig: MainConfig): MonitoringConfig {
|
||||||
return mainConfig.monitoring
|
return mainConfig.monitoring
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -25,4 +25,4 @@ class Defaults {
|
|||||||
val timeoutInternal: Duration = timeout.dividedBy(4)
|
val timeoutInternal: Duration = timeout.dividedBy(4)
|
||||||
val retryConnection: Duration = Duration.ofSeconds(10)
|
val retryConnection: Duration = Duration.ofSeconds(10)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ package io.emeraldpay.dshackle
|
|||||||
import java.io.File
|
import java.io.File
|
||||||
|
|
||||||
open class FileResolver(
|
open class FileResolver(
|
||||||
private val baseDir: File
|
private val baseDir: File
|
||||||
) {
|
) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -35,5 +35,4 @@ open class FileResolver(
|
|||||||
}
|
}
|
||||||
return File(baseDir, path)
|
return File(baseDir, path)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspentDeserializer
|
|||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
||||||
import java.text.SimpleDateFormat
|
import java.text.SimpleDateFormat
|
||||||
import java.util.*
|
import java.util.TimeZone
|
||||||
import java.util.concurrent.Executors
|
import java.util.concurrent.Executors
|
||||||
import java.util.concurrent.ScheduledExecutorService
|
import java.util.concurrent.ScheduledExecutorService
|
||||||
|
|
||||||
@@ -57,12 +57,10 @@ class Global {
|
|||||||
objectMapper.registerModule(JavaTimeModule())
|
objectMapper.registerModule(JavaTimeModule())
|
||||||
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
|
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
|
||||||
objectMapper
|
objectMapper
|
||||||
.setDateFormat(SimpleDateFormat("yyyy-MM-dd\'T\'HH:mm:ss.SSS"))
|
.setDateFormat(SimpleDateFormat("yyyy-MM-dd\'T\'HH:mm:ss.SSS"))
|
||||||
.setTimeZone(TimeZone.getTimeZone("UTC"))
|
.setTimeZone(TimeZone.getTimeZone("UTC"))
|
||||||
|
|
||||||
return objectMapper
|
return objectMapper
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ package io.emeraldpay.dshackle
|
|||||||
|
|
||||||
import io.emeraldpay.dshackle.config.MainConfig
|
import io.emeraldpay.dshackle.config.MainConfig
|
||||||
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerGrpc
|
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerGrpc
|
||||||
import io.grpc.*
|
import io.grpc.Server
|
||||||
import io.grpc.netty.NettyServerBuilder
|
import io.grpc.netty.NettyServerBuilder
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
@@ -29,15 +29,15 @@ import javax.annotation.PreDestroy
|
|||||||
|
|
||||||
@Service
|
@Service
|
||||||
open class GrpcServer(
|
open class GrpcServer(
|
||||||
@Autowired val rpcs: List<io.grpc.BindableService>,
|
@Autowired val rpcs: List<io.grpc.BindableService>,
|
||||||
@Autowired val mainConfig: MainConfig,
|
@Autowired val mainConfig: MainConfig,
|
||||||
@Autowired val tlsSetup: TlsSetup,
|
@Autowired val tlsSetup: TlsSetup,
|
||||||
@Autowired val accessHandler: AccessHandlerGrpc
|
@Autowired val accessHandler: AccessHandlerGrpc
|
||||||
) {
|
) {
|
||||||
|
|
||||||
private val log = LoggerFactory.getLogger(GrpcServer::class.java)
|
private val log = LoggerFactory.getLogger(GrpcServer::class.java)
|
||||||
|
|
||||||
private var server: Server? = null;
|
private var server: Server? = null
|
||||||
|
|
||||||
@PostConstruct
|
@PostConstruct
|
||||||
fun start() {
|
fun start() {
|
||||||
@@ -45,14 +45,14 @@ open class GrpcServer(
|
|||||||
log.debug("Running with DEBUG LOGGING")
|
log.debug("Running with DEBUG LOGGING")
|
||||||
log.info("Listening Native gRPC on ${mainConfig.host}:${mainConfig.port}")
|
log.info("Listening Native gRPC on ${mainConfig.host}:${mainConfig.port}")
|
||||||
val serverBuilder = NettyServerBuilder
|
val serverBuilder = NettyServerBuilder
|
||||||
.forAddress(InetSocketAddress(mainConfig.host, mainConfig.port))
|
.forAddress(InetSocketAddress(mainConfig.host, mainConfig.port))
|
||||||
.let {
|
.let {
|
||||||
if (mainConfig.accessLogConfig.enabled) {
|
if (mainConfig.accessLogConfig.enabled) {
|
||||||
it.intercept(accessHandler)
|
it.intercept(accessHandler)
|
||||||
} else {
|
} else {
|
||||||
it
|
it
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
tlsSetup.setupServer("Native gRPC", mainConfig.tls, true)?.let {
|
tlsSetup.setupServer("Native gRPC", mainConfig.tls, true)?.let {
|
||||||
serverBuilder.sslContext(it)
|
serverBuilder.sslContext(it)
|
||||||
@@ -75,5 +75,4 @@ open class GrpcServer(
|
|||||||
server?.shutdownNow()
|
server?.shutdownNow()
|
||||||
log.info("GRPC Server shot down")
|
log.info("GRPC Server shot down")
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -17,7 +17,6 @@
|
|||||||
package io.emeraldpay.dshackle
|
package io.emeraldpay.dshackle
|
||||||
|
|
||||||
import io.emeraldpay.dshackle.config.MainConfig
|
import io.emeraldpay.dshackle.config.MainConfig
|
||||||
import io.emeraldpay.dshackle.config.ProxyConfig
|
|
||||||
import io.emeraldpay.dshackle.monitoring.MonitoringSetup
|
import io.emeraldpay.dshackle.monitoring.MonitoringSetup
|
||||||
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp
|
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp
|
||||||
import io.emeraldpay.dshackle.proxy.ProxyServer
|
import io.emeraldpay.dshackle.proxy.ProxyServer
|
||||||
@@ -26,8 +25,6 @@ import io.emeraldpay.dshackle.proxy.WriteRpcJson
|
|||||||
import io.emeraldpay.dshackle.rpc.NativeCall
|
import io.emeraldpay.dshackle.rpc.NativeCall
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
import org.springframework.context.annotation.DependsOn
|
|
||||||
import org.springframework.core.env.Environment
|
|
||||||
import org.springframework.stereotype.Service
|
import org.springframework.stereotype.Service
|
||||||
import javax.annotation.PostConstruct
|
import javax.annotation.PostConstruct
|
||||||
|
|
||||||
@@ -36,14 +33,14 @@ import javax.annotation.PostConstruct
|
|||||||
*/
|
*/
|
||||||
@Service
|
@Service
|
||||||
class ProxyStarter(
|
class ProxyStarter(
|
||||||
@Autowired private val mainConfig: MainConfig,
|
@Autowired private val mainConfig: MainConfig,
|
||||||
@Autowired private val readRpcJson: ReadRpcJson,
|
@Autowired private val readRpcJson: ReadRpcJson,
|
||||||
@Autowired private val writeRpcJson: WriteRpcJson,
|
@Autowired private val writeRpcJson: WriteRpcJson,
|
||||||
@Autowired private val nativeCall: NativeCall,
|
@Autowired private val nativeCall: NativeCall,
|
||||||
@Autowired private val tlsSetup: TlsSetup,
|
@Autowired private val tlsSetup: TlsSetup,
|
||||||
@Autowired private val accessHandlerHttp: AccessHandlerHttp,
|
@Autowired private val accessHandlerHttp: AccessHandlerHttp,
|
||||||
// depend on Monitoring, declared here just to ensure it's properly initialized before the Proxy
|
// depend on Monitoring, declared here just to ensure it's properly initialized before the Proxy
|
||||||
@Autowired private val monitoringSetup: MonitoringSetup
|
@Autowired private val monitoringSetup: MonitoringSetup
|
||||||
) {
|
) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -60,5 +57,4 @@ class ProxyStarter(
|
|||||||
val server = ProxyServer(config, readRpcJson, writeRpcJson, nativeCall, tlsSetup, accessHandlerHttp.factory)
|
val server = ProxyServer(config, readRpcJson, writeRpcJson, nativeCall, tlsSetup, accessHandlerHttp.factory)
|
||||||
server.start()
|
server.start()
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -22,7 +22,6 @@ import io.emeraldpay.grpc.Chain
|
|||||||
*/
|
*/
|
||||||
open class SilentException(message: String) : Exception(message) {
|
open class SilentException(message: String) : Exception(message) {
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Blockchain is not available or not supported by current instance of the Dshackle
|
* Blockchain is not available or not supported by current instance of the Dshackle
|
||||||
*/
|
*/
|
||||||
@@ -31,4 +30,4 @@ open class SilentException(message: String) : Exception(message) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
class DataUnavailable(val code: String) : SilentException("Data is unavailable: $code")
|
class DataUnavailable(val code: String) : SilentException("Data is unavailable: $code")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ import org.springframework.context.annotation.Import
|
|||||||
import org.springframework.core.io.ClassPathResource
|
import org.springframework.core.io.ClassPathResource
|
||||||
import org.springframework.core.io.support.ResourcePropertySource
|
import org.springframework.core.io.support.ResourcePropertySource
|
||||||
|
|
||||||
@SpringBootApplication(scanBasePackages = [ "io.emeraldpay.dshackle" ])
|
@SpringBootApplication(scanBasePackages = ["io.emeraldpay.dshackle"])
|
||||||
@Import(Config::class)
|
@Import(Config::class)
|
||||||
open class Starter
|
open class Starter
|
||||||
|
|
||||||
@@ -35,4 +35,4 @@ fun main(args: Array<String>) {
|
|||||||
app.setDefaultProperties(ResourcePropertySource("version.properties").source)
|
app.setDefaultProperties(ResourcePropertySource("version.properties").source)
|
||||||
app.setBanner(ResourceBanner(ClassPathResource("banner.txt")))
|
app.setBanner(ResourceBanner(ClassPathResource("banner.txt")))
|
||||||
app.run(*args)
|
app.run(*args)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ import org.springframework.stereotype.Service
|
|||||||
|
|
||||||
@Service
|
@Service
|
||||||
open class TlsSetup(
|
open class TlsSetup(
|
||||||
@Autowired val fileResolver: FileResolver
|
@Autowired val fileResolver: FileResolver
|
||||||
) {
|
) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -68,19 +68,19 @@ open class TlsSetup(
|
|||||||
|
|
||||||
val sslContextBuilder = if (grpc) {
|
val sslContextBuilder = if (grpc) {
|
||||||
GrpcSslContexts.forServer(
|
GrpcSslContexts.forServer(
|
||||||
fileResolver.resolve(config.certificate!!),
|
fileResolver.resolve(config.certificate!!),
|
||||||
fileResolver.resolve(config.key!!)
|
fileResolver.resolve(config.key!!)
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
SslContextBuilder.forServer(
|
SslContextBuilder.forServer(
|
||||||
fileResolver.resolve(config.certificate!!),
|
fileResolver.resolve(config.certificate!!),
|
||||||
fileResolver.resolve(config.key!!)
|
fileResolver.resolve(config.key!!)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
if (StringUtils.isNotEmpty(config.clientCa)) {
|
if (StringUtils.isNotEmpty(config.clientCa)) {
|
||||||
log.info("Using TLS for client authentication for $category")
|
log.info("Using TLS for client authentication for $category")
|
||||||
sslContextBuilder.trustManager(
|
sslContextBuilder.trustManager(
|
||||||
fileResolver.resolve(config.clientCa!!)
|
fileResolver.resolve(config.clientCa!!)
|
||||||
)
|
)
|
||||||
if (config.clientRequire != null && config.clientRequire!!) {
|
if (config.clientRequire != null && config.clientRequire!!) {
|
||||||
sslContextBuilder.clientAuth(ClientAuth.REQUIRE)
|
sslContextBuilder.clientAuth(ClientAuth.REQUIRE)
|
||||||
@@ -96,5 +96,4 @@ open class TlsSetup(
|
|||||||
}
|
}
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -25,8 +25,8 @@ import reactor.core.publisher.Mono
|
|||||||
* Connects two caches to read through them. First is cache height->hash, second is hash->block.
|
* Connects two caches to read through them. First is cache height->hash, second is hash->block.
|
||||||
*/
|
*/
|
||||||
open class BlockByHeight(
|
open class BlockByHeight(
|
||||||
private val heights: Reader<Long, BlockId>,
|
private val heights: Reader<Long, BlockId>,
|
||||||
private val blocks: Reader<BlockId, BlockContainer>
|
private val blocks: Reader<BlockId, BlockContainer>
|
||||||
) : Reader<Long, BlockContainer> {
|
) : Reader<Long, BlockContainer> {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -35,7 +35,6 @@ open class BlockByHeight(
|
|||||||
|
|
||||||
override fun read(key: Long): Mono<BlockContainer> {
|
override fun read(key: Long): Mono<BlockContainer> {
|
||||||
return heights.read(key)
|
return heights.read(key)
|
||||||
.flatMap { blocks.read(it) }
|
.flatMap { blocks.read(it) }
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -23,12 +23,12 @@ import io.emeraldpay.dshackle.reader.Reader
|
|||||||
import reactor.core.publisher.Mono
|
import reactor.core.publisher.Mono
|
||||||
|
|
||||||
open class BlocksMemCache(
|
open class BlocksMemCache(
|
||||||
maxSize: Int = 64
|
maxSize: Int = 64
|
||||||
) : Reader<BlockId, BlockContainer> {
|
) : Reader<BlockId, BlockContainer> {
|
||||||
|
|
||||||
private val mapping = Caffeine.newBuilder()
|
private val mapping = Caffeine.newBuilder()
|
||||||
.maximumSize(maxSize.toLong())
|
.maximumSize(maxSize.toLong())
|
||||||
.build<BlockId, BlockContainer>()
|
.build<BlockId, BlockContainer>()
|
||||||
|
|
||||||
override fun read(key: BlockId): Mono<BlockContainer> {
|
override fun read(key: BlockId): Mono<BlockContainer> {
|
||||||
return Mono.justOrEmpty(get(key))
|
return Mono.justOrEmpty(get(key))
|
||||||
@@ -45,4 +45,4 @@ open class BlocksMemCache(
|
|||||||
open fun purge() {
|
open fun purge() {
|
||||||
mapping.cleanUp()
|
mapping.cleanUp()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,10 +32,10 @@ import java.time.Instant
|
|||||||
* Cache blocks in Redis database
|
* Cache blocks in Redis database
|
||||||
*/
|
*/
|
||||||
class BlocksRedisCache(
|
class BlocksRedisCache(
|
||||||
redis: RedisReactiveCommands<String, ByteArray>,
|
redis: RedisReactiveCommands<String, ByteArray>,
|
||||||
chain: Chain
|
chain: Chain
|
||||||
) : Reader<BlockId, BlockContainer>,
|
) : Reader<BlockId, BlockContainer>,
|
||||||
OnBlockRedisCache<BlockContainer>(redis, chain, CachesProto.ValueContainer.ValueType.BLOCK) {
|
OnBlockRedisCache<BlockContainer>(redis, chain, CachesProto.ValueContainer.ValueType.BLOCK) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private val log = LoggerFactory.getLogger(BlocksRedisCache::class.java)
|
private val log = LoggerFactory.getLogger(BlocksRedisCache::class.java)
|
||||||
@@ -59,21 +59,21 @@ class BlocksRedisCache(
|
|||||||
}
|
}
|
||||||
val meta = value.blockMeta
|
val meta = value.blockMeta
|
||||||
return BlockContainer(
|
return BlockContainer(
|
||||||
meta.height,
|
meta.height,
|
||||||
BlockId(meta.hash.toByteArray()),
|
BlockId(meta.hash.toByteArray()),
|
||||||
BigInteger(meta.difficulty.toByteArray()),
|
BigInteger(meta.difficulty.toByteArray()),
|
||||||
Instant.ofEpochMilli(meta.timestamp),
|
Instant.ofEpochMilli(meta.timestamp),
|
||||||
false,
|
false,
|
||||||
value.value.toByteArray(),
|
value.value.toByteArray(),
|
||||||
null,
|
null,
|
||||||
meta.txHashesList.map {
|
meta.txHashesList.map {
|
||||||
TxId(it.toByteArray())
|
TxId(it.toByteArray())
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun add(block: BlockContainer): Mono<Void> {
|
fun add(block: BlockContainer): Mono<Void> {
|
||||||
if (block.timestamp == null || block.hash == null) { //null in unit tests
|
if (block.timestamp == null || block.hash == null) { // null in unit tests
|
||||||
return Mono.empty()
|
return Mono.empty()
|
||||||
}
|
}
|
||||||
if (block.full) {
|
if (block.full) {
|
||||||
@@ -81,5 +81,4 @@ class BlocksRedisCache(
|
|||||||
}
|
}
|
||||||
return super.add(block, block)
|
return super.add(block, block)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -16,9 +16,12 @@
|
|||||||
package io.emeraldpay.dshackle.cache
|
package io.emeraldpay.dshackle.cache
|
||||||
|
|
||||||
import io.emeraldpay.dshackle.Global
|
import io.emeraldpay.dshackle.Global
|
||||||
import io.emeraldpay.dshackle.data.*
|
import io.emeraldpay.dshackle.data.BlockContainer
|
||||||
|
import io.emeraldpay.dshackle.data.BlockId
|
||||||
|
import io.emeraldpay.dshackle.data.DefaultContainer
|
||||||
|
import io.emeraldpay.dshackle.data.TxContainer
|
||||||
|
import io.emeraldpay.dshackle.data.TxId
|
||||||
import io.emeraldpay.dshackle.reader.CompoundReader
|
import io.emeraldpay.dshackle.reader.CompoundReader
|
||||||
import io.emeraldpay.dshackle.reader.EmptyReader
|
|
||||||
import io.emeraldpay.dshackle.reader.Reader
|
import io.emeraldpay.dshackle.reader.Reader
|
||||||
import io.emeraldpay.dshackle.upstream.Head
|
import io.emeraldpay.dshackle.upstream.Head
|
||||||
import io.emeraldpay.dshackle.upstream.ethereum.EthereumFullBlocksReader
|
import io.emeraldpay.dshackle.upstream.ethereum.EthereumFullBlocksReader
|
||||||
@@ -30,14 +33,14 @@ import reactor.core.publisher.Flux
|
|||||||
import reactor.core.publisher.Mono
|
import reactor.core.publisher.Mono
|
||||||
|
|
||||||
open class Caches(
|
open class Caches(
|
||||||
private val memBlocksByHash: BlocksMemCache,
|
private val memBlocksByHash: BlocksMemCache,
|
||||||
private val blocksByHeight: HeightCache,
|
private val blocksByHeight: HeightCache,
|
||||||
private val memTxsByHash: TxMemCache,
|
private val memTxsByHash: TxMemCache,
|
||||||
private val memReceipts: ReceiptMemCache,
|
private val memReceipts: ReceiptMemCache,
|
||||||
private val redisBlocksByHash: BlocksRedisCache?,
|
private val redisBlocksByHash: BlocksRedisCache?,
|
||||||
private val redisTxsByHash: TxRedisCache?,
|
private val redisTxsByHash: TxRedisCache?,
|
||||||
private val redisReceipts: ReceiptRedisCache?,
|
private val redisReceipts: ReceiptRedisCache?,
|
||||||
private val redisHeightByHashCache: HeightByHashRedisCache?
|
private val redisHeightByHashCache: HeightByHashRedisCache?
|
||||||
) {
|
) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -91,17 +94,17 @@ open class Caches(
|
|||||||
if (currentHeight != null && data.height != null && memReceipts.acceptsRecentBlocks(currentHeight - data.height)) {
|
if (currentHeight != null && data.height != null && memReceipts.acceptsRecentBlocks(currentHeight - data.height)) {
|
||||||
memReceipts.add(data)
|
memReceipts.add(data)
|
||||||
}
|
}
|
||||||
//TODO move subscription to the caller
|
// TODO move subscription to the caller
|
||||||
redisReceipts?.add(data)?.subscribe()
|
redisReceipts?.add(data)?.subscribe()
|
||||||
}
|
}
|
||||||
|
|
||||||
fun cache(tag: Tag, tx: TxContainer) {
|
fun cache(tag: Tag, tx: TxContainer) {
|
||||||
//do not cache transactions that are not in a block yet
|
// do not cache transactions that are not in a block yet
|
||||||
if (tx.blockId == null) {
|
if (tx.blockId == null) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
memTxsByHash.add(tx)
|
memTxsByHash.add(tx)
|
||||||
//TODO move subscription to the caller
|
// TODO move subscription to the caller
|
||||||
getBlocksByHash().read(tx.blockId).flatMap { block ->
|
getBlocksByHash().read(tx.blockId).flatMap { block ->
|
||||||
redisTxsByHash?.add(tx, block) ?: Mono.empty()
|
redisTxsByHash?.add(tx, block) ?: Mono.empty()
|
||||||
}.subscribe()
|
}.subscribe()
|
||||||
@@ -113,14 +116,14 @@ open class Caches(
|
|||||||
redisHeightByHashCache?.add(block)?.let(job::add)
|
redisHeightByHashCache?.add(block)?.let(job::add)
|
||||||
|
|
||||||
if (tag == Tag.LATEST) {
|
if (tag == Tag.LATEST) {
|
||||||
//for LATEST data cache it in memory, it may be short living so better to avoid Redis
|
// for LATEST data cache it in memory, it may be short living so better to avoid Redis
|
||||||
memoizeBlock(block)
|
memoizeBlock(block)
|
||||||
} else if (tag == Tag.REQUESTED) {
|
} else if (tag == Tag.REQUESTED) {
|
||||||
val blockOnlyContainer: BlockContainer?
|
val blockOnlyContainer: BlockContainer?
|
||||||
var jsonValue: BlockJson<*>? = null
|
var jsonValue: BlockJson<*>? = null
|
||||||
if (block.full) {
|
if (block.full) {
|
||||||
jsonValue = Global.objectMapper.readValue<BlockJson<*>>(block.json, BlockJson::class.java)
|
jsonValue = Global.objectMapper.readValue<BlockJson<*>>(block.json, BlockJson::class.java)
|
||||||
//shouldn't cache block json with transactions, separate txes and blocks with refs
|
// shouldn't cache block json with transactions, separate txes and blocks with refs
|
||||||
val blockOnly = jsonValue.withoutTransactionDetails()
|
val blockOnly = jsonValue.withoutTransactionDetails()
|
||||||
blockOnlyContainer = BlockContainer.from(blockOnly)
|
blockOnlyContainer = BlockContainer.from(blockOnly)
|
||||||
} else {
|
} else {
|
||||||
@@ -138,15 +141,17 @@ open class Caches(
|
|||||||
TxContainer.from(tx)
|
TxContainer.from(tx)
|
||||||
}
|
}
|
||||||
if (redisTxsByHash != null) {
|
if (redisTxsByHash != null) {
|
||||||
job.add(Flux.fromIterable(transactions)
|
job.add(
|
||||||
|
Flux.fromIterable(transactions)
|
||||||
.doOnNext { memTxsByHash.add(it) }
|
.doOnNext { memTxsByHash.add(it) }
|
||||||
.flatMap { redisTxsByHash.add(it, block) }
|
.flatMap { redisTxsByHash.add(it, block) }
|
||||||
.then())
|
.then()
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Flux.fromIterable(job).flatMap { it }.subscribe() //TODO move out to a caller
|
Flux.fromIterable(job).flatMap { it }.subscribe() // TODO move out to a caller
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -156,7 +161,7 @@ open class Caches(
|
|||||||
memBlocksByHash.add(block)
|
memBlocksByHash.add(block)
|
||||||
memHeightByHash.add(block)
|
memHeightByHash.add(block)
|
||||||
val replaced = blocksByHeight.add(block)
|
val replaced = blocksByHeight.add(block)
|
||||||
//evict cached transactions if an existing block was updated
|
// evict cached transactions if an existing block was updated
|
||||||
replaced?.let { evict(it) }
|
replaced?.let { evict(it) }
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -222,7 +227,7 @@ open class Caches(
|
|||||||
REQUESTED
|
REQUESTED
|
||||||
}
|
}
|
||||||
|
|
||||||
class Builder() {
|
class Builder {
|
||||||
private var blocksByHash: BlocksMemCache? = null
|
private var blocksByHash: BlocksMemCache? = null
|
||||||
private var blocksByHeight: HeightCache? = null
|
private var blocksByHeight: HeightCache? = null
|
||||||
private var txsByHash: TxMemCache? = null
|
private var txsByHash: TxMemCache? = null
|
||||||
@@ -285,8 +290,10 @@ open class Caches(
|
|||||||
if (receipts == null) {
|
if (receipts == null) {
|
||||||
receipts = ReceiptMemCache()
|
receipts = ReceiptMemCache()
|
||||||
}
|
}
|
||||||
return Caches(blocksByHash!!, blocksByHeight!!, txsByHash!!, receipts!!,
|
return Caches(
|
||||||
redisBlocksByHash, redisTxsByHash, redisReceiptCache, redisHeightByHashCache)
|
blocksByHash!!, blocksByHeight!!, txsByHash!!, receipts!!,
|
||||||
|
redisBlocksByHash, redisTxsByHash, redisReceiptCache, redisHeightByHashCache
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,5 +21,4 @@ package io.emeraldpay.dshackle.cache
|
|||||||
interface CachesEnabled {
|
interface CachesEnabled {
|
||||||
|
|
||||||
fun setCaches(caches: Caches)
|
fun setCaches(caches: Caches)
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -27,14 +27,13 @@ import io.lettuce.core.codec.StringCodec
|
|||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
import org.springframework.stereotype.Repository
|
import org.springframework.stereotype.Repository
|
||||||
import java.util.*
|
import java.util.EnumMap
|
||||||
import javax.annotation.PostConstruct
|
import javax.annotation.PostConstruct
|
||||||
import kotlin.system.exitProcess
|
import kotlin.system.exitProcess
|
||||||
|
|
||||||
|
|
||||||
@Repository
|
@Repository
|
||||||
open class CachesFactory(
|
open class CachesFactory(
|
||||||
@Autowired private val cacheConfig: CacheConfig
|
@Autowired private val cacheConfig: CacheConfig
|
||||||
) {
|
) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -50,14 +49,14 @@ open class CachesFactory(
|
|||||||
val redisConfig = cacheConfig.redis ?: return
|
val redisConfig = cacheConfig.redis ?: return
|
||||||
|
|
||||||
var uri = RedisURI.builder()
|
var uri = RedisURI.builder()
|
||||||
.withHost(redisConfig.host)
|
.withHost(redisConfig.host)
|
||||||
.withPort(redisConfig.port)
|
.withPort(redisConfig.port)
|
||||||
|
|
||||||
redisConfig.db?.let { value ->
|
redisConfig.db?.let { value ->
|
||||||
uri = uri.withDatabase(value)
|
uri = uri.withDatabase(value)
|
||||||
}
|
}
|
||||||
|
|
||||||
//log URI _before_ adding a password, to avoid leaking it to the log
|
// log URI _before_ adding a password, to avoid leaking it to the log
|
||||||
log.info("Use Redis cache at: ${uri.build().toURI()}")
|
log.info("Use Redis cache at: ${uri.build().toURI()}")
|
||||||
|
|
||||||
redisConfig.password?.let { value ->
|
redisConfig.password?.let { value ->
|
||||||
@@ -113,4 +112,4 @@ open class CachesFactory(
|
|||||||
}
|
}
|
||||||
return existing
|
return existing
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,5 +20,4 @@ class CurrentBlockCache<K, D> : Reader<K, D> {
|
|||||||
fun evict() {
|
fun evict() {
|
||||||
cache.set(ConcurrentHashMap())
|
cache.set(ConcurrentHashMap())
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -29,9 +29,9 @@ import reactor.core.publisher.Mono
|
|||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
class HeightByHashAdding(
|
class HeightByHashAdding(
|
||||||
private val mem: Reader<BlockId, Long>,
|
private val mem: Reader<BlockId, Long>,
|
||||||
private val redis: HeightByHashCache?,
|
private val redis: HeightByHashCache?,
|
||||||
private val upstreamReader: Reader<BlockId, BlockContainer>
|
private val upstreamReader: Reader<BlockId, BlockContainer>
|
||||||
) : Reader<BlockId, Long> {
|
) : Reader<BlockId, Long> {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -39,7 +39,7 @@ class HeightByHashAdding(
|
|||||||
}
|
}
|
||||||
|
|
||||||
constructor(caches: Caches, upstreamReader: Reader<BlockId, BlockContainer>) :
|
constructor(caches: Caches, upstreamReader: Reader<BlockId, BlockContainer>) :
|
||||||
this(caches.getLastHeightByHash(), caches.getRedisHeightByHash(), upstreamReader)
|
this(caches.getLastHeightByHash(), caches.getRedisHeightByHash(), upstreamReader)
|
||||||
|
|
||||||
private val delegate: Reader<BlockId, Long>
|
private val delegate: Reader<BlockId, Long>
|
||||||
|
|
||||||
@@ -48,26 +48,26 @@ class HeightByHashAdding(
|
|||||||
delegate = object : Reader<BlockId, Long> {
|
delegate = object : Reader<BlockId, Long> {
|
||||||
override fun read(key: BlockId): Mono<Long> {
|
override fun read(key: BlockId): Mono<Long> {
|
||||||
return mem.read(key)
|
return mem.read(key)
|
||||||
.switchIfEmpty(
|
.switchIfEmpty(
|
||||||
Mono.just(key)
|
Mono.just(key)
|
||||||
.flatMap { redis.read(it) }
|
.flatMap { redis.read(it) }
|
||||||
)
|
)
|
||||||
.switchIfEmpty(
|
.switchIfEmpty(
|
||||||
Mono.just(key)
|
Mono.just(key)
|
||||||
.flatMap { upstreamReader.read(it) }
|
.flatMap { upstreamReader.read(it) }
|
||||||
.flatMap { redis.add(it).then(Mono.just(it.height)) }
|
.flatMap { redis.add(it).then(Mono.just(it.height)) }
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
delegate = object : Reader<BlockId, Long> {
|
delegate = object : Reader<BlockId, Long> {
|
||||||
override fun read(key: BlockId): Mono<Long> {
|
override fun read(key: BlockId): Mono<Long> {
|
||||||
return mem.read(key)
|
return mem.read(key)
|
||||||
.switchIfEmpty(
|
.switchIfEmpty(
|
||||||
Mono.just(key)
|
Mono.just(key)
|
||||||
.flatMap { upstreamReader.read(it) }
|
.flatMap { upstreamReader.read(it) }
|
||||||
.map { it.height }
|
.map { it.height }
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -76,5 +76,4 @@ class HeightByHashAdding(
|
|||||||
override fun read(key: BlockId): Mono<Long> {
|
override fun read(key: BlockId): Mono<Long> {
|
||||||
return delegate.read(key)
|
return delegate.read(key)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -23,4 +23,4 @@ import reactor.core.publisher.Mono
|
|||||||
interface HeightByHashCache : Reader<BlockId, Long> {
|
interface HeightByHashCache : Reader<BlockId, Long> {
|
||||||
|
|
||||||
fun add(block: BlockContainer): Mono<Void>
|
fun add(block: BlockContainer): Mono<Void>
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import org.slf4j.LoggerFactory
|
|||||||
import reactor.core.publisher.Mono
|
import reactor.core.publisher.Mono
|
||||||
|
|
||||||
class HeightByHashMemCache(
|
class HeightByHashMemCache(
|
||||||
maxSize: Int = 256
|
maxSize: Int = 256
|
||||||
) : Reader<BlockId, Long> {
|
) : Reader<BlockId, Long> {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -31,8 +31,8 @@ class HeightByHashMemCache(
|
|||||||
}
|
}
|
||||||
|
|
||||||
private val heights = Caffeine.newBuilder()
|
private val heights = Caffeine.newBuilder()
|
||||||
.maximumSize(maxSize.toLong())
|
.maximumSize(maxSize.toLong())
|
||||||
.build<BlockId, Long>()
|
.build<BlockId, Long>()
|
||||||
|
|
||||||
override fun read(key: BlockId): Mono<Long> {
|
override fun read(key: BlockId): Mono<Long> {
|
||||||
return Mono.justOrEmpty(heights.getIfPresent(key))
|
return Mono.justOrEmpty(heights.getIfPresent(key))
|
||||||
@@ -41,4 +41,4 @@ class HeightByHashMemCache(
|
|||||||
fun add(block: BlockContainer) {
|
fun add(block: BlockContainer) {
|
||||||
heights.put(block.hash, block.height)
|
heights.put(block.hash, block.height)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,8 +32,8 @@ import java.util.concurrent.TimeUnit
|
|||||||
* reader would use full block's cache to find out height).
|
* reader would use full block's cache to find out height).
|
||||||
*/
|
*/
|
||||||
class HeightByHashRedisCache(
|
class HeightByHashRedisCache(
|
||||||
private val redis: RedisReactiveCommands<String, ByteArray>,
|
private val redis: RedisReactiveCommands<String, ByteArray>,
|
||||||
private val chain: Chain
|
private val chain: Chain
|
||||||
) : Reader<BlockId, Long>, HeightByHashCache {
|
) : Reader<BlockId, Long>, HeightByHashCache {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -45,41 +45,41 @@ class HeightByHashRedisCache(
|
|||||||
@Suppress("UNCHECKED_CAST")
|
@Suppress("UNCHECKED_CAST")
|
||||||
override fun read(key: BlockId): Mono<Long> {
|
override fun read(key: BlockId): Mono<Long> {
|
||||||
return redis.get(key(key))
|
return redis.get(key(key))
|
||||||
.flatMap { data ->
|
.flatMap { data ->
|
||||||
Mono.justOrEmpty(fromBytes(data)) as Mono<Long>
|
Mono.justOrEmpty(fromBytes(data)) as Mono<Long>
|
||||||
}.onErrorResume {
|
}.onErrorResume {
|
||||||
log.warn("Failed to read Block Height. ${it.javaClass}:${it.message}")
|
log.warn("Failed to read Block Height. ${it.javaClass}:${it.message}")
|
||||||
Mono.empty()
|
Mono.empty()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun add(block: BlockContainer): Mono<Void> {
|
override fun add(block: BlockContainer): Mono<Void> {
|
||||||
return Mono.just(block)
|
return Mono.just(block)
|
||||||
.flatMap { blockData ->
|
.flatMap { blockData ->
|
||||||
// even if block replaced, the mapping hash-long is still valid, so can be cached for long time
|
// even if block replaced, the mapping hash-long is still valid, so can be cached for long time
|
||||||
// even for fresh blocks
|
// even for fresh blocks
|
||||||
val ttl = TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES)
|
val ttl = TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES)
|
||||||
|
|
||||||
val key = key(blockData.hash)
|
val key = key(blockData.hash)
|
||||||
val value = asBytes(blockData.height)
|
val value = asBytes(blockData.height)
|
||||||
redis.setex(key, ttl, value)
|
redis.setex(key, ttl, value)
|
||||||
}
|
}
|
||||||
.doOnError {
|
.doOnError {
|
||||||
log.warn("Failed to save Block Height. ${it.javaClass}:${it.message}")
|
log.warn("Failed to save Block Height. ${it.javaClass}:${it.message}")
|
||||||
}
|
}
|
||||||
//if failed to cache, just continue without it
|
// if failed to cache, just continue without it
|
||||||
.onErrorResume {
|
.onErrorResume {
|
||||||
Mono.empty()
|
Mono.empty()
|
||||||
}
|
}
|
||||||
.then()
|
.then()
|
||||||
}
|
}
|
||||||
|
|
||||||
fun asBytes(value: Long): ByteArray {
|
fun asBytes(value: Long): ByteArray {
|
||||||
val result = ByteArray(8)
|
val result = ByteArray(8)
|
||||||
val bb = ByteBuffer.allocate(8)
|
val bb = ByteBuffer.allocate(8)
|
||||||
.order(ByteOrder.BIG_ENDIAN)
|
.order(ByteOrder.BIG_ENDIAN)
|
||||||
bb.asLongBuffer()
|
bb.asLongBuffer()
|
||||||
.put(value)
|
.put(value)
|
||||||
bb.get(result)
|
bb.get(result)
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
@@ -89,9 +89,9 @@ class HeightByHashRedisCache(
|
|||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
return ByteBuffer.wrap(value)
|
return ByteBuffer.wrap(value)
|
||||||
.order(ByteOrder.BIG_ENDIAN)
|
.order(ByteOrder.BIG_ENDIAN)
|
||||||
.asLongBuffer()
|
.asLongBuffer()
|
||||||
.get()
|
.get()
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -100,4 +100,4 @@ class HeightByHashRedisCache(
|
|||||||
fun key(hash: BlockId): String {
|
fun key(hash: BlockId): String {
|
||||||
return "height:${chain.id}:${hash.toHex()}"
|
return "height:${chain.id}:${hash.toHex()}"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,12 +25,12 @@ import reactor.core.publisher.Mono
|
|||||||
* Memory cache for blocks heights, keeps mapping height->hash.
|
* Memory cache for blocks heights, keeps mapping height->hash.
|
||||||
*/
|
*/
|
||||||
open class HeightCache(
|
open class HeightCache(
|
||||||
maxSize: Int = 512
|
maxSize: Int = 512
|
||||||
) : Reader<Long, BlockId> {
|
) : Reader<Long, BlockId> {
|
||||||
|
|
||||||
private val heights = Caffeine.newBuilder()
|
private val heights = Caffeine.newBuilder()
|
||||||
.maximumSize(maxSize.toLong())
|
.maximumSize(maxSize.toLong())
|
||||||
.build<Long, BlockId>()
|
.build<Long, BlockId>()
|
||||||
|
|
||||||
override fun read(key: Long): Mono<BlockId> {
|
override fun read(key: Long): Mono<BlockId> {
|
||||||
return Mono.justOrEmpty(heights.getIfPresent(key))
|
return Mono.justOrEmpty(heights.getIfPresent(key))
|
||||||
@@ -45,5 +45,4 @@ open class HeightCache(
|
|||||||
fun purge() {
|
fun purge() {
|
||||||
heights.cleanUp()
|
heights.cleanUp()
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -30,9 +30,9 @@ import java.util.concurrent.TimeUnit
|
|||||||
import kotlin.math.min
|
import kotlin.math.min
|
||||||
|
|
||||||
abstract class OnBlockRedisCache<T>(
|
abstract class OnBlockRedisCache<T>(
|
||||||
private val redis: RedisReactiveCommands<String, ByteArray>,
|
private val redis: RedisReactiveCommands<String, ByteArray>,
|
||||||
private val chain: Chain,
|
private val chain: Chain,
|
||||||
private val valueType: ValueContainer.ValueType
|
private val valueType: ValueContainer.ValueType
|
||||||
) : Reader<BlockId, T> {
|
) : Reader<BlockId, T> {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -51,18 +51,18 @@ abstract class OnBlockRedisCache<T>(
|
|||||||
|
|
||||||
fun toProto(block: BlockContainer, value: T): ValueContainer {
|
fun toProto(block: BlockContainer, value: T): ValueContainer {
|
||||||
return ValueContainer.newBuilder()
|
return ValueContainer.newBuilder()
|
||||||
.setType(valueType)
|
.setType(valueType)
|
||||||
.setValue(ByteString.copyFrom(serializeValue(value)))
|
.setValue(ByteString.copyFrom(serializeValue(value)))
|
||||||
.setBlockMeta(buildMeta(block))
|
.setBlockMeta(buildMeta(block))
|
||||||
.build()
|
.build()
|
||||||
}
|
}
|
||||||
|
|
||||||
open fun buildMeta(block: BlockContainer): CachesProto.BlockMeta.Builder {
|
open fun buildMeta(block: BlockContainer): CachesProto.BlockMeta.Builder {
|
||||||
return CachesProto.BlockMeta.newBuilder()
|
return CachesProto.BlockMeta.newBuilder()
|
||||||
.setHash(ByteString.copyFrom(block.hash.value))
|
.setHash(ByteString.copyFrom(block.hash.value))
|
||||||
.setHeight(block.height)
|
.setHeight(block.height)
|
||||||
.setDifficulty(ByteString.copyFrom(block.difficulty.toByteArray()))
|
.setDifficulty(ByteString.copyFrom(block.difficulty.toByteArray()))
|
||||||
.setTimestamp(block.timestamp.toEpochMilli())
|
.setTimestamp(block.timestamp.toEpochMilli())
|
||||||
}
|
}
|
||||||
|
|
||||||
abstract fun serializeValue(value: T): ByteArray
|
abstract fun serializeValue(value: T): ByteArray
|
||||||
@@ -83,7 +83,7 @@ abstract class OnBlockRedisCache<T>(
|
|||||||
* Key in Redis
|
* Key in Redis
|
||||||
*/
|
*/
|
||||||
fun key(hash: BlockId): String {
|
fun key(hash: BlockId): String {
|
||||||
return "${prefix}:${chain.id}:${hash.toHex()}"
|
return "$prefix:${chain.id}:${hash.toHex()}"
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -92,51 +92,51 @@ abstract class OnBlockRedisCache<T>(
|
|||||||
*/
|
*/
|
||||||
open fun add(container: BlockContainer, value: T): Mono<Void> {
|
open fun add(container: BlockContainer, value: T): Mono<Void> {
|
||||||
return Mono.just(container)
|
return Mono.just(container)
|
||||||
.flatMap { block ->
|
.flatMap { block ->
|
||||||
val ttl = cachingTime(block.timestamp)
|
val ttl = cachingTime(block.timestamp)
|
||||||
if (ttl > MIN_CACHE_TIME_SECONDS) {
|
if (ttl > MIN_CACHE_TIME_SECONDS) {
|
||||||
val key = key(block.hash)
|
val key = key(block.hash)
|
||||||
val proto = toProto(block, value)
|
val proto = toProto(block, value)
|
||||||
redis.setex(key, ttl, proto.toByteArray())
|
redis.setex(key, ttl, proto.toByteArray())
|
||||||
} else {
|
} else {
|
||||||
Mono.empty()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
.doOnError {
|
|
||||||
log.warn("Failed to save Block to Redis: ${it.message}")
|
|
||||||
}
|
|
||||||
//if failed to cache, just continue without it
|
|
||||||
.onErrorResume {
|
|
||||||
Mono.empty()
|
Mono.empty()
|
||||||
}
|
}
|
||||||
.then()
|
}
|
||||||
|
.doOnError {
|
||||||
|
log.warn("Failed to save Block to Redis: ${it.message}")
|
||||||
|
}
|
||||||
|
// if failed to cache, just continue without it
|
||||||
|
.onErrorResume {
|
||||||
|
Mono.empty()
|
||||||
|
}
|
||||||
|
.then()
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Calculate time to cache the value
|
* Calculate time to cache the value
|
||||||
*/
|
*/
|
||||||
fun cachingTime(blockTime: Instant): Long {
|
fun cachingTime(blockTime: Instant): Long {
|
||||||
//default caching time is age of the block, i.e. block create hour ago
|
// default caching time is age of the block, i.e. block create hour ago
|
||||||
//keep for hour, but block created 10 seconds ago cache only for 10 seconds, because it
|
// keep for hour, but block created 10 seconds ago cache only for 10 seconds, because it
|
||||||
//still can be replaced in the blockchain
|
// still can be replaced in the blockchain
|
||||||
val age = Instant.now().epochSecond - blockTime.epochSecond
|
val age = Instant.now().epochSecond - blockTime.epochSecond
|
||||||
return min(age, TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES))
|
return min(age, TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES))
|
||||||
}
|
}
|
||||||
|
|
||||||
fun evict(id: BlockId): Mono<Void> {
|
fun evict(id: BlockId): Mono<Void> {
|
||||||
return Mono.just(id)
|
return Mono.just(id)
|
||||||
.flatMap {
|
.flatMap {
|
||||||
redis.del(key(it))
|
redis.del(key(it))
|
||||||
}
|
}
|
||||||
.then()
|
.then()
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun read(key: BlockId): Mono<T> {
|
override fun read(key: BlockId): Mono<T> {
|
||||||
return redis.get(key(key))
|
return redis.get(key(key))
|
||||||
.map { data ->
|
.map { data ->
|
||||||
fromProto(data)
|
fromProto(data)
|
||||||
}.onErrorResume {
|
}.onErrorResume {
|
||||||
Mono.empty()
|
Mono.empty()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,9 +30,9 @@ import java.util.concurrent.TimeUnit
|
|||||||
import kotlin.math.min
|
import kotlin.math.min
|
||||||
|
|
||||||
abstract class OnTxRedisCache<T>(
|
abstract class OnTxRedisCache<T>(
|
||||||
private val redis: RedisReactiveCommands<String, ByteArray>,
|
private val redis: RedisReactiveCommands<String, ByteArray>,
|
||||||
private val chain: Chain,
|
private val chain: Chain,
|
||||||
private val valueType: CachesProto.ValueContainer.ValueType
|
private val valueType: CachesProto.ValueContainer.ValueType
|
||||||
) : Reader<TxId, T> {
|
) : Reader<TxId, T> {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -56,42 +56,42 @@ abstract class OnTxRedisCache<T>(
|
|||||||
* Key in Redis
|
* Key in Redis
|
||||||
*/
|
*/
|
||||||
fun key(hash: TxId): String {
|
fun key(hash: TxId): String {
|
||||||
return "${prefix}:${chain.id}:${hash.toHex()}"
|
return "$prefix:${chain.id}:${hash.toHex()}"
|
||||||
}
|
}
|
||||||
|
|
||||||
fun evict(container: BlockContainer): Mono<Void> {
|
fun evict(container: BlockContainer): Mono<Void> {
|
||||||
return Mono.just(container)
|
return Mono.just(container)
|
||||||
.map { block ->
|
.map { block ->
|
||||||
block.transactions.map {
|
block.transactions.map {
|
||||||
key(it)
|
key(it)
|
||||||
}.toTypedArray()
|
}.toTypedArray()
|
||||||
}.flatMap { keys ->
|
}.flatMap { keys ->
|
||||||
redis.del(*keys)
|
redis.del(*keys)
|
||||||
}.then()
|
}.then()
|
||||||
}
|
}
|
||||||
|
|
||||||
fun evict(id: TxId): Mono<Void> {
|
fun evict(id: TxId): Mono<Void> {
|
||||||
return Mono.just(id)
|
return Mono.just(id)
|
||||||
.flatMap {
|
.flatMap {
|
||||||
redis.del(key(it))
|
redis.del(key(it))
|
||||||
}
|
}
|
||||||
.then()
|
.then()
|
||||||
}
|
}
|
||||||
|
|
||||||
fun toProto(id: TxId, value: T): ByteArray {
|
fun toProto(id: TxId, value: T): ByteArray {
|
||||||
val meta = buildMeta(id, value)
|
val meta = buildMeta(id, value)
|
||||||
|
|
||||||
return CachesProto.ValueContainer.newBuilder()
|
return CachesProto.ValueContainer.newBuilder()
|
||||||
.setType(valueType)
|
.setType(valueType)
|
||||||
.setValue(ByteString.copyFrom(serializeValue(value)))
|
.setValue(ByteString.copyFrom(serializeValue(value)))
|
||||||
.setTxMeta(meta)
|
.setTxMeta(meta)
|
||||||
.build()
|
.build()
|
||||||
.toByteArray()
|
.toByteArray()
|
||||||
}
|
}
|
||||||
|
|
||||||
open fun buildMeta(id: TxId, value: T): CachesProto.TxMeta.Builder {
|
open fun buildMeta(id: TxId, value: T): CachesProto.TxMeta.Builder {
|
||||||
return CachesProto.TxMeta.newBuilder()
|
return CachesProto.TxMeta.newBuilder()
|
||||||
.setHash(ByteString.copyFrom(id.value))
|
.setHash(ByteString.copyFrom(id.value))
|
||||||
}
|
}
|
||||||
|
|
||||||
abstract fun serializeValue(value: T): ByteArray
|
abstract fun serializeValue(value: T): ByteArray
|
||||||
@@ -110,43 +110,43 @@ abstract class OnTxRedisCache<T>(
|
|||||||
|
|
||||||
override fun read(key: TxId): Mono<T> {
|
override fun read(key: TxId): Mono<T> {
|
||||||
return redis.get(key(key))
|
return redis.get(key(key))
|
||||||
.map { data ->
|
.map { data ->
|
||||||
fromProto(data)
|
fromProto(data)
|
||||||
}.onErrorResume {
|
}.onErrorResume {
|
||||||
Mono.empty()
|
Mono.empty()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
open fun add(id: TxId, value: T, block: BlockContainer?, blockHeight: Long?): Mono<Void> {
|
open fun add(id: TxId, value: T, block: BlockContainer?, blockHeight: Long?): Mono<Void> {
|
||||||
return Mono.just(id)
|
return Mono.just(id)
|
||||||
.flatMap {
|
.flatMap {
|
||||||
val key = key(it)
|
val key = key(it)
|
||||||
val encodedValue = toProto(it, value)
|
val encodedValue = toProto(it, value)
|
||||||
val ttl = if (block?.timestamp != null) {
|
val ttl = if (block?.timestamp != null) {
|
||||||
cachingTime(block.timestamp)
|
cachingTime(block.timestamp)
|
||||||
} else {
|
} else {
|
||||||
cachingTime(blockHeight)
|
cachingTime(blockHeight)
|
||||||
}
|
|
||||||
//store
|
|
||||||
redis.setex(key, ttl, encodedValue)
|
|
||||||
}
|
}
|
||||||
.doOnError {
|
// store
|
||||||
log.warn("Failed to save TX to Redis: ${it.message}", it)
|
redis.setex(key, ttl, encodedValue)
|
||||||
}
|
}
|
||||||
//if failed to cache, just continue without it
|
.doOnError {
|
||||||
.onErrorResume {
|
log.warn("Failed to save TX to Redis: ${it.message}", it)
|
||||||
Mono.empty()
|
}
|
||||||
}
|
// if failed to cache, just continue without it
|
||||||
.then()
|
.onErrorResume {
|
||||||
|
Mono.empty()
|
||||||
|
}
|
||||||
|
.then()
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Calculate time to cache the value, based on block time
|
* Calculate time to cache the value, based on block time
|
||||||
*/
|
*/
|
||||||
fun cachingTime(blockTime: Instant): Long {
|
fun cachingTime(blockTime: Instant): Long {
|
||||||
//default caching time is age of the block, i.e. block create hour ago
|
// default caching time is age of the block, i.e. block create hour ago
|
||||||
//keep for hour, but block create 10 seconds ago cache for 10 seconds, as it
|
// keep for hour, but block create 10 seconds ago cache for 10 seconds, as it
|
||||||
//still can be replaced in the blockchain
|
// still can be replaced in the blockchain
|
||||||
val age = Instant.now().epochSecond - blockTime.epochSecond
|
val age = Instant.now().epochSecond - blockTime.epochSecond
|
||||||
return min(age, TimeUnit.HOURS.toSeconds(MAX_CACHE_TIME_HOURS))
|
return min(age, TimeUnit.HOURS.toSeconds(MAX_CACHE_TIME_HOURS))
|
||||||
}
|
}
|
||||||
@@ -165,4 +165,4 @@ abstract class OnTxRedisCache<T>(
|
|||||||
}
|
}
|
||||||
return min(confirmations * BLOCK_TIME_SECONDS, TimeUnit.HOURS.toSeconds(MAX_CACHE_TIME_HOURS))
|
return min(confirmations * BLOCK_TIME_SECONDS, TimeUnit.HOURS.toSeconds(MAX_CACHE_TIME_HOURS))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,8 +28,8 @@ import reactor.core.publisher.Mono
|
|||||||
* Keeps receipts for recent blocks in memory
|
* Keeps receipts for recent blocks in memory
|
||||||
*/
|
*/
|
||||||
open class ReceiptMemCache(
|
open class ReceiptMemCache(
|
||||||
// how many blocks to keeps in memory
|
// how many blocks to keeps in memory
|
||||||
val blocks: Int = 6
|
val blocks: Int = 6
|
||||||
) : Reader<TxId, ByteArray> {
|
) : Reader<TxId, ByteArray> {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -37,8 +37,8 @@ open class ReceiptMemCache(
|
|||||||
}
|
}
|
||||||
|
|
||||||
private val mapping = Caffeine.newBuilder()
|
private val mapping = Caffeine.newBuilder()
|
||||||
.maximumSize(blocks * 200L)
|
.maximumSize(blocks * 200L)
|
||||||
.build<TxId, ByteArray>()
|
.build<TxId, ByteArray>()
|
||||||
|
|
||||||
open fun evict(block: BlockContainer) {
|
open fun evict(block: BlockContainer) {
|
||||||
block.transactions.forEach {
|
block.transactions.forEach {
|
||||||
@@ -60,5 +60,4 @@ open class ReceiptMemCache(
|
|||||||
open fun acceptsRecentBlocks(heightDelta: Long): Boolean {
|
open fun acceptsRecentBlocks(heightDelta: Long): Boolean {
|
||||||
return blocks <= heightDelta && heightDelta >= 0
|
return blocks <= heightDelta && heightDelta >= 0
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -16,16 +16,15 @@
|
|||||||
package io.emeraldpay.dshackle.cache
|
package io.emeraldpay.dshackle.cache
|
||||||
|
|
||||||
import io.emeraldpay.dshackle.data.DefaultContainer
|
import io.emeraldpay.dshackle.data.DefaultContainer
|
||||||
import io.emeraldpay.dshackle.data.TxId
|
|
||||||
import io.emeraldpay.dshackle.proto.CachesProto
|
import io.emeraldpay.dshackle.proto.CachesProto
|
||||||
import io.emeraldpay.grpc.Chain
|
|
||||||
import io.emeraldpay.etherjar.rpc.json.TransactionReceiptJson
|
import io.emeraldpay.etherjar.rpc.json.TransactionReceiptJson
|
||||||
|
import io.emeraldpay.grpc.Chain
|
||||||
import io.lettuce.core.api.reactive.RedisReactiveCommands
|
import io.lettuce.core.api.reactive.RedisReactiveCommands
|
||||||
import reactor.core.publisher.Mono
|
import reactor.core.publisher.Mono
|
||||||
|
|
||||||
open class ReceiptRedisCache(
|
open class ReceiptRedisCache(
|
||||||
redis: RedisReactiveCommands<String, ByteArray>,
|
redis: RedisReactiveCommands<String, ByteArray>,
|
||||||
chain: Chain
|
chain: Chain
|
||||||
) : OnTxRedisCache<ByteArray>(redis, chain, CachesProto.ValueContainer.ValueType.TX_RECEIPT) {
|
) : OnTxRedisCache<ByteArray>(redis, chain, CachesProto.ValueContainer.ValueType.TX_RECEIPT) {
|
||||||
|
|
||||||
override fun deserializeValue(value: CachesProto.ValueContainer): ByteArray {
|
override fun deserializeValue(value: CachesProto.ValueContainer): ByteArray {
|
||||||
@@ -39,4 +38,4 @@ open class ReceiptRedisCache(
|
|||||||
fun add(json: DefaultContainer<TransactionReceiptJson>): Mono<Void> {
|
fun add(json: DefaultContainer<TransactionReceiptJson>): Mono<Void> {
|
||||||
return super.add(json.txId!!, json.json!!, null, json.height)
|
return super.add(json.txId!!, json.json!!, null, json.height)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,8 +28,8 @@ import reactor.core.publisher.Mono
|
|||||||
* Memory cache for transactions
|
* Memory cache for transactions
|
||||||
*/
|
*/
|
||||||
open class TxMemCache(
|
open class TxMemCache(
|
||||||
// usually there is 100-150 tx per block on Ethereum, we keep data for about 32 blocks by default
|
// usually there is 100-150 tx per block on Ethereum, we keep data for about 32 blocks by default
|
||||||
private val maxSize: Int = 125 * 32
|
private val maxSize: Int = 125 * 32
|
||||||
) : Reader<TxId, TxContainer> {
|
) : Reader<TxId, TxContainer> {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -37,8 +37,8 @@ open class TxMemCache(
|
|||||||
}
|
}
|
||||||
|
|
||||||
private val mapping = Caffeine.newBuilder()
|
private val mapping = Caffeine.newBuilder()
|
||||||
.maximumSize(maxSize.toLong())
|
.maximumSize(maxSize.toLong())
|
||||||
.build<TxId, TxContainer>()
|
.build<TxId, TxContainer>()
|
||||||
|
|
||||||
override fun read(key: TxId): Mono<TxContainer> {
|
override fun read(key: TxId): Mono<TxContainer> {
|
||||||
return Mono.justOrEmpty(mapping.getIfPresent(key))
|
return Mono.justOrEmpty(mapping.getIfPresent(key))
|
||||||
@@ -52,13 +52,13 @@ open class TxMemCache(
|
|||||||
|
|
||||||
open fun evict(block: BlockId) {
|
open fun evict(block: BlockId) {
|
||||||
val ids = mapping.asMap()
|
val ids = mapping.asMap()
|
||||||
.filter { it.value.blockId == block }
|
.filter { it.value.blockId == block }
|
||||||
.map { it.key }
|
.map { it.key }
|
||||||
mapping.invalidateAll(ids)
|
mapping.invalidateAll(ids)
|
||||||
}
|
}
|
||||||
|
|
||||||
open fun add(tx: TxContainer) {
|
open fun add(tx: TxContainer) {
|
||||||
//do not cache fresh transactions
|
// do not cache fresh transactions
|
||||||
if (tx.blockId == null) {
|
if (tx.blockId == null) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -68,4 +68,4 @@ open class TxMemCache(
|
|||||||
open fun purge() {
|
open fun purge() {
|
||||||
mapping.cleanUp()
|
mapping.cleanUp()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,31 +15,26 @@
|
|||||||
*/
|
*/
|
||||||
package io.emeraldpay.dshackle.cache
|
package io.emeraldpay.dshackle.cache
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper
|
|
||||||
import com.google.protobuf.ByteString
|
import com.google.protobuf.ByteString
|
||||||
import io.emeraldpay.dshackle.data.BlockContainer
|
import io.emeraldpay.dshackle.data.BlockContainer
|
||||||
import io.emeraldpay.dshackle.data.BlockId
|
import io.emeraldpay.dshackle.data.BlockId
|
||||||
import io.emeraldpay.dshackle.data.TxContainer
|
import io.emeraldpay.dshackle.data.TxContainer
|
||||||
import io.emeraldpay.dshackle.data.TxId
|
import io.emeraldpay.dshackle.data.TxId
|
||||||
|
import io.emeraldpay.dshackle.proto.CachesProto
|
||||||
import io.emeraldpay.dshackle.reader.Reader
|
import io.emeraldpay.dshackle.reader.Reader
|
||||||
import io.emeraldpay.grpc.Chain
|
import io.emeraldpay.grpc.Chain
|
||||||
import io.emeraldpay.dshackle.proto.CachesProto
|
|
||||||
import io.lettuce.core.api.reactive.RedisReactiveCommands
|
import io.lettuce.core.api.reactive.RedisReactiveCommands
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import reactor.core.publisher.Mono
|
import reactor.core.publisher.Mono
|
||||||
import reactor.util.function.Tuples
|
|
||||||
import java.time.Instant
|
|
||||||
import java.util.concurrent.TimeUnit
|
|
||||||
import kotlin.math.min
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Cache transactions in Redis, up to 24 hours.
|
* Cache transactions in Redis, up to 24 hours.
|
||||||
*/
|
*/
|
||||||
open class TxRedisCache(
|
open class TxRedisCache(
|
||||||
private val redis: RedisReactiveCommands<String, ByteArray>,
|
private val redis: RedisReactiveCommands<String, ByteArray>,
|
||||||
private val chain: Chain
|
private val chain: Chain
|
||||||
) : Reader<TxId, TxContainer>,
|
) : Reader<TxId, TxContainer>,
|
||||||
OnTxRedisCache<TxContainer>(redis, chain, CachesProto.ValueContainer.ValueType.TX) {
|
OnTxRedisCache<TxContainer>(redis, chain, CachesProto.ValueContainer.ValueType.TX) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private val log = LoggerFactory.getLogger(TxRedisCache::class.java)
|
private val log = LoggerFactory.getLogger(TxRedisCache::class.java)
|
||||||
@@ -67,15 +62,14 @@ open class TxRedisCache(
|
|||||||
}
|
}
|
||||||
val meta = value.txMeta
|
val meta = value.txMeta
|
||||||
return TxContainer(
|
return TxContainer(
|
||||||
meta.height,
|
meta.height,
|
||||||
TxId(meta.hash.toByteArray()),
|
TxId(meta.hash.toByteArray()),
|
||||||
BlockId(meta.blockHash.toByteArray()),
|
BlockId(meta.blockHash.toByteArray()),
|
||||||
value.value.toByteArray()
|
value.value.toByteArray()
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
open fun add(tx: TxContainer, block: BlockContainer): Mono<Void> {
|
open fun add(tx: TxContainer, block: BlockContainer): Mono<Void> {
|
||||||
return super.add(tx.hash, tx, block, tx.height)
|
return super.add(tx.hash, tx, block, tx.height)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
package io.emeraldpay.dshackle.config
|
package io.emeraldpay.dshackle.config
|
||||||
|
|
||||||
class AccessLogConfig(
|
class AccessLogConfig(
|
||||||
val enabled: Boolean = false
|
val enabled: Boolean = false
|
||||||
) {
|
) {
|
||||||
|
|
||||||
var filename: String = "./access_log.jsonl"
|
var filename: String = "./access_log.jsonl"
|
||||||
@@ -14,9 +14,8 @@ class AccessLogConfig(
|
|||||||
|
|
||||||
fun disabled(): AccessLogConfig {
|
fun disabled(): AccessLogConfig {
|
||||||
return AccessLogConfig(
|
return AccessLogConfig(
|
||||||
enabled = false
|
enabled = false
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -23,5 +23,4 @@ class AccessLogReader : YamlConfigReader(), ConfigReader<AccessLogConfig> {
|
|||||||
}
|
}
|
||||||
} ?: AccessLogConfig.default()
|
} ?: AccessLogConfig.default()
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -29,14 +29,14 @@ class AuthConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
class ClientBasicAuth(
|
class ClientBasicAuth(
|
||||||
val username: String,
|
val username: String,
|
||||||
val password: String
|
val password: String
|
||||||
) : ClientAuth()
|
) : ClientAuth()
|
||||||
|
|
||||||
class ClientTlsAuth(
|
class ClientTlsAuth(
|
||||||
var ca: String? = null,
|
var ca: String? = null,
|
||||||
var certificate: String? = null,
|
var certificate: String? = null,
|
||||||
var key: String? = null
|
var key: String? = null
|
||||||
) : ClientAuth()
|
) : ClientAuth()
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -58,4 +58,4 @@ class AuthConfig {
|
|||||||
var clientRequire: Boolean? = null
|
var clientRequire: Boolean? = null
|
||||||
var clientCa: String? = null
|
var clientCa: String? = null
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -82,5 +82,4 @@ class AuthConfigReader : YamlConfigReader() {
|
|||||||
auth
|
auth
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -17,12 +17,12 @@ package io.emeraldpay.dshackle.config
|
|||||||
|
|
||||||
class CacheConfig {
|
class CacheConfig {
|
||||||
|
|
||||||
var redis: Redis? = null;
|
var redis: Redis? = null
|
||||||
|
|
||||||
class Redis(
|
class Redis(
|
||||||
var host: String = "127.0.0.1",
|
var host: String = "127.0.0.1",
|
||||||
var port: Int = 6379,
|
var port: Int = 6379,
|
||||||
var db: Int? = 0,
|
var db: Int? = 0,
|
||||||
var password: String? = null
|
var password: String? = null
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -58,4 +58,4 @@ class CacheConfigReader : YamlConfigReader(), ConfigReader<CacheConfig> {
|
|||||||
config
|
config
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,5 +20,4 @@ import org.yaml.snakeyaml.nodes.MappingNode
|
|||||||
interface ConfigReader<T> {
|
interface ConfigReader<T> {
|
||||||
|
|
||||||
fun read(input: MappingNode?): T?
|
fun read(input: MappingNode?): T?
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -31,4 +31,4 @@ class EnvVariables {
|
|||||||
} ?: ""
|
} ?: ""
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,11 +19,11 @@ package io.emeraldpay.dshackle.config
|
|||||||
import org.yaml.snakeyaml.error.Mark
|
import org.yaml.snakeyaml.error.Mark
|
||||||
|
|
||||||
open class InvalidConfigException(
|
open class InvalidConfigException(
|
||||||
message: String
|
message: String
|
||||||
) : Exception(message)
|
) : Exception(message)
|
||||||
|
|
||||||
class InvalidConfigYamlException(
|
class InvalidConfigYamlException(
|
||||||
filename: String,
|
filename: String,
|
||||||
mark: Mark,
|
mark: Mark,
|
||||||
message: String
|
message: String
|
||||||
) : InvalidConfigException("Invalid YAML configuration ${message}, at ${filename}:${mark.line}")
|
) : InvalidConfigException("Invalid YAML configuration $message, at $filename:${mark.line}")
|
||||||
|
|||||||
@@ -25,4 +25,4 @@ class MainConfig {
|
|||||||
var tokens: TokensConfig? = null
|
var tokens: TokensConfig? = null
|
||||||
var monitoring: MonitoringConfig = MonitoringConfig.default()
|
var monitoring: MonitoringConfig = MonitoringConfig.default()
|
||||||
var accessLogConfig: AccessLogConfig = AccessLogConfig.default()
|
var accessLogConfig: AccessLogConfig = AccessLogConfig.default()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import org.yaml.snakeyaml.nodes.MappingNode
|
|||||||
import java.io.InputStream
|
import java.io.InputStream
|
||||||
|
|
||||||
class MainConfigReader(
|
class MainConfigReader(
|
||||||
fileResolver: FileResolver
|
fileResolver: FileResolver
|
||||||
) : YamlConfigReader(), ConfigReader<MainConfig> {
|
) : YamlConfigReader(), ConfigReader<MainConfig> {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -73,5 +73,4 @@ class MainConfigReader(
|
|||||||
}
|
}
|
||||||
return config
|
return config
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -16,14 +16,15 @@
|
|||||||
package io.emeraldpay.dshackle.config
|
package io.emeraldpay.dshackle.config
|
||||||
|
|
||||||
class MonitoringConfig(
|
class MonitoringConfig(
|
||||||
val enabled: Boolean,
|
val enabled: Boolean,
|
||||||
val prometheus: PrometheusConfig
|
val prometheus: PrometheusConfig
|
||||||
) {
|
) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
fun default(): MonitoringConfig {
|
fun default(): MonitoringConfig {
|
||||||
return MonitoringConfig(true, PrometheusConfig.default())
|
return MonitoringConfig(true, PrometheusConfig.default())
|
||||||
}
|
}
|
||||||
|
|
||||||
fun disabled(): MonitoringConfig {
|
fun disabled(): MonitoringConfig {
|
||||||
return MonitoringConfig(false, PrometheusConfig.disabled())
|
return MonitoringConfig(false, PrometheusConfig.disabled())
|
||||||
}
|
}
|
||||||
@@ -33,19 +34,19 @@ class MonitoringConfig(
|
|||||||
var enableExtended: Boolean = false
|
var enableExtended: Boolean = false
|
||||||
|
|
||||||
data class PrometheusConfig(
|
data class PrometheusConfig(
|
||||||
val enabled: Boolean,
|
val enabled: Boolean,
|
||||||
val path: String,
|
val path: String,
|
||||||
val host: String,
|
val host: String,
|
||||||
val port: Int
|
val port: Int
|
||||||
) {
|
) {
|
||||||
companion object {
|
companion object {
|
||||||
fun default(): PrometheusConfig {
|
fun default(): PrometheusConfig {
|
||||||
return PrometheusConfig(true, "/metrics", "127.0.0.1", 8081)
|
return PrometheusConfig(true, "/metrics", "127.0.0.1", 8081)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun disabled(): PrometheusConfig {
|
fun disabled(): PrometheusConfig {
|
||||||
return PrometheusConfig(false, "/", "127.0.0.1", 0)
|
return PrometheusConfig(false, "/", "127.0.0.1", 0)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ import org.slf4j.LoggerFactory
|
|||||||
import org.yaml.snakeyaml.nodes.MappingNode
|
import org.yaml.snakeyaml.nodes.MappingNode
|
||||||
import java.io.InputStream
|
import java.io.InputStream
|
||||||
|
|
||||||
class MonitoringConfigReader: YamlConfigReader(), ConfigReader<MonitoringConfig> {
|
class MonitoringConfigReader : YamlConfigReader(), ConfigReader<MonitoringConfig> {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private val log = LoggerFactory.getLogger(MonitoringConfigReader::class.java)
|
private val log = LoggerFactory.getLogger(MonitoringConfigReader::class.java)
|
||||||
@@ -64,5 +64,4 @@ class MonitoringConfigReader: YamlConfigReader(), ConfigReader<MonitoringConfig>
|
|||||||
val port = getValueAsInt(input, "port") ?: default.port
|
val port = getValueAsInt(input, "port") ?: default.port
|
||||||
return MonitoringConfig.PrometheusConfig(enabled, path, host, port)
|
return MonitoringConfig.PrometheusConfig(enabled, path, host, port)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ import io.emeraldpay.grpc.Chain
|
|||||||
open class ProxyConfig {
|
open class ProxyConfig {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
public const val CONFIG_ID = "parsed.proxy"
|
const val CONFIG_ID = "parsed.proxy"
|
||||||
}
|
}
|
||||||
|
|
||||||
var enabled: Boolean = true
|
var enabled: Boolean = true
|
||||||
@@ -50,13 +50,13 @@ open class ProxyConfig {
|
|||||||
var routes: List<Route> = ArrayList()
|
var routes: List<Route> = ArrayList()
|
||||||
|
|
||||||
class Route(
|
class Route(
|
||||||
/**
|
/**
|
||||||
* URL binding for the route. http://$host:$port/$id
|
* URL binding for the route. http://$host:$port/$id
|
||||||
*/
|
*/
|
||||||
val id: String,
|
val id: String,
|
||||||
/**
|
/**
|
||||||
* Blockchain to dispatch requests
|
* Blockchain to dispatch requests
|
||||||
*/
|
*/
|
||||||
val blockchain: Chain
|
val blockchain: Chain
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,10 +19,8 @@ package io.emeraldpay.dshackle.config
|
|||||||
import io.emeraldpay.grpc.Chain
|
import io.emeraldpay.grpc.Chain
|
||||||
import org.apache.commons.lang3.StringUtils
|
import org.apache.commons.lang3.StringUtils
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.yaml.snakeyaml.Yaml
|
|
||||||
import org.yaml.snakeyaml.nodes.MappingNode
|
import org.yaml.snakeyaml.nodes.MappingNode
|
||||||
import java.io.InputStream
|
import java.io.InputStream
|
||||||
import java.io.InputStreamReader
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Read YAML config, part related to Proxy configuration
|
* Read YAML config, part related to Proxy configuration
|
||||||
@@ -84,5 +82,4 @@ class ProxyConfigReader : YamlConfigReader(), ConfigReader<ProxyConfig> {
|
|||||||
config.tls = authConfigReader.readServerTls(input)
|
config.tls = authConfigReader.readServerTls(input)
|
||||||
return config
|
return config
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -15,12 +15,12 @@
|
|||||||
*/
|
*/
|
||||||
package io.emeraldpay.dshackle.config
|
package io.emeraldpay.dshackle.config
|
||||||
|
|
||||||
|
import io.emeraldpay.etherjar.domain.Address
|
||||||
import io.emeraldpay.grpc.BlockchainType
|
import io.emeraldpay.grpc.BlockchainType
|
||||||
import io.emeraldpay.grpc.Chain
|
import io.emeraldpay.grpc.Chain
|
||||||
import io.emeraldpay.etherjar.domain.Address
|
|
||||||
|
|
||||||
class TokensConfig(
|
class TokensConfig(
|
||||||
val tokens: List<Token>
|
val tokens: List<Token>
|
||||||
) {
|
) {
|
||||||
|
|
||||||
class Token {
|
class Token {
|
||||||
@@ -30,7 +30,7 @@ class TokensConfig(
|
|||||||
|
|
||||||
// coin name
|
// coin name
|
||||||
var name: String? = null
|
var name: String? = null
|
||||||
var type: Type? = null;
|
var type: Type? = null
|
||||||
var address: String? = null
|
var address: String? = null
|
||||||
|
|
||||||
fun validate(): String? {
|
fun validate(): String? {
|
||||||
@@ -40,9 +40,9 @@ class TokensConfig(
|
|||||||
name.isNullOrBlank() -> "name"
|
name.isNullOrBlank() -> "name"
|
||||||
type == null -> type
|
type == null -> type
|
||||||
address.isNullOrBlank() -> "address"
|
address.isNullOrBlank() -> "address"
|
||||||
blockchain != null
|
blockchain != null &&
|
||||||
&& BlockchainType.from(blockchain!!) == BlockchainType.ETHEREUM
|
BlockchainType.from(blockchain!!) == BlockchainType.ETHEREUM &&
|
||||||
&& !Address.isValidAddress(address) -> "address"
|
!Address.isValidAddress(address) -> "address"
|
||||||
else -> null
|
else -> null
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -51,5 +51,4 @@ class TokensConfig(
|
|||||||
enum class Type {
|
enum class Type {
|
||||||
ERC20
|
ERC20
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ package io.emeraldpay.dshackle.config
|
|||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.yaml.snakeyaml.nodes.MappingNode
|
import org.yaml.snakeyaml.nodes.MappingNode
|
||||||
import java.io.InputStream
|
import java.io.InputStream
|
||||||
import java.util.*
|
import java.util.Locale
|
||||||
|
|
||||||
class TokensConfigReader : YamlConfigReader(), ConfigReader<TokensConfig> {
|
class TokensConfigReader : YamlConfigReader(), ConfigReader<TokensConfig> {
|
||||||
|
|
||||||
@@ -60,5 +60,4 @@ class TokensConfigReader : YamlConfigReader(), ConfigReader<TokensConfig> {
|
|||||||
TokensConfig(it)
|
TokensConfig(it)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -17,11 +17,9 @@
|
|||||||
package io.emeraldpay.dshackle.config
|
package io.emeraldpay.dshackle.config
|
||||||
|
|
||||||
import io.emeraldpay.dshackle.Defaults
|
import io.emeraldpay.dshackle.Defaults
|
||||||
import java.lang.ClassCastException
|
|
||||||
import java.net.URI
|
import java.net.URI
|
||||||
import java.util.*
|
import java.util.Arrays
|
||||||
import kotlin.collections.ArrayList
|
import java.util.Locale
|
||||||
import kotlin.collections.HashMap
|
|
||||||
|
|
||||||
open class UpstreamsConfig {
|
open class UpstreamsConfig {
|
||||||
var defaultOptions: MutableList<DefaultOptions> = ArrayList<DefaultOptions>()
|
var defaultOptions: MutableList<DefaultOptions> = ArrayList<DefaultOptions>()
|
||||||
@@ -46,8 +44,10 @@ open class UpstreamsConfig {
|
|||||||
}
|
}
|
||||||
val copy = Options()
|
val copy = Options()
|
||||||
copy.minPeers = if (this.minPeers != null) this.minPeers else additional.minPeers
|
copy.minPeers = if (this.minPeers != null) this.minPeers else additional.minPeers
|
||||||
copy.disableValidation = if (this.disableValidation != null) this.disableValidation else additional.disableValidation
|
copy.disableValidation =
|
||||||
copy.providesBalance = if (this.providesBalance != null) this.providesBalance else additional.providesBalance
|
if (this.disableValidation != null) this.disableValidation else additional.disableValidation
|
||||||
|
copy.providesBalance =
|
||||||
|
if (this.providesBalance != null) this.providesBalance else additional.providesBalance
|
||||||
return copy
|
return copy
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -60,7 +60,6 @@ open class UpstreamsConfig {
|
|||||||
return options
|
return options
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
class DefaultOptions : Options() {
|
class DefaultOptions : Options() {
|
||||||
@@ -125,14 +124,14 @@ open class UpstreamsConfig {
|
|||||||
var msgSize: Int? = null
|
var msgSize: Int? = null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO make it unmodifiable after initial load
|
||||||
//TODO make it unmodifiable after initial load
|
class Labels : HashMap<String, String>() {
|
||||||
class Labels: HashMap<String, String>() {
|
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@JvmStatic fun fromMap(map: Map<String, String>): Labels {
|
@JvmStatic
|
||||||
|
fun fromMap(map: Map<String, String>): Labels {
|
||||||
val labels = Labels()
|
val labels = Labels()
|
||||||
map.entries.forEach() { kv ->
|
map.entries.forEach { kv ->
|
||||||
labels.put(kv.key, kv.value)
|
labels.put(kv.key, kv.value)
|
||||||
}
|
}
|
||||||
return labels
|
return labels
|
||||||
@@ -140,7 +139,7 @@ open class UpstreamsConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
enum class UpstreamType private constructor(vararg code: String) {
|
enum class UpstreamType(vararg code: String) {
|
||||||
ETHEREUM_JSON_RPC("ethereum"),
|
ETHEREUM_JSON_RPC("ethereum"),
|
||||||
BITCOIN_JSON_RPC("bitcoin"),
|
BITCOIN_JSON_RPC("bitcoin"),
|
||||||
DSHACKLE("dshackle", "grpc"),
|
DSHACKLE("dshackle", "grpc"),
|
||||||
@@ -168,12 +167,12 @@ open class UpstreamsConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
class Methods(
|
class Methods(
|
||||||
val enabled: Set<Method>,
|
val enabled: Set<Method>,
|
||||||
val disabled: Set<Method>
|
val disabled: Set<Method>
|
||||||
)
|
)
|
||||||
|
|
||||||
class Method(
|
class Method(
|
||||||
val name: String,
|
val name: String,
|
||||||
val quorum: String? = null
|
val quorum: String? = null
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,14 +23,12 @@ import org.yaml.snakeyaml.nodes.MappingNode
|
|||||||
import org.yaml.snakeyaml.nodes.ScalarNode
|
import org.yaml.snakeyaml.nodes.ScalarNode
|
||||||
import reactor.util.function.Tuples
|
import reactor.util.function.Tuples
|
||||||
import java.io.InputStream
|
import java.io.InputStream
|
||||||
import java.lang.IllegalArgumentException
|
|
||||||
import java.net.URI
|
import java.net.URI
|
||||||
import java.time.Duration
|
import java.time.Duration
|
||||||
import java.util.*
|
import java.util.Locale
|
||||||
import kotlin.collections.ArrayList
|
|
||||||
|
|
||||||
class UpstreamsConfigReader(
|
class UpstreamsConfigReader(
|
||||||
private val fileResolver: FileResolver
|
private val fileResolver: FileResolver
|
||||||
) : YamlConfigReader(), ConfigReader<UpstreamsConfig> {
|
) : YamlConfigReader(), ConfigReader<UpstreamsConfig> {
|
||||||
|
|
||||||
private val log = LoggerFactory.getLogger(UpstreamsConfigReader::class.java)
|
private val log = LoggerFactory.getLogger(UpstreamsConfigReader::class.java)
|
||||||
@@ -198,7 +196,10 @@ class UpstreamsConfigReader(
|
|||||||
upstream.methods = tryReadMethods(upNode)
|
upstream.methods = tryReadMethods(upNode)
|
||||||
}
|
}
|
||||||
|
|
||||||
internal fun readUpstreamGrpc(upNode: MappingNode, upstream: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>) {
|
internal fun readUpstreamGrpc(
|
||||||
|
upNode: MappingNode,
|
||||||
|
upstream: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>
|
||||||
|
) {
|
||||||
if (hasAny(upNode, "labels")) {
|
if (hasAny(upNode, "labels")) {
|
||||||
log.warn("Labels should be not applied to gRPC upstream")
|
log.warn("Labels should be not applied to gRPC upstream")
|
||||||
}
|
}
|
||||||
@@ -221,13 +222,13 @@ class UpstreamsConfigReader(
|
|||||||
if (hasAny(upNode, "labels")) {
|
if (hasAny(upNode, "labels")) {
|
||||||
getMapping(upNode, "labels")?.let { labels ->
|
getMapping(upNode, "labels")?.let { labels ->
|
||||||
labels.value.stream()
|
labels.value.stream()
|
||||||
.filter { n -> n.keyNode is ScalarNode && n.valueNode is ScalarNode }
|
.filter { n -> n.keyNode is ScalarNode && n.valueNode is ScalarNode }
|
||||||
.map { n -> Tuples.of((n.keyNode as ScalarNode).value, (n.valueNode as ScalarNode).value) }
|
.map { n -> Tuples.of((n.keyNode as ScalarNode).value, (n.valueNode as ScalarNode).value) }
|
||||||
.map { kv -> Tuples.of(kv.t1.trim(), kv.t2.trim()) }
|
.map { kv -> Tuples.of(kv.t1.trim(), kv.t2.trim()) }
|
||||||
.filter { kv -> StringUtils.isNotEmpty(kv.t1) && StringUtils.isNotEmpty(kv.t2) }
|
.filter { kv -> StringUtils.isNotEmpty(kv.t1) && StringUtils.isNotEmpty(kv.t2) }
|
||||||
.forEach { kv ->
|
.forEach { kv ->
|
||||||
upstream.labels[kv.t1] = kv.t2
|
upstream.labels[kv.t1] = kv.t2
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -247,21 +248,21 @@ class UpstreamsConfigReader(
|
|||||||
val enabled = getList<MappingNode>(mnode, "enabled")?.value?.map { m ->
|
val enabled = getList<MappingNode>(mnode, "enabled")?.value?.map { m ->
|
||||||
getValueAsString(m, "name")?.let { name ->
|
getValueAsString(m, "name")?.let { name ->
|
||||||
UpstreamsConfig.Method(
|
UpstreamsConfig.Method(
|
||||||
name = name,
|
name = name,
|
||||||
quorum = getValueAsString(m, "quorum")
|
quorum = getValueAsString(m, "quorum")
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}?.filterNotNull()?.toSet() ?: emptySet()
|
}?.filterNotNull()?.toSet() ?: emptySet()
|
||||||
val disabled = getList<MappingNode>(mnode, "disabled")?.value?.map { m ->
|
val disabled = getList<MappingNode>(mnode, "disabled")?.value?.map { m ->
|
||||||
getValueAsString(m, "name")?.let { name ->
|
getValueAsString(m, "name")?.let { name ->
|
||||||
UpstreamsConfig.Method(
|
UpstreamsConfig.Method(
|
||||||
name = name
|
name = name
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}?.filterNotNull()?.toSet() ?: emptySet()
|
}?.filterNotNull()?.toSet() ?: emptySet()
|
||||||
|
|
||||||
UpstreamsConfig.Methods(
|
UpstreamsConfig.Methods(
|
||||||
enabled, disabled
|
enabled, disabled
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -282,5 +283,4 @@ class UpstreamsConfigReader(
|
|||||||
}
|
}
|
||||||
return options
|
return options
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ import org.yaml.snakeyaml.nodes.Node
|
|||||||
import org.yaml.snakeyaml.nodes.ScalarNode
|
import org.yaml.snakeyaml.nodes.ScalarNode
|
||||||
import java.io.InputStream
|
import java.io.InputStream
|
||||||
import java.io.InputStreamReader
|
import java.io.InputStreamReader
|
||||||
import java.util.*
|
import java.util.Locale
|
||||||
|
|
||||||
abstract class YamlConfigReader {
|
abstract class YamlConfigReader {
|
||||||
private val envVariables = EnvVariables()
|
private val envVariables = EnvVariables()
|
||||||
@@ -43,12 +43,12 @@ abstract class YamlConfigReader {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
return mappingNode.value
|
return mappingNode.value
|
||||||
.stream()
|
.stream()
|
||||||
.filter { n -> n.keyNode is ScalarNode }
|
.filter { n -> n.keyNode is ScalarNode }
|
||||||
.filter { n ->
|
.filter { n ->
|
||||||
val sn = n.keyNode as ScalarNode
|
val sn = n.keyNode as ScalarNode
|
||||||
key == sn.value
|
key == sn.value
|
||||||
}.count() > 0
|
}.count() > 0
|
||||||
}
|
}
|
||||||
|
|
||||||
@Suppress("UNCHECKED_CAST")
|
@Suppress("UNCHECKED_CAST")
|
||||||
@@ -57,20 +57,20 @@ abstract class YamlConfigReader {
|
|||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
return mappingNode.value
|
return mappingNode.value
|
||||||
.stream()
|
.stream()
|
||||||
.filter { n -> n.keyNode is ScalarNode && type.isAssignableFrom(n.valueNode.javaClass) }
|
.filter { n -> n.keyNode is ScalarNode && type.isAssignableFrom(n.valueNode.javaClass) }
|
||||||
.filter { n ->
|
.filter { n ->
|
||||||
val sn = n.keyNode as ScalarNode
|
val sn = n.keyNode as ScalarNode
|
||||||
key == sn.value
|
key == sn.value
|
||||||
}
|
}
|
||||||
.map { n -> n.valueNode as T }
|
.map { n -> n.valueNode as T }
|
||||||
.findFirst().let {
|
.findFirst().let {
|
||||||
if (it.isPresent) {
|
if (it.isPresent) {
|
||||||
it.get()
|
it.get()
|
||||||
} else {
|
} else {
|
||||||
null
|
null
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected fun getMapping(mappingNode: MappingNode?, key: String): MappingNode? {
|
protected fun getMapping(mappingNode: MappingNode?, key: String): MappingNode? {
|
||||||
@@ -89,8 +89,8 @@ abstract class YamlConfigReader {
|
|||||||
|
|
||||||
protected fun getListOfString(mappingNode: MappingNode?, key: String): List<String>? {
|
protected fun getListOfString(mappingNode: MappingNode?, key: String): List<String>? {
|
||||||
return getList<ScalarNode>(mappingNode, key)?.value
|
return getList<ScalarNode>(mappingNode, key)?.value
|
||||||
?.map { it.value }
|
?.map { it.value }
|
||||||
?.map(envVariables::postProcess)
|
?.map(envVariables::postProcess)
|
||||||
}
|
}
|
||||||
|
|
||||||
protected fun getValueAsString(mappingNode: MappingNode?, key: String): String? {
|
protected fun getValueAsString(mappingNode: MappingNode?, key: String): String? {
|
||||||
@@ -130,7 +130,7 @@ abstract class YamlConfigReader {
|
|||||||
fun getValueAsBytes(mappingNode: MappingNode?, key: String): Int? {
|
fun getValueAsBytes(mappingNode: MappingNode?, key: String): Int? {
|
||||||
return getValueAsString(mappingNode, key)?.let(envVariables::postProcess)?.let {
|
return getValueAsString(mappingNode, key)?.let(envVariables::postProcess)?.let {
|
||||||
val m = Regex("^(\\d+)(m|mb|k|kb|b)?$").find(it.lowercase().trim())
|
val m = Regex("^(\\d+)(m|mb|k|kb|b)?$").find(it.lowercase().trim())
|
||||||
?: throw IllegalArgumentException("Not a data size: ${it}. Example of correct values: '1024', '1kb', '5mb'")
|
?: throw IllegalArgumentException("Not a data size: $it. Example of correct values: '1024', '1kb', '5mb'")
|
||||||
val multiplier = m.groups[2]?.let {
|
val multiplier = m.groups[2]?.let {
|
||||||
when (it.value) {
|
when (it.value) {
|
||||||
"k", "kb" -> 1024
|
"k", "kb" -> 1024
|
||||||
@@ -147,9 +147,9 @@ abstract class YamlConfigReader {
|
|||||||
|
|
||||||
fun getBlockchain(id: String): Chain {
|
fun getBlockchain(id: String): Chain {
|
||||||
return Chain.values().find { chain ->
|
return Chain.values().find { chain ->
|
||||||
chain.name == id.uppercase(Locale.getDefault())
|
chain.name == id.uppercase(Locale.getDefault()) ||
|
||||||
|| chain.chainCode.uppercase(Locale.getDefault()) == id.uppercase(Locale.getDefault())
|
chain.chainCode.uppercase(Locale.getDefault()) == id.uppercase(Locale.getDefault()) ||
|
||||||
|| chain.id.toString() == id
|
chain.id.toString() == id
|
||||||
} ?: Chain.UNSPECIFIED
|
} ?: Chain.UNSPECIFIED
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,14 +23,14 @@ import java.math.BigInteger
|
|||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
|
|
||||||
class BlockContainer(
|
class BlockContainer(
|
||||||
val height: Long,
|
val height: Long,
|
||||||
val hash: BlockId,
|
val hash: BlockId,
|
||||||
val difficulty: BigInteger,
|
val difficulty: BigInteger,
|
||||||
val timestamp: Instant,
|
val timestamp: Instant,
|
||||||
val full: Boolean,
|
val full: Boolean,
|
||||||
json: ByteArray?,
|
json: ByteArray?,
|
||||||
val parsed: Any?,
|
val parsed: Any?,
|
||||||
val transactions: List<TxId> = emptyList()
|
val transactions: List<TxId> = emptyList()
|
||||||
) : SourceContainer(json, parsed) {
|
) : SourceContainer(json, parsed) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -38,14 +38,14 @@ class BlockContainer(
|
|||||||
fun from(block: BlockJson<*>, raw: ByteArray): BlockContainer {
|
fun from(block: BlockJson<*>, raw: ByteArray): BlockContainer {
|
||||||
val hasTransactions = block.transactions?.filterIsInstance<TransactionJson>()?.count() ?: 0 > 0
|
val hasTransactions = block.transactions?.filterIsInstance<TransactionJson>()?.count() ?: 0 > 0
|
||||||
return BlockContainer(
|
return BlockContainer(
|
||||||
block.number,
|
block.number,
|
||||||
BlockId.from(block),
|
BlockId.from(block),
|
||||||
block.totalDifficulty,
|
block.totalDifficulty,
|
||||||
block.timestamp,
|
block.timestamp,
|
||||||
hasTransactions,
|
hasTransactions,
|
||||||
raw,
|
raw,
|
||||||
block,
|
block,
|
||||||
block.transactions?.map { TxId.from(it.hash) } ?: emptyList()
|
block.transactions?.map { TxId.from(it.hash) } ?: emptyList()
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -88,6 +88,4 @@ class BlockContainer(
|
|||||||
result = 31 * result + hash.hashCode()
|
result = 31 * result + hash.hashCode()
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import io.emeraldpay.etherjar.rpc.json.BlockJson
|
|||||||
import org.bouncycastle.util.encoders.Hex
|
import org.bouncycastle.util.encoders.Hex
|
||||||
|
|
||||||
class BlockId(
|
class BlockId(
|
||||||
value: ByteArray
|
value: ByteArray
|
||||||
) : HashId(value) {
|
) : HashId(value) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -51,6 +51,4 @@ class BlockId(
|
|||||||
return BlockId(bytes)
|
return BlockId(bytes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -18,14 +18,14 @@ package io.emeraldpay.dshackle.data
|
|||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
|
|
||||||
class DefaultContainer<T>(
|
class DefaultContainer<T>(
|
||||||
val txId: TxId?,
|
val txId: TxId?,
|
||||||
val blockId: BlockId?,
|
val blockId: BlockId?,
|
||||||
val height: Long?,
|
val height: Long?,
|
||||||
json: ByteArray,
|
json: ByteArray,
|
||||||
parsed: T
|
parsed: T
|
||||||
) : SourceContainer(json, parsed) {
|
) : SourceContainer(json, parsed) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private val log = LoggerFactory.getLogger(DefaultContainer::class.java)
|
private val log = LoggerFactory.getLogger(DefaultContainer::class.java)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,7 +17,7 @@
|
|||||||
package io.emeraldpay.dshackle.data
|
package io.emeraldpay.dshackle.data
|
||||||
|
|
||||||
open class HashId(
|
open class HashId(
|
||||||
val value: ByteArray
|
val value: ByteArray
|
||||||
) {
|
) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -56,6 +56,4 @@ open class HashId(
|
|||||||
override fun hashCode(): Int {
|
override fun hashCode(): Int {
|
||||||
return value.contentHashCode()
|
return value.contentHashCode()
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ class RawJsonBuilder {
|
|||||||
buf.write(START)
|
buf.write(START)
|
||||||
buf.write(COMMA)
|
buf.write(COMMA)
|
||||||
buf.write(ID_START)
|
buf.write(ID_START)
|
||||||
buf.write(id.toString().toByteArray());
|
buf.write(id.toString().toByteArray())
|
||||||
buf.write(COMMA)
|
buf.write(COMMA)
|
||||||
buf.write(RESULT_START)
|
buf.write(RESULT_START)
|
||||||
buf.write(data)
|
buf.write(data)
|
||||||
@@ -44,6 +44,4 @@ class RawJsonBuilder {
|
|||||||
|
|
||||||
return buf.toByteArray()
|
return buf.toByteArray()
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -16,11 +16,9 @@
|
|||||||
*/
|
*/
|
||||||
package io.emeraldpay.dshackle.data
|
package io.emeraldpay.dshackle.data
|
||||||
|
|
||||||
import java.lang.ClassCastException
|
|
||||||
|
|
||||||
abstract class SourceContainer(
|
abstract class SourceContainer(
|
||||||
val json: ByteArray?,
|
val json: ByteArray?,
|
||||||
private val parsed: Any?
|
private val parsed: Any?
|
||||||
) {
|
) {
|
||||||
|
|
||||||
@Suppress("UNCHECKED_CAST")
|
@Suppress("UNCHECKED_CAST")
|
||||||
@@ -34,7 +32,6 @@ abstract class SourceContainer(
|
|||||||
throw ClassCastException("Cannot cast ${parsed.javaClass} to $clazz")
|
throw ClassCastException("Cannot cast ${parsed.javaClass} to $clazz")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
override fun equals(other: Any?): Boolean {
|
override fun equals(other: Any?): Boolean {
|
||||||
if (this === other) return true
|
if (this === other) return true
|
||||||
if (other !is SourceContainer) return false
|
if (other !is SourceContainer) return false
|
||||||
@@ -50,4 +47,4 @@ abstract class SourceContainer(
|
|||||||
override fun hashCode(): Int {
|
override fun hashCode(): Int {
|
||||||
return json?.contentHashCode() ?: 0
|
return json?.contentHashCode() ?: 0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,11 +20,11 @@ import io.emeraldpay.dshackle.Global
|
|||||||
import io.emeraldpay.etherjar.rpc.json.TransactionJson
|
import io.emeraldpay.etherjar.rpc.json.TransactionJson
|
||||||
|
|
||||||
class TxContainer(
|
class TxContainer(
|
||||||
val height: Long?,
|
val height: Long?,
|
||||||
val hash: TxId,
|
val hash: TxId,
|
||||||
val blockId: BlockId?,
|
val blockId: BlockId?,
|
||||||
json: ByteArray?,
|
json: ByteArray?,
|
||||||
parsed: Any? = null
|
parsed: Any? = null
|
||||||
) : SourceContainer(json, parsed) {
|
) : SourceContainer(json, parsed) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -41,11 +41,11 @@ class TxContainer(
|
|||||||
|
|
||||||
fun from(tx: TransactionJson, raw: ByteArray): TxContainer {
|
fun from(tx: TransactionJson, raw: ByteArray): TxContainer {
|
||||||
return TxContainer(
|
return TxContainer(
|
||||||
tx.blockNumber,
|
tx.blockNumber,
|
||||||
TxId.from(tx.hash),
|
TxId.from(tx.hash),
|
||||||
tx.blockHash?.let { BlockId.from(it) },
|
tx.blockHash?.let { BlockId.from(it) },
|
||||||
raw,
|
raw,
|
||||||
tx
|
tx
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -70,6 +70,4 @@ class TxContainer(
|
|||||||
result = 31 * result + hash.hashCode()
|
result = 31 * result + hash.hashCode()
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -19,10 +19,9 @@ package io.emeraldpay.dshackle.data
|
|||||||
import io.emeraldpay.etherjar.domain.TransactionId
|
import io.emeraldpay.etherjar.domain.TransactionId
|
||||||
import io.emeraldpay.etherjar.rpc.json.TransactionJson
|
import io.emeraldpay.etherjar.rpc.json.TransactionJson
|
||||||
import org.bouncycastle.util.encoders.Hex
|
import org.bouncycastle.util.encoders.Hex
|
||||||
import java.math.BigInteger
|
|
||||||
|
|
||||||
class TxId(
|
class TxId(
|
||||||
value: ByteArray
|
value: ByteArray
|
||||||
) : HashId(value) {
|
) : HashId(value) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -47,4 +46,4 @@ class TxId(
|
|||||||
return TxId(bytes)
|
return TxId(bytes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,20 +26,18 @@ import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics
|
|||||||
import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics
|
import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics
|
||||||
import io.micrometer.core.instrument.binder.system.ProcessorMetrics
|
import io.micrometer.core.instrument.binder.system.ProcessorMetrics
|
||||||
import io.micrometer.core.instrument.config.MeterFilter
|
import io.micrometer.core.instrument.config.MeterFilter
|
||||||
import org.slf4j.LoggerFactory
|
|
||||||
import org.springframework.stereotype.Service
|
|
||||||
import io.micrometer.prometheus.PrometheusConfig
|
import io.micrometer.prometheus.PrometheusConfig
|
||||||
|
|
||||||
import io.micrometer.prometheus.PrometheusMeterRegistry
|
import io.micrometer.prometheus.PrometheusMeterRegistry
|
||||||
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
|
import org.springframework.stereotype.Service
|
||||||
import java.io.IOException
|
import java.io.IOException
|
||||||
import java.net.InetSocketAddress
|
import java.net.InetSocketAddress
|
||||||
import javax.annotation.PostConstruct
|
import javax.annotation.PostConstruct
|
||||||
|
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
class MonitoringSetup(
|
class MonitoringSetup(
|
||||||
@Autowired private val monitoringConfig: MonitoringConfig
|
@Autowired private val monitoringConfig: MonitoringConfig
|
||||||
) {
|
) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -50,7 +48,7 @@ class MonitoringSetup(
|
|||||||
fun setup() {
|
fun setup() {
|
||||||
val prometheusRegistry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT)
|
val prometheusRegistry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT)
|
||||||
Metrics.globalRegistry.add(prometheusRegistry)
|
Metrics.globalRegistry.add(prometheusRegistry)
|
||||||
Metrics.globalRegistry.config().meterFilter(object: MeterFilter {
|
Metrics.globalRegistry.config().meterFilter(object : MeterFilter {
|
||||||
override fun map(id: Meter.Id): Meter.Id {
|
override fun map(id: Meter.Id): Meter.Id {
|
||||||
if (id.name.startsWith("jvm") || id.name.startsWith("process") || id.name.startsWith("system")) {
|
if (id.name.startsWith("jvm") || id.name.startsWith("process") || id.name.startsWith("system")) {
|
||||||
return id
|
return id
|
||||||
@@ -76,18 +74,24 @@ class MonitoringSetup(
|
|||||||
// prometheus is a single thread periodic call, no reason to setup anything complex
|
// prometheus is a single thread periodic call, no reason to setup anything complex
|
||||||
try {
|
try {
|
||||||
log.info("Run Prometheus metrics on ${monitoringConfig.prometheus.host}:${monitoringConfig.prometheus.port}${monitoringConfig.prometheus.path}")
|
log.info("Run Prometheus metrics on ${monitoringConfig.prometheus.host}:${monitoringConfig.prometheus.port}${monitoringConfig.prometheus.path}")
|
||||||
val server = HttpServer.create(InetSocketAddress(monitoringConfig.prometheus.host, monitoringConfig.prometheus.port), 0);
|
val server = HttpServer.create(
|
||||||
|
InetSocketAddress(
|
||||||
|
monitoringConfig.prometheus.host,
|
||||||
|
monitoringConfig.prometheus.port
|
||||||
|
),
|
||||||
|
0
|
||||||
|
)
|
||||||
server.createContext(monitoringConfig.prometheus.path) { httpExchange ->
|
server.createContext(monitoringConfig.prometheus.path) { httpExchange ->
|
||||||
val response = prometheusRegistry.scrape()
|
val response = prometheusRegistry.scrape()
|
||||||
httpExchange.sendResponseHeaders(200, response.toByteArray().size.toLong());
|
httpExchange.sendResponseHeaders(200, response.toByteArray().size.toLong())
|
||||||
httpExchange.responseBody.use { os ->
|
httpExchange.responseBody.use { os ->
|
||||||
os.write(response.toByteArray())
|
os.write(response.toByteArray())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Thread(server::start).start();
|
Thread(server::start).start()
|
||||||
} catch (e: IOException) {
|
} catch (e: IOException) {
|
||||||
log.error("Failed to start Prometheus Server", e)
|
log.error("Failed to start Prometheus Server", e)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,14 +15,20 @@
|
|||||||
*/
|
*/
|
||||||
package io.emeraldpay.dshackle.monitoring.accesslog
|
package io.emeraldpay.dshackle.monitoring.accesslog
|
||||||
|
|
||||||
import io.grpc.*
|
import io.grpc.ForwardingServerCall
|
||||||
|
import io.grpc.ForwardingServerCallListener
|
||||||
|
import io.grpc.Metadata
|
||||||
|
import io.grpc.MethodDescriptor
|
||||||
|
import io.grpc.ServerCall
|
||||||
|
import io.grpc.ServerCallHandler
|
||||||
|
import io.grpc.ServerInterceptor
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
import org.springframework.stereotype.Service
|
import org.springframework.stereotype.Service
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
class AccessHandlerGrpc(
|
class AccessHandlerGrpc(
|
||||||
@Autowired private val accessLogWriter: AccessLogWriter
|
@Autowired private val accessLogWriter: AccessLogWriter
|
||||||
) : ServerInterceptor {
|
) : ServerInterceptor {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -30,9 +36,10 @@ class AccessHandlerGrpc(
|
|||||||
}
|
}
|
||||||
|
|
||||||
override fun <ReqT : Any, RespT : Any> interceptCall(
|
override fun <ReqT : Any, RespT : Any> interceptCall(
|
||||||
call: ServerCall<ReqT, RespT>,
|
call: ServerCall<ReqT, RespT>,
|
||||||
headers: Metadata,
|
headers: Metadata,
|
||||||
next: ServerCallHandler<ReqT, RespT>): ServerCall.Listener<ReqT> {
|
next: ServerCallHandler<ReqT, RespT>
|
||||||
|
): ServerCall.Listener<ReqT> {
|
||||||
|
|
||||||
return when (val method = call.methodDescriptor.bareMethodName) {
|
return when (val method = call.methodDescriptor.bareMethodName) {
|
||||||
"SubscribeHead" -> processSubscribeHead(call, headers, next)
|
"SubscribeHead" -> processSubscribeHead(call, headers, next)
|
||||||
@@ -51,103 +58,109 @@ class AccessHandlerGrpc(
|
|||||||
}
|
}
|
||||||
|
|
||||||
private fun <ReqT : Any, RespT : Any, E> process(
|
private fun <ReqT : Any, RespT : Any, E> process(
|
||||||
call: ServerCall<ReqT, RespT>,
|
call: ServerCall<ReqT, RespT>,
|
||||||
headers: Metadata,
|
headers: Metadata,
|
||||||
next: ServerCallHandler<ReqT, RespT>,
|
next: ServerCallHandler<ReqT, RespT>,
|
||||||
builder: EventsBuilder.RequestReply<E, ReqT, RespT>
|
builder: EventsBuilder.RequestReply<E, ReqT, RespT>
|
||||||
): ServerCall.Listener<ReqT> {
|
): ServerCall.Listener<ReqT> {
|
||||||
builder.start(headers, call.attributes)
|
builder.start(headers, call.attributes)
|
||||||
val callWrapper: ServerCall<ReqT, RespT> = StdCallResponse(
|
val callWrapper: ServerCall<ReqT, RespT> = StdCallResponse(
|
||||||
call, builder, accessLogWriter
|
call, builder, accessLogWriter
|
||||||
)
|
)
|
||||||
return StdCallListener(
|
return StdCallListener(
|
||||||
next.startCall(callWrapper, headers),
|
next.startCall(callWrapper, headers),
|
||||||
builder
|
builder
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@Suppress("UNCHECKED_CAST")
|
@Suppress("UNCHECKED_CAST")
|
||||||
private fun <ReqT : Any, RespT : Any> processSubscribeHead(
|
private fun <ReqT : Any, RespT : Any> processSubscribeHead(
|
||||||
call: ServerCall<ReqT, RespT>,
|
call: ServerCall<ReqT, RespT>,
|
||||||
headers: Metadata,
|
headers: Metadata,
|
||||||
next: ServerCallHandler<ReqT, RespT>
|
next: ServerCallHandler<ReqT, RespT>
|
||||||
): ServerCall.Listener<ReqT> {
|
): ServerCall.Listener<ReqT> {
|
||||||
return process(call, headers, next,
|
return process(
|
||||||
EventsBuilder.SubscribeHead() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
call, headers, next,
|
||||||
|
EventsBuilder.SubscribeHead() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@Suppress("UNCHECKED_CAST")
|
@Suppress("UNCHECKED_CAST")
|
||||||
private fun <ReqT : Any, RespT : Any> processSubscribeBalance(
|
private fun <ReqT : Any, RespT : Any> processSubscribeBalance(
|
||||||
call: ServerCall<ReqT, RespT>,
|
call: ServerCall<ReqT, RespT>,
|
||||||
headers: Metadata,
|
headers: Metadata,
|
||||||
next: ServerCallHandler<ReqT, RespT>,
|
next: ServerCallHandler<ReqT, RespT>,
|
||||||
subscribe: Boolean
|
subscribe: Boolean
|
||||||
): ServerCall.Listener<ReqT> {
|
): ServerCall.Listener<ReqT> {
|
||||||
return process(call, headers, next,
|
return process(
|
||||||
EventsBuilder.SubscribeBalance(subscribe) as EventsBuilder.RequestReply<*, ReqT, RespT>
|
call, headers, next,
|
||||||
|
EventsBuilder.SubscribeBalance(subscribe) as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@Suppress("UNCHECKED_CAST")
|
@Suppress("UNCHECKED_CAST")
|
||||||
private fun <ReqT : Any, RespT : Any> processSubscribeTxStatus(
|
private fun <ReqT : Any, RespT : Any> processSubscribeTxStatus(
|
||||||
call: ServerCall<ReqT, RespT>,
|
call: ServerCall<ReqT, RespT>,
|
||||||
headers: Metadata,
|
headers: Metadata,
|
||||||
next: ServerCallHandler<ReqT, RespT>
|
next: ServerCallHandler<ReqT, RespT>
|
||||||
): ServerCall.Listener<ReqT> {
|
): ServerCall.Listener<ReqT> {
|
||||||
return process(call, headers, next,
|
return process(
|
||||||
EventsBuilder.TxStatus() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
call, headers, next,
|
||||||
|
EventsBuilder.TxStatus() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@Suppress("UNCHECKED_CAST")
|
@Suppress("UNCHECKED_CAST")
|
||||||
private fun <ReqT : Any, RespT : Any> processNativeCall(
|
private fun <ReqT : Any, RespT : Any> processNativeCall(
|
||||||
call: ServerCall<ReqT, RespT>,
|
call: ServerCall<ReqT, RespT>,
|
||||||
headers: Metadata,
|
headers: Metadata,
|
||||||
next: ServerCallHandler<ReqT, RespT>
|
next: ServerCallHandler<ReqT, RespT>
|
||||||
): ServerCall.Listener<ReqT> {
|
): ServerCall.Listener<ReqT> {
|
||||||
return process(call, headers, next,
|
return process(
|
||||||
EventsBuilder.NativeCall() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
call, headers, next,
|
||||||
|
EventsBuilder.NativeCall() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@Suppress("UNCHECKED_CAST")
|
@Suppress("UNCHECKED_CAST")
|
||||||
private fun <ReqT : Any, RespT : Any> processNativeSubscribe(
|
private fun <ReqT : Any, RespT : Any> processNativeSubscribe(
|
||||||
call: ServerCall<ReqT, RespT>,
|
call: ServerCall<ReqT, RespT>,
|
||||||
headers: Metadata,
|
headers: Metadata,
|
||||||
next: ServerCallHandler<ReqT, RespT>
|
next: ServerCallHandler<ReqT, RespT>
|
||||||
): ServerCall.Listener<ReqT> {
|
): ServerCall.Listener<ReqT> {
|
||||||
return process(call, headers, next,
|
return process(
|
||||||
EventsBuilder.NativeSubscribe() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
call, headers, next,
|
||||||
|
EventsBuilder.NativeSubscribe() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Suppress("UNCHECKED_CAST")
|
@Suppress("UNCHECKED_CAST")
|
||||||
private fun <ReqT : Any, RespT : Any> processDescribe(
|
private fun <ReqT : Any, RespT : Any> processDescribe(
|
||||||
call: ServerCall<ReqT, RespT>,
|
call: ServerCall<ReqT, RespT>,
|
||||||
headers: Metadata,
|
headers: Metadata,
|
||||||
next: ServerCallHandler<ReqT, RespT>
|
next: ServerCallHandler<ReqT, RespT>
|
||||||
): ServerCall.Listener<ReqT> {
|
): ServerCall.Listener<ReqT> {
|
||||||
return process(call, headers, next,
|
return process(
|
||||||
EventsBuilder.Describe() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
call, headers, next,
|
||||||
|
EventsBuilder.Describe() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@Suppress("UNCHECKED_CAST")
|
@Suppress("UNCHECKED_CAST")
|
||||||
private fun <ReqT : Any, RespT : Any> processStatus(
|
private fun <ReqT : Any, RespT : Any> processStatus(
|
||||||
call: ServerCall<ReqT, RespT>,
|
call: ServerCall<ReqT, RespT>,
|
||||||
headers: Metadata,
|
headers: Metadata,
|
||||||
next: ServerCallHandler<ReqT, RespT>
|
next: ServerCallHandler<ReqT, RespT>
|
||||||
): ServerCall.Listener<ReqT> {
|
): ServerCall.Listener<ReqT> {
|
||||||
return process(call, headers, next,
|
return process(
|
||||||
EventsBuilder.Status() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
call, headers, next,
|
||||||
|
EventsBuilder.Status() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
open class StdCallListener<Req, EB : EventsBuilder.RequestReply<*, Req, *>>(
|
open class StdCallListener<Req, EB : EventsBuilder.RequestReply<*, Req, *>>(
|
||||||
val next: ServerCall.Listener<Req>,
|
val next: ServerCall.Listener<Req>,
|
||||||
val builder: EB
|
val builder: EB
|
||||||
) : ForwardingServerCallListener<Req>() {
|
) : ForwardingServerCallListener<Req>() {
|
||||||
|
|
||||||
override fun onMessage(message: Req) {
|
override fun onMessage(message: Req) {
|
||||||
@@ -161,9 +174,9 @@ class AccessHandlerGrpc(
|
|||||||
}
|
}
|
||||||
|
|
||||||
open class StdCallResponse<ReqT : Any, RespT : Any, EB : EventsBuilder.RequestReply<*, ReqT, RespT>>(
|
open class StdCallResponse<ReqT : Any, RespT : Any, EB : EventsBuilder.RequestReply<*, ReqT, RespT>>(
|
||||||
val next: ServerCall<ReqT, RespT>,
|
val next: ServerCall<ReqT, RespT>,
|
||||||
val builder: EB,
|
val builder: EB,
|
||||||
val accessLogWriter: AccessLogWriter
|
val accessLogWriter: AccessLogWriter
|
||||||
) : ForwardingServerCall<ReqT, RespT>() {
|
) : ForwardingServerCall<ReqT, RespT>() {
|
||||||
|
|
||||||
override fun getMethodDescriptor(): MethodDescriptor<ReqT, RespT> {
|
override fun getMethodDescriptor(): MethodDescriptor<ReqT, RespT> {
|
||||||
@@ -177,9 +190,8 @@ class AccessHandlerGrpc(
|
|||||||
override fun sendMessage(message: RespT) {
|
override fun sendMessage(message: RespT) {
|
||||||
super.sendMessage(message)
|
super.sendMessage(message)
|
||||||
accessLogWriter.submit(
|
accessLogWriter.submit(
|
||||||
builder.onReply(message)!!
|
builder.onReply(message)!!
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -19,8 +19,8 @@ import kotlin.concurrent.withLock
|
|||||||
*/
|
*/
|
||||||
@Service
|
@Service
|
||||||
class AccessHandlerHttp(
|
class AccessHandlerHttp(
|
||||||
@Autowired private val mainConfig: MainConfig,
|
@Autowired private val mainConfig: MainConfig,
|
||||||
@Autowired accessLogWriter: AccessLogWriter
|
@Autowired accessLogWriter: AccessLogWriter
|
||||||
) {
|
) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -40,7 +40,7 @@ class AccessHandlerHttp(
|
|||||||
fun create(req: HttpServerRequest, blockchain: Chain): RequestHandler
|
fun create(req: HttpServerRequest, blockchain: Chain): RequestHandler
|
||||||
}
|
}
|
||||||
|
|
||||||
class NoOpFactory() : HandlerFactory {
|
class NoOpFactory : HandlerFactory {
|
||||||
override fun create(req: HttpServerRequest, blockchain: Chain): RequestHandler {
|
override fun create(req: HttpServerRequest, blockchain: Chain): RequestHandler {
|
||||||
return NoOpHandler()
|
return NoOpHandler()
|
||||||
}
|
}
|
||||||
@@ -70,9 +70,9 @@ class AccessHandlerHttp(
|
|||||||
}
|
}
|
||||||
|
|
||||||
class StandardHandler(
|
class StandardHandler(
|
||||||
private val accessLogWriter: AccessLogWriter,
|
private val accessLogWriter: AccessLogWriter,
|
||||||
private val httpRequest: HttpServerRequest,
|
private val httpRequest: HttpServerRequest,
|
||||||
private val blockchain: Chain
|
private val blockchain: Chain
|
||||||
) : RequestHandler {
|
) : RequestHandler {
|
||||||
|
|
||||||
private var request: BlockchainOuterClass.NativeCallRequest? = null
|
private var request: BlockchainOuterClass.NativeCallRequest? = null
|
||||||
@@ -89,13 +89,13 @@ class AccessHandlerHttp(
|
|||||||
builder.start(httpRequest)
|
builder.start(httpRequest)
|
||||||
builder.onRequest(request!!)
|
builder.onRequest(request!!)
|
||||||
responses
|
responses
|
||||||
.map {
|
.map {
|
||||||
builder.onReply(it, Events.Channel.JSONRPC).also { item ->
|
builder.onReply(it, Events.Channel.JSONRPC).also { item ->
|
||||||
//since for JSON RPC you get a single response then the timestamp of all items included in it must have the same timestamp
|
// since for JSON RPC you get a single response then the timestamp of all items included in it must have the same timestamp
|
||||||
item.ts = responseTime
|
item.ts = responseTime
|
||||||
}
|
|
||||||
}
|
}
|
||||||
.let(accessLogWriter::submit)
|
}
|
||||||
|
.let(accessLogWriter::submit)
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun onRequest(request: BlockchainOuterClass.NativeCallRequest) {
|
override fun onRequest(request: BlockchainOuterClass.NativeCallRequest) {
|
||||||
@@ -108,4 +108,4 @@ class AccessHandlerHttp(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,7 +20,9 @@ import io.emeraldpay.dshackle.config.MainConfig
|
|||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
import org.springframework.stereotype.Repository
|
import org.springframework.stereotype.Repository
|
||||||
import java.io.*
|
import java.io.BufferedOutputStream
|
||||||
|
import java.io.File
|
||||||
|
import java.io.FileOutputStream
|
||||||
import java.time.Duration
|
import java.time.Duration
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
import java.util.concurrent.ConcurrentLinkedQueue
|
import java.util.concurrent.ConcurrentLinkedQueue
|
||||||
@@ -30,7 +32,7 @@ import javax.annotation.PostConstruct
|
|||||||
|
|
||||||
@Repository
|
@Repository
|
||||||
class AccessLogWriter(
|
class AccessLogWriter(
|
||||||
@Autowired mainConfig: MainConfig
|
@Autowired mainConfig: MainConfig
|
||||||
) {
|
) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -120,5 +122,4 @@ class AccessLogWriter(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonInclude
|
|||||||
import io.emeraldpay.grpc.Chain
|
import io.emeraldpay.grpc.Chain
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
import java.util.*
|
import java.util.UUID
|
||||||
|
|
||||||
class Events {
|
class Events {
|
||||||
|
|
||||||
@@ -32,140 +32,152 @@ class Events {
|
|||||||
}
|
}
|
||||||
|
|
||||||
abstract class Base(
|
abstract class Base(
|
||||||
val id: UUID,
|
val id: UUID,
|
||||||
val method: String,
|
val method: String,
|
||||||
val channel: Channel
|
val channel: Channel
|
||||||
) {
|
) {
|
||||||
val version = "accesslog/v1beta"
|
val version = "accesslog/v1beta"
|
||||||
var ts = Instant.now()
|
var ts = Instant.now()
|
||||||
}
|
}
|
||||||
|
|
||||||
abstract class ChainBase(
|
abstract class ChainBase(
|
||||||
val blockchain: Chain, method: String, id: UUID, channel: Channel
|
val blockchain: Chain,
|
||||||
|
method: String,
|
||||||
|
id: UUID,
|
||||||
|
channel: Channel
|
||||||
) : Base(id, method, channel)
|
) : Base(id, method, channel)
|
||||||
|
|
||||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
class SubscribeHead(
|
class SubscribeHead(
|
||||||
blockchain: Chain, id: UUID,
|
blockchain: Chain,
|
||||||
// initial request details
|
id: UUID,
|
||||||
val request: StreamRequestDetails,
|
// initial request details
|
||||||
// index of the current response
|
val request: StreamRequestDetails,
|
||||||
val index: Int
|
// index of the current response
|
||||||
|
val index: Int
|
||||||
) : ChainBase(blockchain, "SubscribeHead", id, Channel.GRPC)
|
) : ChainBase(blockchain, "SubscribeHead", id, Channel.GRPC)
|
||||||
|
|
||||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
class SubscribeBalance(
|
class SubscribeBalance(
|
||||||
blockchain: Chain, id: UUID, subscribe: Boolean,
|
blockchain: Chain,
|
||||||
// initial request details
|
id: UUID,
|
||||||
val request: StreamRequestDetails,
|
subscribe: Boolean,
|
||||||
val balanceRequest: BalanceRequest,
|
// initial request details
|
||||||
val addressBalance: AddressBalance,
|
val request: StreamRequestDetails,
|
||||||
// index of the current response
|
val balanceRequest: BalanceRequest,
|
||||||
val index: Int
|
val addressBalance: AddressBalance,
|
||||||
|
// index of the current response
|
||||||
|
val index: Int
|
||||||
) : ChainBase(blockchain, if (subscribe) "SubscribeBalance" else "GetBalance", id, Channel.GRPC)
|
) : ChainBase(blockchain, if (subscribe) "SubscribeBalance" else "GetBalance", id, Channel.GRPC)
|
||||||
|
|
||||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
class TxStatus(
|
class TxStatus(
|
||||||
blockchain: Chain, id: UUID,
|
blockchain: Chain,
|
||||||
val request: StreamRequestDetails,
|
id: UUID,
|
||||||
val txStatusRequest: TxStatusRequest,
|
val request: StreamRequestDetails,
|
||||||
val txStatus: TxStatusResponse,
|
val txStatusRequest: TxStatusRequest,
|
||||||
// index of the current response
|
val txStatus: TxStatusResponse,
|
||||||
val index: Int
|
// index of the current response
|
||||||
|
val index: Int
|
||||||
) : ChainBase(blockchain, "SubscribeTxStatus", id, Channel.GRPC)
|
) : ChainBase(blockchain, "SubscribeTxStatus", id, Channel.GRPC)
|
||||||
|
|
||||||
data class TxStatusRequest(
|
data class TxStatusRequest(
|
||||||
val txId: String
|
val txId: String
|
||||||
)
|
)
|
||||||
|
|
||||||
data class TxStatusResponse(
|
data class TxStatusResponse(
|
||||||
val confirmations: Int
|
val confirmations: Int
|
||||||
)
|
)
|
||||||
|
|
||||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
class NativeCall(
|
class NativeCall(
|
||||||
blockchain: Chain, id: UUID, channel: Channel,
|
blockchain: Chain,
|
||||||
|
id: UUID,
|
||||||
|
channel: Channel,
|
||||||
|
|
||||||
// info about the initial request, that may include several native calls
|
// info about the initial request, that may include several native calls
|
||||||
val request: StreamRequestDetails,
|
val request: StreamRequestDetails,
|
||||||
// total native calls passes within the initial request
|
// total native calls passes within the initial request
|
||||||
val total: Int,
|
val total: Int,
|
||||||
// index of the call specific for the current response
|
// index of the call specific for the current response
|
||||||
val index: Int,
|
val index: Int,
|
||||||
val selector: String? = null,
|
val selector: String? = null,
|
||||||
val quorum: Long? = null,
|
val quorum: Long? = null,
|
||||||
val minAvailability: String? = null,
|
val minAvailability: String? = null,
|
||||||
|
|
||||||
val succeed: Boolean,
|
val succeed: Boolean,
|
||||||
val rpcError: Int? = null,
|
val rpcError: Int? = null,
|
||||||
val payloadSizeBytes: Long,
|
val payloadSizeBytes: Long,
|
||||||
val nativeCall: NativeCallItemDetails
|
val nativeCall: NativeCallItemDetails
|
||||||
) : ChainBase(blockchain, "NativeCall", id, channel)
|
) : ChainBase(blockchain, "NativeCall", id, channel)
|
||||||
|
|
||||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
class NativeSubscribe(
|
class NativeSubscribe(
|
||||||
blockchain: Chain, id: UUID, channel: Channel,
|
blockchain: Chain,
|
||||||
|
id: UUID,
|
||||||
|
channel: Channel,
|
||||||
|
|
||||||
// info about the initial request, that may include several native calls
|
// info about the initial request, that may include several native calls
|
||||||
val request: StreamRequestDetails,
|
val request: StreamRequestDetails,
|
||||||
val payloadSizeBytes: Long,
|
val payloadSizeBytes: Long,
|
||||||
val nativeSubscribe: NativeSubscribeItemDetails
|
val nativeSubscribe: NativeSubscribeItemDetails
|
||||||
) : ChainBase(blockchain, "NativeSubscribe", id, channel)
|
) : ChainBase(blockchain, "NativeSubscribe", id, channel)
|
||||||
|
|
||||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
class Describe(
|
class Describe(
|
||||||
id: UUID,
|
id: UUID,
|
||||||
val request: StreamRequestDetails
|
val request: StreamRequestDetails
|
||||||
) : Base(id, "Describe", Channel.GRPC)
|
) : Base(id, "Describe", Channel.GRPC)
|
||||||
|
|
||||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
class Status(
|
class Status(
|
||||||
blockchain: Chain, id: UUID,
|
blockchain: Chain,
|
||||||
val request: StreamRequestDetails
|
id: UUID,
|
||||||
|
val request: StreamRequestDetails
|
||||||
) : ChainBase(blockchain, "Status", id, Channel.GRPC)
|
) : ChainBase(blockchain, "Status", id, Channel.GRPC)
|
||||||
|
|
||||||
data class StreamRequestDetails(
|
data class StreamRequestDetails(
|
||||||
val id: UUID,
|
val id: UUID,
|
||||||
val start: Instant,
|
val start: Instant,
|
||||||
val remote: Remote
|
val remote: Remote
|
||||||
)
|
)
|
||||||
|
|
||||||
data class Remote(
|
data class Remote(
|
||||||
val ips: List<String>,
|
val ips: List<String>,
|
||||||
val ip: String,
|
val ip: String,
|
||||||
val userAgent: String
|
val userAgent: String
|
||||||
)
|
)
|
||||||
|
|
||||||
data class NativeCallItemDetails(
|
data class NativeCallItemDetails(
|
||||||
val method: String,
|
val method: String,
|
||||||
val id: Int,
|
val id: Int,
|
||||||
val payloadSizeBytes: Long
|
val payloadSizeBytes: Long
|
||||||
)
|
)
|
||||||
|
|
||||||
data class NativeCallReplyDetails(
|
data class NativeCallReplyDetails(
|
||||||
val id: Int,
|
val id: Int,
|
||||||
val succeed: Boolean,
|
val succeed: Boolean,
|
||||||
val replySizeBytes: Long,
|
val replySizeBytes: Long,
|
||||||
val ts: Instant = Instant.now()
|
val ts: Instant = Instant.now()
|
||||||
)
|
)
|
||||||
|
|
||||||
data class NativeSubscribeItemDetails(
|
data class NativeSubscribeItemDetails(
|
||||||
val method: String,
|
val method: String,
|
||||||
val payloadSizeBytes: Long
|
val payloadSizeBytes: Long
|
||||||
)
|
)
|
||||||
|
|
||||||
data class NativeSubscribeReplyDetails(
|
data class NativeSubscribeReplyDetails(
|
||||||
val replySizeBytes: Long,
|
val replySizeBytes: Long,
|
||||||
val ts: Instant = Instant.now()
|
val ts: Instant = Instant.now()
|
||||||
)
|
)
|
||||||
|
|
||||||
data class BalanceRequest(
|
data class BalanceRequest(
|
||||||
val asset: String,
|
val asset: String,
|
||||||
val addressType: String
|
val addressType: String
|
||||||
)
|
)
|
||||||
|
|
||||||
data class AddressBalance(
|
data class AddressBalance(
|
||||||
val asset: String,
|
val asset: String,
|
||||||
val address: String
|
val address: String
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,7 +27,8 @@ import reactor.netty.http.server.HttpServerRequest
|
|||||||
import java.net.InetAddress
|
import java.net.InetAddress
|
||||||
import java.net.InetSocketAddress
|
import java.net.InetSocketAddress
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
import java.util.*
|
import java.util.Locale
|
||||||
|
import java.util.UUID
|
||||||
|
|
||||||
class EventsBuilder {
|
class EventsBuilder {
|
||||||
|
|
||||||
@@ -48,23 +49,23 @@ class EventsBuilder {
|
|||||||
fun onReply(msg: Resp): E
|
fun onReply(msg: Resp): E
|
||||||
}
|
}
|
||||||
|
|
||||||
abstract class Base<T>() : StartingHttp2Request, StartingHttp1Request {
|
abstract class Base<T> : StartingHttp2Request, StartingHttp1Request {
|
||||||
companion object {
|
companion object {
|
||||||
private val remoteIpHeaders = listOf(
|
private val remoteIpHeaders = listOf(
|
||||||
"x-real-ip",
|
"x-real-ip",
|
||||||
"x-forwarded-for"
|
"x-forwarded-for"
|
||||||
)
|
)
|
||||||
private val remoteIpKeys = listOf(
|
private val remoteIpKeys = listOf(
|
||||||
Metadata.Key.of("x-real-ip", Metadata.ASCII_STRING_MARSHALLER),
|
Metadata.Key.of("x-real-ip", Metadata.ASCII_STRING_MARSHALLER),
|
||||||
Metadata.Key.of("x-forwarded-for", Metadata.ASCII_STRING_MARSHALLER)
|
Metadata.Key.of("x-forwarded-for", Metadata.ASCII_STRING_MARSHALLER)
|
||||||
)
|
)
|
||||||
private val invalidCharacters = Regex("[\n\t]+")
|
private val invalidCharacters = Regex("[\n\t]+")
|
||||||
}
|
}
|
||||||
|
|
||||||
var requestDetails = Events.StreamRequestDetails(
|
var requestDetails = Events.StreamRequestDetails(
|
||||||
UUID.randomUUID(),
|
UUID.randomUUID(),
|
||||||
Instant.now(),
|
Instant.now(),
|
||||||
Events.Remote(emptyList(), "", "")
|
Events.Remote(emptyList(), "", "")
|
||||||
)
|
)
|
||||||
|
|
||||||
var chainId: Int = Chain.UNSPECIFIED.id
|
var chainId: Int = Chain.UNSPECIFIED.id
|
||||||
@@ -84,35 +85,37 @@ class EventsBuilder {
|
|||||||
|
|
||||||
private fun findBestIp(ips: List<InetAddress>): InetAddress? {
|
private fun findBestIp(ips: List<InetAddress>): InetAddress? {
|
||||||
// check if a real remote address is provided, otherwise use any local address
|
// check if a real remote address is provided, otherwise use any local address
|
||||||
return ips.sortedWith(kotlin.Comparator { a, b ->
|
return ips.sortedWith(
|
||||||
val aLocal = a.isLoopbackAddress || a.isSiteLocalAddress
|
kotlin.Comparator { a, b ->
|
||||||
val bLocal = b.isLoopbackAddress || b.isSiteLocalAddress
|
val aLocal = a.isLoopbackAddress || a.isSiteLocalAddress
|
||||||
when {
|
val bLocal = b.isLoopbackAddress || b.isSiteLocalAddress
|
||||||
aLocal && bLocal -> 0
|
when {
|
||||||
aLocal -> 1
|
aLocal && bLocal -> 0
|
||||||
else -> -1
|
aLocal -> 1
|
||||||
|
else -> -1
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}).firstOrNull()
|
).firstOrNull()
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun clean(s: String): String {
|
private fun clean(s: String): String {
|
||||||
return StringUtils.truncate(s, 128)
|
return StringUtils.truncate(s, 128)
|
||||||
.replace(invalidCharacters, " ")
|
.replace(invalidCharacters, " ")
|
||||||
.trim()
|
.trim()
|
||||||
}
|
}
|
||||||
|
|
||||||
protected abstract fun getT(): T
|
protected abstract fun getT(): T
|
||||||
|
|
||||||
override fun start(metadata: Metadata, attributes: Attributes) {
|
override fun start(metadata: Metadata, attributes: Attributes) {
|
||||||
val userAgent = metadata.get(Metadata.Key.of("user-agent", Metadata.ASCII_STRING_MARSHALLER))
|
val userAgent = metadata.get(Metadata.Key.of("user-agent", Metadata.ASCII_STRING_MARSHALLER))
|
||||||
?.let(this@Base::clean)
|
?.let(this@Base::clean)
|
||||||
?: ""
|
?: ""
|
||||||
val ips = ArrayList<InetAddress>()
|
val ips = ArrayList<InetAddress>()
|
||||||
remoteIpKeys.forEach { key ->
|
remoteIpKeys.forEach { key ->
|
||||||
metadata.get(key)?.let {
|
metadata.get(key)?.let {
|
||||||
it.trim().ifEmpty { null }
|
it.trim().ifEmpty { null }
|
||||||
?.let(this@Base::toInetAddress)
|
?.let(this@Base::toInetAddress)
|
||||||
?.let(ips::add)
|
?.let(ips::add)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
attributes.get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR)?.let { addr ->
|
attributes.get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR)?.let { addr ->
|
||||||
@@ -122,24 +125,26 @@ class EventsBuilder {
|
|||||||
}
|
}
|
||||||
val ip = findBestIp(ips)?.hostAddress ?: ""
|
val ip = findBestIp(ips)?.hostAddress ?: ""
|
||||||
this.requestDetails = this.requestDetails
|
this.requestDetails = this.requestDetails
|
||||||
.copy(remote = Events.Remote(
|
.copy(
|
||||||
ips = ips.map { it.hostAddress },
|
remote = Events.Remote(
|
||||||
ip = ip,
|
ips = ips.map { it.hostAddress },
|
||||||
userAgent = userAgent
|
ip = ip,
|
||||||
))
|
userAgent = userAgent
|
||||||
|
)
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun start(request: HttpServerRequest) {
|
override fun start(request: HttpServerRequest) {
|
||||||
val headers = request.requestHeaders()
|
val headers = request.requestHeaders()
|
||||||
val userAgent = headers.get("user-agent")
|
val userAgent = headers.get("user-agent")
|
||||||
?.let(this@Base::clean)
|
?.let(this@Base::clean)
|
||||||
?: ""
|
?: ""
|
||||||
val ips = ArrayList<InetAddress>()
|
val ips = ArrayList<InetAddress>()
|
||||||
remoteIpHeaders.forEach { key ->
|
remoteIpHeaders.forEach { key ->
|
||||||
headers.get(key)?.let {
|
headers.get(key)?.let {
|
||||||
it.trim().ifEmpty { null }
|
it.trim().ifEmpty { null }
|
||||||
?.let(this@Base::toInetAddress)
|
?.let(this@Base::toInetAddress)
|
||||||
?.let(ips::add)
|
?.let(ips::add)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
request.remoteAddress()?.let { addr ->
|
request.remoteAddress()?.let { addr ->
|
||||||
@@ -147,11 +152,13 @@ class EventsBuilder {
|
|||||||
}
|
}
|
||||||
val ip = findBestIp(ips)?.hostAddress ?: ""
|
val ip = findBestIp(ips)?.hostAddress ?: ""
|
||||||
this.requestDetails = this.requestDetails
|
this.requestDetails = this.requestDetails
|
||||||
.copy(remote = Events.Remote(
|
.copy(
|
||||||
ips = ips.map { it.hostAddress },
|
remote = Events.Remote(
|
||||||
ip = ip,
|
ips = ips.map { it.hostAddress },
|
||||||
userAgent = userAgent
|
ip = ip,
|
||||||
))
|
userAgent = userAgent
|
||||||
|
)
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun withChain(chain: Int): T {
|
fun withChain(chain: Int): T {
|
||||||
@@ -161,9 +168,9 @@ class EventsBuilder {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class SubscribeHead() :
|
class SubscribeHead :
|
||||||
Base<SubscribeHead>(),
|
Base<SubscribeHead>(),
|
||||||
RequestReply<Events.SubscribeHead, Common.Chain, BlockchainOuterClass.ChainHead> {
|
RequestReply<Events.SubscribeHead, Common.Chain, BlockchainOuterClass.ChainHead> {
|
||||||
|
|
||||||
private var index = 0
|
private var index = 0
|
||||||
|
|
||||||
@@ -177,14 +184,14 @@ class EventsBuilder {
|
|||||||
|
|
||||||
override fun onReply(msg: BlockchainOuterClass.ChainHead): Events.SubscribeHead {
|
override fun onReply(msg: BlockchainOuterClass.ChainHead): Events.SubscribeHead {
|
||||||
return Events.SubscribeHead(
|
return Events.SubscribeHead(
|
||||||
chain, UUID.randomUUID(), requestDetails, index++
|
chain, UUID.randomUUID(), requestDetails, index++
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class SubscribeBalance(val subscribe: Boolean) :
|
class SubscribeBalance(val subscribe: Boolean) :
|
||||||
Base<SubscribeBalance>(),
|
Base<SubscribeBalance>(),
|
||||||
RequestReply<Events.SubscribeBalance, BlockchainOuterClass.BalanceRequest, BlockchainOuterClass.AddressBalance> {
|
RequestReply<Events.SubscribeBalance, BlockchainOuterClass.BalanceRequest, BlockchainOuterClass.AddressBalance> {
|
||||||
|
|
||||||
private var index = 0
|
private var index = 0
|
||||||
private var balanceRequest: Events.BalanceRequest? = null
|
private var balanceRequest: Events.BalanceRequest? = null
|
||||||
@@ -195,8 +202,8 @@ class EventsBuilder {
|
|||||||
|
|
||||||
override fun onRequest(msg: BlockchainOuterClass.BalanceRequest) {
|
override fun onRequest(msg: BlockchainOuterClass.BalanceRequest) {
|
||||||
balanceRequest = Events.BalanceRequest(
|
balanceRequest = Events.BalanceRequest(
|
||||||
msg.asset.code.uppercase(Locale.getDefault()),
|
msg.asset.code.uppercase(Locale.getDefault()),
|
||||||
msg.address.addrTypeCase.name
|
msg.address.addrTypeCase.name
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -207,14 +214,14 @@ class EventsBuilder {
|
|||||||
val addressBalance = Events.AddressBalance(msg.asset.code, msg.address.address)
|
val addressBalance = Events.AddressBalance(msg.asset.code, msg.address.address)
|
||||||
val chain = Chain.byId(msg.asset.chain.number)
|
val chain = Chain.byId(msg.asset.chain.number)
|
||||||
return Events.SubscribeBalance(
|
return Events.SubscribeBalance(
|
||||||
chain, UUID.randomUUID(), subscribe, requestDetails, balanceRequest!!, addressBalance, index++
|
chain, UUID.randomUUID(), subscribe, requestDetails, balanceRequest!!, addressBalance, index++
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class TxStatus() :
|
class TxStatus :
|
||||||
Base<TxStatus>(),
|
Base<TxStatus>(),
|
||||||
RequestReply<Events.TxStatus, BlockchainOuterClass.TxStatusRequest, BlockchainOuterClass.TxStatus> {
|
RequestReply<Events.TxStatus, BlockchainOuterClass.TxStatusRequest, BlockchainOuterClass.TxStatus> {
|
||||||
private var index = 0
|
private var index = 0
|
||||||
private var txStatusRequest: Events.TxStatusRequest? = null
|
private var txStatusRequest: Events.TxStatusRequest? = null
|
||||||
|
|
||||||
@@ -225,21 +232,20 @@ class EventsBuilder {
|
|||||||
|
|
||||||
override fun onReply(msg: BlockchainOuterClass.TxStatus): Events.TxStatus {
|
override fun onReply(msg: BlockchainOuterClass.TxStatus): Events.TxStatus {
|
||||||
return Events.TxStatus(
|
return Events.TxStatus(
|
||||||
chain, UUID.randomUUID(), requestDetails, txStatusRequest!!,
|
chain, UUID.randomUUID(), requestDetails, txStatusRequest!!,
|
||||||
Events.TxStatusResponse(msg.confirmations),
|
Events.TxStatusResponse(msg.confirmations),
|
||||||
index++
|
index++
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun getT(): TxStatus {
|
override fun getT(): TxStatus {
|
||||||
return this
|
return this
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
class NativeCall :
|
class NativeCall :
|
||||||
Base<NativeCall>(),
|
Base<NativeCall>(),
|
||||||
RequestReply<Events.NativeCall, BlockchainOuterClass.NativeCallRequest, BlockchainOuterClass.NativeCallReplyItem> {
|
RequestReply<Events.NativeCall, BlockchainOuterClass.NativeCallRequest, BlockchainOuterClass.NativeCallReplyItem> {
|
||||||
val items = ArrayList<Events.NativeCallItemDetails>()
|
val items = ArrayList<Events.NativeCallItemDetails>()
|
||||||
val replies = HashMap<Int, Events.NativeCallReplyDetails>()
|
val replies = HashMap<Int, Events.NativeCallReplyDetails>()
|
||||||
private var index = 0
|
private var index = 0
|
||||||
@@ -252,11 +258,11 @@ class EventsBuilder {
|
|||||||
withChain(msg.chain.number)
|
withChain(msg.chain.number)
|
||||||
msg.itemsList.forEach { item ->
|
msg.itemsList.forEach { item ->
|
||||||
this.items.add(
|
this.items.add(
|
||||||
Events.NativeCallItemDetails(
|
Events.NativeCallItemDetails(
|
||||||
item.method,
|
item.method,
|
||||||
item.id,
|
item.id,
|
||||||
item.payload.size().toLong()
|
item.payload.size().toLong()
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -264,38 +270,40 @@ class EventsBuilder {
|
|||||||
override fun onReply(msg: BlockchainOuterClass.NativeCallReplyItem): Events.NativeCall {
|
override fun onReply(msg: BlockchainOuterClass.NativeCallReplyItem): Events.NativeCall {
|
||||||
val item = items.find { it.id == msg.id }!!
|
val item = items.find { it.id == msg.id }!!
|
||||||
return Events.NativeCall(
|
return Events.NativeCall(
|
||||||
request = requestDetails,
|
request = requestDetails,
|
||||||
total = items.size,
|
total = items.size,
|
||||||
index = index++,
|
index = index++,
|
||||||
succeed = msg.succeed,
|
succeed = msg.succeed,
|
||||||
blockchain = chain,
|
blockchain = chain,
|
||||||
nativeCall = item,
|
nativeCall = item,
|
||||||
payloadSizeBytes = item.payloadSizeBytes,
|
payloadSizeBytes = item.payloadSizeBytes,
|
||||||
id = UUID.randomUUID(),
|
id = UUID.randomUUID(),
|
||||||
channel = Events.Channel.GRPC
|
channel = Events.Channel.GRPC
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun onReply(reply: io.emeraldpay.dshackle.rpc.NativeCall.CallResult,
|
fun onReply(
|
||||||
channel: Events.Channel): Events.NativeCall {
|
reply: io.emeraldpay.dshackle.rpc.NativeCall.CallResult,
|
||||||
|
channel: Events.Channel
|
||||||
|
): Events.NativeCall {
|
||||||
val item = items.find { it.id == reply.id }!!
|
val item = items.find { it.id == reply.id }!!
|
||||||
return Events.NativeCall(
|
return Events.NativeCall(
|
||||||
request = requestDetails,
|
request = requestDetails,
|
||||||
total = items.size,
|
total = items.size,
|
||||||
index = index++,
|
index = index++,
|
||||||
succeed = !reply.isError(),
|
succeed = !reply.isError(),
|
||||||
blockchain = chain,
|
blockchain = chain,
|
||||||
nativeCall = item,
|
nativeCall = item,
|
||||||
payloadSizeBytes = item.payloadSizeBytes,
|
payloadSizeBytes = item.payloadSizeBytes,
|
||||||
id = UUID.randomUUID(),
|
id = UUID.randomUUID(),
|
||||||
channel = channel
|
channel = channel
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class NativeSubscribe :
|
class NativeSubscribe :
|
||||||
Base<NativeSubscribe>(),
|
Base<NativeSubscribe>(),
|
||||||
RequestReply<Events.NativeSubscribe, BlockchainOuterClass.NativeSubscribeRequest, BlockchainOuterClass.NativeSubscribeReplyItem> {
|
RequestReply<Events.NativeSubscribe, BlockchainOuterClass.NativeSubscribeRequest, BlockchainOuterClass.NativeSubscribeReplyItem> {
|
||||||
var item: Events.NativeSubscribeItemDetails? = null
|
var item: Events.NativeSubscribeItemDetails? = null
|
||||||
val replies = HashMap<Int, Events.NativeSubscribeReplyDetails>()
|
val replies = HashMap<Int, Events.NativeSubscribeReplyDetails>()
|
||||||
|
|
||||||
@@ -306,26 +314,26 @@ class EventsBuilder {
|
|||||||
override fun onRequest(msg: BlockchainOuterClass.NativeSubscribeRequest) {
|
override fun onRequest(msg: BlockchainOuterClass.NativeSubscribeRequest) {
|
||||||
withChain(msg.chain.number)
|
withChain(msg.chain.number)
|
||||||
this.item = Events.NativeSubscribeItemDetails(
|
this.item = Events.NativeSubscribeItemDetails(
|
||||||
msg.method,
|
msg.method,
|
||||||
msg.payload.size().toLong()
|
msg.payload.size().toLong()
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun onReply(msg: BlockchainOuterClass.NativeSubscribeReplyItem): Events.NativeSubscribe {
|
override fun onReply(msg: BlockchainOuterClass.NativeSubscribeReplyItem): Events.NativeSubscribe {
|
||||||
return Events.NativeSubscribe(
|
return Events.NativeSubscribe(
|
||||||
request = requestDetails,
|
request = requestDetails,
|
||||||
blockchain = chain,
|
blockchain = chain,
|
||||||
nativeSubscribe = item!!,
|
nativeSubscribe = item!!,
|
||||||
payloadSizeBytes = msg.payload?.size()?.toLong() ?: 0L,
|
payloadSizeBytes = msg.payload?.size()?.toLong() ?: 0L,
|
||||||
id = UUID.randomUUID(),
|
id = UUID.randomUUID(),
|
||||||
channel = Events.Channel.GRPC
|
channel = Events.Channel.GRPC
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class Describe :
|
class Describe :
|
||||||
Base<Describe>(),
|
Base<Describe>(),
|
||||||
RequestReply<Events.Describe, BlockchainOuterClass.DescribeRequest, BlockchainOuterClass.DescribeResponse> {
|
RequestReply<Events.Describe, BlockchainOuterClass.DescribeRequest, BlockchainOuterClass.DescribeResponse> {
|
||||||
|
|
||||||
override fun getT(): Describe {
|
override fun getT(): Describe {
|
||||||
return this
|
return this
|
||||||
@@ -336,15 +344,15 @@ class EventsBuilder {
|
|||||||
|
|
||||||
override fun onReply(msg: BlockchainOuterClass.DescribeResponse): Events.Describe {
|
override fun onReply(msg: BlockchainOuterClass.DescribeResponse): Events.Describe {
|
||||||
return Events.Describe(
|
return Events.Describe(
|
||||||
id = UUID.randomUUID(),
|
id = UUID.randomUUID(),
|
||||||
request = requestDetails
|
request = requestDetails
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class Status :
|
class Status :
|
||||||
Base<Status>(),
|
Base<Status>(),
|
||||||
RequestReply<Events.Status, BlockchainOuterClass.StatusRequest, BlockchainOuterClass.ChainStatus> {
|
RequestReply<Events.Status, BlockchainOuterClass.StatusRequest, BlockchainOuterClass.ChainStatus> {
|
||||||
override fun getT(): Status {
|
override fun getT(): Status {
|
||||||
return this
|
return this
|
||||||
}
|
}
|
||||||
@@ -355,11 +363,10 @@ class EventsBuilder {
|
|||||||
override fun onReply(msg: BlockchainOuterClass.ChainStatus): Events.Status {
|
override fun onReply(msg: BlockchainOuterClass.ChainStatus): Events.Status {
|
||||||
val chain = Chain.byId(msg.chainValue)
|
val chain = Chain.byId(msg.chainValue)
|
||||||
return Events.Status(
|
return Events.Status(
|
||||||
blockchain = chain,
|
blockchain = chain,
|
||||||
request = requestDetails,
|
request = requestDetails,
|
||||||
id = UUID.randomUUID()
|
id = UUID.randomUUID()
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -23,10 +23,10 @@ import org.slf4j.LoggerFactory
|
|||||||
* JSON RPC call to the proxy
|
* JSON RPC call to the proxy
|
||||||
*/
|
*/
|
||||||
class ProxyCall(
|
class ProxyCall(
|
||||||
/**
|
/**
|
||||||
* Type of the request. The response format depends on it
|
* Type of the request. The response format depends on it
|
||||||
*/
|
*/
|
||||||
val type: RpcType
|
val type: RpcType
|
||||||
) {
|
) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -55,4 +55,4 @@ class ProxyCall(
|
|||||||
*/
|
*/
|
||||||
BATCH
|
BATCH
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,8 +24,8 @@ import io.emeraldpay.dshackle.config.ProxyConfig
|
|||||||
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp
|
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp
|
||||||
import io.emeraldpay.dshackle.rpc.NativeCall
|
import io.emeraldpay.dshackle.rpc.NativeCall
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
||||||
import io.emeraldpay.grpc.Chain
|
|
||||||
import io.emeraldpay.etherjar.rpc.RpcException
|
import io.emeraldpay.etherjar.rpc.RpcException
|
||||||
|
import io.emeraldpay.grpc.Chain
|
||||||
import io.micrometer.core.instrument.Counter
|
import io.micrometer.core.instrument.Counter
|
||||||
import io.micrometer.core.instrument.Metrics
|
import io.micrometer.core.instrument.Metrics
|
||||||
import io.micrometer.core.instrument.Timer
|
import io.micrometer.core.instrument.Timer
|
||||||
@@ -38,16 +38,14 @@ import org.slf4j.LoggerFactory
|
|||||||
import org.springframework.http.HttpHeaders
|
import org.springframework.http.HttpHeaders
|
||||||
import reactor.core.publisher.Flux
|
import reactor.core.publisher.Flux
|
||||||
import reactor.core.publisher.Mono
|
import reactor.core.publisher.Mono
|
||||||
import reactor.netty.DisposableServer
|
|
||||||
import reactor.netty.http.server.HttpServer
|
import reactor.netty.http.server.HttpServer
|
||||||
import reactor.netty.http.server.HttpServerRequest
|
import reactor.netty.http.server.HttpServerRequest
|
||||||
import reactor.netty.http.server.HttpServerResponse
|
import reactor.netty.http.server.HttpServerResponse
|
||||||
import reactor.netty.http.server.HttpServerRoutes
|
import reactor.netty.http.server.HttpServerRoutes
|
||||||
import java.util.*
|
import java.util.EnumMap
|
||||||
import java.util.concurrent.TimeUnit
|
import java.util.concurrent.TimeUnit
|
||||||
import java.util.concurrent.locks.ReentrantReadWriteLock
|
import java.util.concurrent.locks.ReentrantReadWriteLock
|
||||||
import java.util.function.BiFunction
|
import java.util.function.BiFunction
|
||||||
import kotlin.collections.HashMap
|
|
||||||
import kotlin.concurrent.read
|
import kotlin.concurrent.read
|
||||||
import kotlin.concurrent.write
|
import kotlin.concurrent.write
|
||||||
|
|
||||||
@@ -55,12 +53,12 @@ import kotlin.concurrent.write
|
|||||||
* HTTP Proxy Server
|
* HTTP Proxy Server
|
||||||
*/
|
*/
|
||||||
class ProxyServer(
|
class ProxyServer(
|
||||||
private var config: ProxyConfig,
|
private var config: ProxyConfig,
|
||||||
private val readRpcJson: ReadRpcJson,
|
private val readRpcJson: ReadRpcJson,
|
||||||
private val writeRpcJson: WriteRpcJson,
|
private val writeRpcJson: WriteRpcJson,
|
||||||
private val nativeCall: NativeCall,
|
private val nativeCall: NativeCall,
|
||||||
private val tlsSetup: TlsSetup,
|
private val tlsSetup: TlsSetup,
|
||||||
private val accessHandler: AccessHandlerHttp.HandlerFactory
|
private val accessHandler: AccessHandlerHttp.HandlerFactory
|
||||||
) {
|
) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -101,19 +99,19 @@ class ProxyServer(
|
|||||||
}
|
}
|
||||||
log.info("Listening Proxy on ${config.host}:${config.port}")
|
log.info("Listening Proxy on ${config.host}:${config.port}")
|
||||||
var serverBuilder = HttpServer.create()
|
var serverBuilder = HttpServer.create()
|
||||||
.doOnChannelInit { _, channel, _ ->
|
.doOnChannelInit { _, channel, _ ->
|
||||||
channel.pipeline().addFirst(errorHandler)
|
channel.pipeline().addFirst(errorHandler)
|
||||||
}
|
}
|
||||||
.host(config.host)
|
.host(config.host)
|
||||||
.port(config.port)
|
.port(config.port)
|
||||||
|
|
||||||
tlsSetup.setupServer("proxy", config.tls, false)?.let { sslContext ->
|
tlsSetup.setupServer("proxy", config.tls, false)?.let { sslContext ->
|
||||||
serverBuilder = serverBuilder.secure { secure -> secure.sslContext(sslContext) }
|
serverBuilder = serverBuilder.secure { secure -> secure.sslContext(sslContext) }
|
||||||
}
|
}
|
||||||
|
|
||||||
serverBuilder
|
serverBuilder
|
||||||
.route(this::setupRoutes)
|
.route(this::setupRoutes)
|
||||||
.bindNow()
|
.bindNow()
|
||||||
}
|
}
|
||||||
|
|
||||||
fun setupRoutes(routes: HttpServerRoutes) {
|
fun setupRoutes(routes: HttpServerRoutes) {
|
||||||
@@ -139,26 +137,26 @@ class ProxyServer(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
val request = BlockchainOuterClass.NativeCallRequest.newBuilder()
|
val request = BlockchainOuterClass.NativeCallRequest.newBuilder()
|
||||||
.setChain(Common.ChainRef.forNumber(chain.id))
|
.setChain(Common.ChainRef.forNumber(chain.id))
|
||||||
.addAllItems(call.items)
|
.addAllItems(call.items)
|
||||||
.build()
|
.build()
|
||||||
handler.onRequest(request)
|
handler.onRequest(request)
|
||||||
val jsons = nativeCall
|
val jsons = nativeCall
|
||||||
.nativeCallResult(Mono.just(request))
|
.nativeCallResult(Mono.just(request))
|
||||||
.doOnNext {
|
.doOnNext {
|
||||||
metricById(it.id)?.requestMetric?.increment()
|
metricById(it.id)?.requestMetric?.increment()
|
||||||
|
}
|
||||||
|
.doOnNext {
|
||||||
|
handler.onResponse(it)
|
||||||
|
metricById(it.id)?.callMetric?.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS)
|
||||||
|
}
|
||||||
|
.doOnError {
|
||||||
|
// when error happened the whole flux is stopped and no result is produced, so we should mark all the requests as failed
|
||||||
|
call.items.forEach { item ->
|
||||||
|
requestMetrics.get(chain, item.method).errorMetric.increment()
|
||||||
}
|
}
|
||||||
.doOnNext {
|
}
|
||||||
handler.onResponse(it)
|
.transform(writeRpcJson.toJsons(call))
|
||||||
metricById(it.id)?.callMetric?.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS)
|
|
||||||
}
|
|
||||||
.doOnError {
|
|
||||||
//when error happened the whole flux is stopped and no result is produced, so we should mark all the requests as failed
|
|
||||||
call.items.forEach { item ->
|
|
||||||
requestMetrics.get(chain, item.method).errorMetric.increment()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
.transform(writeRpcJson.toJsons(call))
|
|
||||||
return if (call.type == ProxyCall.RpcType.SINGLE) {
|
return if (call.type == ProxyCall.RpcType.SINGLE) {
|
||||||
jsons.next()
|
jsons.next()
|
||||||
} else {
|
} else {
|
||||||
@@ -166,21 +164,25 @@ class ProxyServer(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun processRequest(chain: Chain, request: Mono<ByteArray>, handler: AccessHandlerHttp.RequestHandler): Flux<ByteBuf> {
|
fun processRequest(
|
||||||
|
chain: Chain,
|
||||||
|
request: Mono<ByteArray>,
|
||||||
|
handler: AccessHandlerHttp.RequestHandler
|
||||||
|
): Flux<ByteBuf> {
|
||||||
return request
|
return request
|
||||||
.map(readRpcJson)
|
.map(readRpcJson)
|
||||||
.flatMapMany { call ->
|
.flatMapMany { call ->
|
||||||
execute(chain, call, handler)
|
execute(chain, call, handler)
|
||||||
}
|
}
|
||||||
.onErrorResume(RpcException::class.java) { err ->
|
.onErrorResume(RpcException::class.java) { err ->
|
||||||
val id = err.details?.let {
|
val id = err.details?.let {
|
||||||
if (it is JsonRpcResponse.Id) it else JsonRpcResponse.NumberId(-1)
|
if (it is JsonRpcResponse.Id) it else JsonRpcResponse.NumberId(-1)
|
||||||
} ?: JsonRpcResponse.NumberId(-1)
|
} ?: JsonRpcResponse.NumberId(-1)
|
||||||
|
|
||||||
val json = JsonRpcResponse.error(err.code, err.rpcMessage, id)
|
val json = JsonRpcResponse.error(err.code, err.rpcMessage, id)
|
||||||
Mono.just(Global.objectMapper.writeValueAsString(json))
|
Mono.just(Global.objectMapper.writeValueAsString(json))
|
||||||
}
|
}
|
||||||
.map { Unpooled.wrappedBuffer(it.toByteArray()) }
|
.map { Unpooled.wrappedBuffer(it.toByteArray()) }
|
||||||
}
|
}
|
||||||
|
|
||||||
fun proxy(routeConfig: ProxyConfig.Route): BiFunction<HttpServerRequest, HttpServerResponse, Publisher<Void>> {
|
fun proxy(routeConfig: ProxyConfig.Route): BiFunction<HttpServerRequest, HttpServerResponse, Publisher<Void>> {
|
||||||
@@ -188,13 +190,13 @@ class ProxyServer(
|
|||||||
// handle access events
|
// handle access events
|
||||||
val eventHandler = accessHandler.create(req, routeConfig.blockchain)
|
val eventHandler = accessHandler.create(req, routeConfig.blockchain)
|
||||||
val request = req.receive()
|
val request = req.receive()
|
||||||
.aggregate()
|
.aggregate()
|
||||||
.asByteArray()
|
.asByteArray()
|
||||||
val results = processRequest(routeConfig.blockchain, request, eventHandler)
|
val results = processRequest(routeConfig.blockchain, request, eventHandler)
|
||||||
// make sure that the access log handler is closed at the end, so it can render the logs
|
// make sure that the access log handler is closed at the end, so it can render the logs
|
||||||
.doFinally { eventHandler.close() }
|
.doFinally { eventHandler.close() }
|
||||||
resp.addHeader(HttpHeaders.CONTENT_TYPE, "application/json")
|
resp.addHeader(HttpHeaders.CONTENT_TYPE, "application/json")
|
||||||
.send(results)
|
.send(results)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,25 +253,25 @@ class ProxyServer(
|
|||||||
|
|
||||||
class RequestMetricsBasic(chain: Chain) : RequestMetrics {
|
class RequestMetricsBasic(chain: Chain) : RequestMetrics {
|
||||||
override val callMetric = Timer.builder("request.jsonrpc.call")
|
override val callMetric = Timer.builder("request.jsonrpc.call")
|
||||||
.tags("chain", chain.chainCode)
|
.tags("chain", chain.chainCode)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
override val errorMetric = Counter.builder("request.jsonrpc.err")
|
override val errorMetric = Counter.builder("request.jsonrpc.err")
|
||||||
.tags("chain", chain.chainCode)
|
.tags("chain", chain.chainCode)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
override val requestMetric = Counter.builder("request.jsonrpc.request.total")
|
override val requestMetric = Counter.builder("request.jsonrpc.request.total")
|
||||||
.tags("chain", chain.chainCode)
|
.tags("chain", chain.chainCode)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
}
|
}
|
||||||
|
|
||||||
class RequestMetricsWithMethod(chain: Chain, method: String) : RequestMetrics {
|
class RequestMetricsWithMethod(chain: Chain, method: String) : RequestMetrics {
|
||||||
override val callMetric = Timer.builder("request.jsonrpc.call")
|
override val callMetric = Timer.builder("request.jsonrpc.call")
|
||||||
.tags("chain", chain.chainCode, "method", method)
|
.tags("chain", chain.chainCode, "method", method)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
override val errorMetric = Counter.builder("request.jsonrpc.err")
|
override val errorMetric = Counter.builder("request.jsonrpc.err")
|
||||||
.tags("chain", chain.chainCode, "method", method)
|
.tags("chain", chain.chainCode, "method", method)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
override val requestMetric = Counter.builder("request.jsonrpc.request.total")
|
override val requestMetric = Counter.builder("request.jsonrpc.request.total")
|
||||||
.tags("chain", chain.chainCode, "method", method)
|
.tags("chain", chain.chainCode, "method", method)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,19 +25,16 @@ import io.emeraldpay.etherjar.rpc.RpcException
|
|||||||
import io.emeraldpay.etherjar.rpc.RpcResponseError
|
import io.emeraldpay.etherjar.rpc.RpcResponseError
|
||||||
import io.emeraldpay.etherjar.rpc.json.RequestJson
|
import io.emeraldpay.etherjar.rpc.json.RequestJson
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
|
||||||
import org.springframework.stereotype.Service
|
import org.springframework.stereotype.Service
|
||||||
import java.io.IOException
|
import java.io.IOException
|
||||||
import java.util.*
|
|
||||||
import java.util.function.Function
|
import java.util.function.Function
|
||||||
import java.util.stream.Collectors
|
import java.util.stream.Collectors
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reader for JSON RPC request
|
* Reader for JSON RPC request
|
||||||
*/
|
*/
|
||||||
@Service
|
@Service
|
||||||
open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
|
open class ReadRpcJson : Function<ByteArray, ProxyCall> {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private val log = LoggerFactory.getLogger(ReadRpcJson::class.java)
|
private val log = LoggerFactory.getLogger(ReadRpcJson::class.java)
|
||||||
@@ -55,21 +52,37 @@ open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
|
|||||||
val id = json["id"]
|
val id = json["id"]
|
||||||
if ("2.0" != json["jsonrpc"]) {
|
if ("2.0" != json["jsonrpc"]) {
|
||||||
if (json["jsonrpc"] == null) {
|
if (json["jsonrpc"] == null) {
|
||||||
throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "jsonrpc version is not set", id?.let { JsonRpcResponse.Id.from(it) })
|
throw RpcException(
|
||||||
|
RpcResponseError.CODE_INVALID_REQUEST,
|
||||||
|
"jsonrpc version is not set",
|
||||||
|
id?.let { JsonRpcResponse.Id.from(it) }
|
||||||
|
)
|
||||||
}
|
}
|
||||||
throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "Unsupported JSON RPC version: " + json["jsonrpc"].toString(), id?.let { JsonRpcResponse.Id.from(it) })
|
throw RpcException(
|
||||||
|
RpcResponseError.CODE_INVALID_REQUEST,
|
||||||
|
"Unsupported JSON RPC version: " + json["jsonrpc"].toString(),
|
||||||
|
id?.let { JsonRpcResponse.Id.from(it) }
|
||||||
|
)
|
||||||
}
|
}
|
||||||
if (!(json["method"] != null && json["method"] is String)) {
|
if (!(json["method"] != null && json["method"] is String)) {
|
||||||
throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "Method is not set", id?.let { JsonRpcResponse.Id.from(it) })
|
throw RpcException(
|
||||||
|
RpcResponseError.CODE_INVALID_REQUEST,
|
||||||
|
"Method is not set",
|
||||||
|
id?.let { JsonRpcResponse.Id.from(it) }
|
||||||
|
)
|
||||||
}
|
}
|
||||||
if (json.containsKey("params") && json["params"] !is List<*>) {
|
if (json.containsKey("params") && json["params"] !is List<*>) {
|
||||||
throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "Params must be an array", id?.let { JsonRpcResponse.Id.from(it) })
|
throw RpcException(
|
||||||
|
RpcResponseError.CODE_INVALID_REQUEST,
|
||||||
|
"Params must be an array",
|
||||||
|
id?.let { JsonRpcResponse.Id.from(it) }
|
||||||
|
)
|
||||||
}
|
}
|
||||||
RequestJson<Any>(
|
RequestJson<Any>(
|
||||||
json["method"].toString(),
|
json["method"].toString(),
|
||||||
//params MAY be omitted
|
// params MAY be omitted
|
||||||
(json["params"] ?: emptyList<Any>()) as List<*>,
|
(json["params"] ?: emptyList<Any>()) as List<*>,
|
||||||
id
|
id
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -81,7 +94,7 @@ open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
|
|||||||
fun getStartOfJson(buf: ByteArray): Byte {
|
fun getStartOfJson(buf: ByteArray): Byte {
|
||||||
val count = buf.size
|
val count = buf.size
|
||||||
var i = 0
|
var i = 0
|
||||||
//if cannot find anything in the first 255 bytes, just consider it as invalid
|
// if cannot find anything in the first 255 bytes, just consider it as invalid
|
||||||
while (i < 256 && i < count) {
|
while (i < 256 && i < count) {
|
||||||
if (buf[i] != spaces[0] && buf[i] != spaces[1] && buf[i] != spaces[2]) {
|
if (buf[i] != spaces[0] && buf[i] != spaces[1] && buf[i] != spaces[2]) {
|
||||||
return buf[i]
|
return buf[i]
|
||||||
@@ -127,17 +140,17 @@ open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
|
|||||||
// our internal ids for calls
|
// our internal ids for calls
|
||||||
var seq = 0
|
var seq = 0
|
||||||
val batch = list.stream()
|
val batch = list.stream()
|
||||||
.map<RequestJson<Any>>(jsonExtractor)
|
.map<RequestJson<Any>>(jsonExtractor)
|
||||||
.map { json ->
|
.map { json ->
|
||||||
val id = seq++
|
val id = seq++
|
||||||
context.ids[id] = json.id
|
context.ids[id] = json.id
|
||||||
BlockchainOuterClass.NativeCallItem.newBuilder()
|
BlockchainOuterClass.NativeCallItem.newBuilder()
|
||||||
.setId(id)
|
.setId(id)
|
||||||
.setMethod(json.method)
|
.setMethod(json.method)
|
||||||
.setPayload(ByteString.copyFrom(objectMapper.writeValueAsBytes(json.params)))
|
.setPayload(ByteString.copyFrom(objectMapper.writeValueAsBytes(json.params)))
|
||||||
.build()
|
.build()
|
||||||
}
|
}
|
||||||
.collect(Collectors.toList())
|
.collect(Collectors.toList())
|
||||||
context.items.addAll(batch)
|
context.items.addAll(batch)
|
||||||
return context
|
return context
|
||||||
} catch (e: RpcException) {
|
} catch (e: RpcException) {
|
||||||
@@ -147,5 +160,4 @@ open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
|
|||||||
throw RpcException(RpcResponseError.CODE_INVALID_JSON, e.message)
|
throw RpcException(RpcResponseError.CODE_INVALID_JSON, e.message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -17,7 +17,6 @@
|
|||||||
package io.emeraldpay.dshackle.proxy
|
package io.emeraldpay.dshackle.proxy
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper
|
import com.fasterxml.jackson.databind.ObjectMapper
|
||||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
|
||||||
import io.emeraldpay.dshackle.Global
|
import io.emeraldpay.dshackle.Global
|
||||||
import io.emeraldpay.dshackle.rpc.NativeCall
|
import io.emeraldpay.dshackle.rpc.NativeCall
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
||||||
@@ -31,7 +30,7 @@ import java.util.function.Function
|
|||||||
* Writer for JSON RPC requests
|
* Writer for JSON RPC requests
|
||||||
*/
|
*/
|
||||||
@Service
|
@Service
|
||||||
open class WriteRpcJson() {
|
open class WriteRpcJson {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private val log = LoggerFactory.getLogger(WriteRpcJson::class.java)
|
private val log = LoggerFactory.getLogger(WriteRpcJson::class.java)
|
||||||
@@ -45,35 +44,35 @@ open class WriteRpcJson() {
|
|||||||
open fun toJsons(call: ProxyCall): Function<Flux<NativeCall.CallResult>, Flux<String>> {
|
open fun toJsons(call: ProxyCall): Function<Flux<NativeCall.CallResult>, Flux<String>> {
|
||||||
return Function { flux ->
|
return Function { flux ->
|
||||||
flux
|
flux
|
||||||
.flatMap { response ->
|
.flatMap { response ->
|
||||||
if (!call.ids.containsKey(response.id)) {
|
if (!call.ids.containsKey(response.id)) {
|
||||||
log.warn("ID wasn't requested: ${response.id}")
|
log.warn("ID wasn't requested: ${response.id}")
|
||||||
return@flatMap Flux.empty<String>()
|
return@flatMap Flux.empty<String>()
|
||||||
}
|
|
||||||
val json = toJson(call, response)
|
|
||||||
if (json == null) {
|
|
||||||
Flux.empty<String>()
|
|
||||||
} else {
|
|
||||||
Flux.just(json)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
.onErrorResume { t ->
|
val json = toJson(call, response)
|
||||||
if (t is NativeCall.CallFailure) {
|
if (json == null) {
|
||||||
Mono.just(toJson(call, t)!!)
|
Flux.empty<String>()
|
||||||
} else {
|
} else {
|
||||||
Mono.empty()
|
Flux.just(json)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
.onErrorContinue { t, _ ->
|
}
|
||||||
log.warn("Failed to convert to JSON", t)
|
.onErrorResume { t ->
|
||||||
|
if (t is NativeCall.CallFailure) {
|
||||||
|
Mono.just(toJson(call, t)!!)
|
||||||
|
} else {
|
||||||
|
Mono.empty()
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
.onErrorContinue { t, _ ->
|
||||||
|
log.warn("Failed to convert to JSON", t)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
open fun toJson(call: ProxyCall, response: NativeCall.CallResult): String? {
|
open fun toJson(call: ProxyCall, response: NativeCall.CallResult): String? {
|
||||||
val id = call.ids[response.id]?.let {
|
val id = call.ids[response.id]?.let {
|
||||||
JsonRpcResponse.Id.from(it)
|
JsonRpcResponse.Id.from(it)
|
||||||
} ?: return null;
|
} ?: return null
|
||||||
val json = if (response.isError()) {
|
val json = if (response.isError()) {
|
||||||
val error = response.error!!
|
val error = response.error!!
|
||||||
error.upstreamError?.let { upstreamError ->
|
error.upstreamError?.let { upstreamError ->
|
||||||
@@ -86,7 +85,7 @@ open class WriteRpcJson() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fun toJson(call: ProxyCall, error: NativeCall.CallFailure): String? {
|
fun toJson(call: ProxyCall, error: NativeCall.CallFailure): String? {
|
||||||
val id = call.ids[error.id] ?: return null;
|
val id = call.ids[error.id] ?: return null
|
||||||
val json = JsonRpcResponse.error(-32003, error.reason.message ?: "", JsonRpcResponse.Id.from(id))
|
val json = JsonRpcResponse.error(-32003, error.reason.message ?: "", JsonRpcResponse.Id.from(id))
|
||||||
return objectMapper.writeValueAsString(json)
|
return objectMapper.writeValueAsString(json)
|
||||||
}
|
}
|
||||||
@@ -97,18 +96,18 @@ open class WriteRpcJson() {
|
|||||||
fun asArray(): Function<Flux<String>, Flux<String>> {
|
fun asArray(): Function<Flux<String>, Flux<String>> {
|
||||||
return Function { flux ->
|
return Function { flux ->
|
||||||
val body = flux.zipWith(Flux.concat(Mono.just(false), Flux.just(true).repeat()))
|
val body = flux.zipWith(Flux.concat(Mono.just(false), Flux.just(true).repeat()))
|
||||||
.map {
|
.map {
|
||||||
if (it.t2) {
|
if (it.t2) {
|
||||||
"," + it.t1
|
"," + it.t1
|
||||||
} else {
|
} else {
|
||||||
it.t1
|
it.t1
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
Flux.concat(
|
Flux.concat(
|
||||||
Mono.just("["),
|
Mono.just("["),
|
||||||
body,
|
body,
|
||||||
Mono.just("]")
|
Mono.just("]")
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,9 +20,8 @@ import io.emeraldpay.dshackle.upstream.Head
|
|||||||
import io.emeraldpay.dshackle.upstream.Upstream
|
import io.emeraldpay.dshackle.upstream.Upstream
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
||||||
import io.emeraldpay.etherjar.rpc.RpcException
|
|
||||||
|
|
||||||
open class AlwaysQuorum: CallQuorum {
|
open class AlwaysQuorum : CallQuorum {
|
||||||
|
|
||||||
private var resolved = false
|
private var resolved = false
|
||||||
private var result: ByteArray? = null
|
private var result: ByteArray? = null
|
||||||
@@ -60,4 +59,4 @@ open class AlwaysQuorum: CallQuorum {
|
|||||||
override fun toString(): String {
|
override fun toString(): String {
|
||||||
return "Quorum: Accept Any"
|
return "Quorum: Accept Any"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ import io.emeraldpay.dshackle.upstream.Head
|
|||||||
import io.emeraldpay.dshackle.upstream.Upstream
|
import io.emeraldpay.dshackle.upstream.Upstream
|
||||||
|
|
||||||
open class BroadcastQuorum(
|
open class BroadcastQuorum(
|
||||||
val quorum: Int = 3
|
val quorum: Int = 3
|
||||||
) : CallQuorum, ValueAwareQuorum<String>(String::class.java) {
|
) : CallQuorum, ValueAwareQuorum<String>(String::class.java) {
|
||||||
|
|
||||||
private var result: ByteArray? = null
|
private var result: ByteArray? = null
|
||||||
@@ -61,4 +61,4 @@ open class BroadcastQuorum(
|
|||||||
override fun toString(): String {
|
override fun toString(): String {
|
||||||
return "Quorum: Broadcast to $quorum upstreams"
|
return "Quorum: Broadcast to $quorum upstreams"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,10 +20,6 @@ import io.emeraldpay.dshackle.upstream.Head
|
|||||||
import io.emeraldpay.dshackle.upstream.Upstream
|
import io.emeraldpay.dshackle.upstream.Upstream
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
||||||
import io.emeraldpay.etherjar.domain.TransactionId
|
|
||||||
import io.emeraldpay.etherjar.rpc.RpcException
|
|
||||||
import io.emeraldpay.etherjar.rpc.json.BlockJson
|
|
||||||
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
|
|
||||||
import reactor.util.function.Tuple2
|
import reactor.util.function.Tuple2
|
||||||
import java.util.function.BiFunction
|
import java.util.function.BiFunction
|
||||||
import java.util.function.Predicate
|
import java.util.function.Predicate
|
||||||
@@ -54,4 +50,4 @@ interface CallQuorum {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,15 +16,11 @@
|
|||||||
*/
|
*/
|
||||||
package io.emeraldpay.dshackle.quorum
|
package io.emeraldpay.dshackle.quorum
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper
|
|
||||||
import io.emeraldpay.dshackle.upstream.Head
|
import io.emeraldpay.dshackle.upstream.Head
|
||||||
import io.emeraldpay.dshackle.upstream.Upstream
|
import io.emeraldpay.dshackle.upstream.Upstream
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
|
||||||
import io.emeraldpay.etherjar.rpc.JacksonRpcConverter
|
|
||||||
import io.emeraldpay.etherjar.rpc.RpcException
|
|
||||||
|
|
||||||
open class NonEmptyQuorum(
|
open class NonEmptyQuorum(
|
||||||
val maxTries: Int = 3
|
val maxTries: Int = 3
|
||||||
) : CallQuorum, ValueAwareQuorum<Any>(Any::class.java) {
|
) : CallQuorum, ValueAwareQuorum<Any>(Any::class.java) {
|
||||||
|
|
||||||
private var result: ByteArray? = null
|
private var result: ByteArray? = null
|
||||||
@@ -59,4 +55,4 @@ open class NonEmptyQuorum(
|
|||||||
override fun toString(): String {
|
override fun toString(): String {
|
||||||
return "Quorum: Accept Non Error Result"
|
return "Quorum: Accept Non Error Result"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,17 +16,14 @@
|
|||||||
*/
|
*/
|
||||||
package io.emeraldpay.dshackle.quorum
|
package io.emeraldpay.dshackle.quorum
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper
|
|
||||||
import io.emeraldpay.dshackle.upstream.Head
|
import io.emeraldpay.dshackle.upstream.Head
|
||||||
import io.emeraldpay.dshackle.upstream.Upstream
|
import io.emeraldpay.dshackle.upstream.Upstream
|
||||||
import io.emeraldpay.etherjar.hex.HexQuantity
|
import io.emeraldpay.etherjar.hex.HexQuantity
|
||||||
import io.emeraldpay.etherjar.rpc.JacksonRpcConverter
|
|
||||||
import io.emeraldpay.etherjar.rpc.RpcException
|
|
||||||
import java.util.concurrent.locks.ReentrantLock
|
import java.util.concurrent.locks.ReentrantLock
|
||||||
import kotlin.concurrent.withLock
|
import kotlin.concurrent.withLock
|
||||||
|
|
||||||
open class NonceQuorum(
|
open class NonceQuorum(
|
||||||
val tries: Int = 3
|
val tries: Int = 3
|
||||||
) : CallQuorum, ValueAwareQuorum<String>(String::class.java) {
|
) : CallQuorum, ValueAwareQuorum<String>(String::class.java) {
|
||||||
|
|
||||||
private val lock = ReentrantLock()
|
private val lock = ReentrantLock()
|
||||||
@@ -74,4 +71,4 @@ open class NonceQuorum(
|
|||||||
override fun toString(): String {
|
override fun toString(): String {
|
||||||
return "Quorum: Confirm with $tries upstreams"
|
return "Quorum: Confirm with $tries upstreams"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ import io.emeraldpay.dshackle.upstream.Head
|
|||||||
import io.emeraldpay.dshackle.upstream.Upstream
|
import io.emeraldpay.dshackle.upstream.Upstream
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
||||||
import io.emeraldpay.etherjar.rpc.RpcException
|
|
||||||
import java.util.concurrent.atomic.AtomicReference
|
import java.util.concurrent.atomic.AtomicReference
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -28,7 +27,7 @@ import java.util.concurrent.atomic.AtomicReference
|
|||||||
*
|
*
|
||||||
* NOTE: NativeCall checks the quorums and applies a HeightSelector if NotLaggingQuorum is enabled for a call
|
* NOTE: NativeCall checks the quorums and applies a HeightSelector if NotLaggingQuorum is enabled for a call
|
||||||
*/
|
*/
|
||||||
class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum {
|
class NotLaggingQuorum(val maxLag: Long = 0) : CallQuorum {
|
||||||
|
|
||||||
private val result: AtomicReference<ByteArray> = AtomicReference()
|
private val result: AtomicReference<ByteArray> = AtomicReference()
|
||||||
private val failed = AtomicReference(false)
|
private val failed = AtomicReference(false)
|
||||||
@@ -73,4 +72,4 @@ class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum {
|
|||||||
override fun toString(): String {
|
override fun toString(): String {
|
||||||
return "Quorum: late <= $maxLag blocks"
|
return "Quorum: late <= $maxLag blocks"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,4 +35,4 @@ interface QuorumReaderFactory {
|
|||||||
return QuorumRpcReader(apis, quorum)
|
return QuorumRpcReader(apis, quorum)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,8 +30,8 @@ import reactor.util.function.Tuples
|
|||||||
* Makes request with applying Quorum
|
* Makes request with applying Quorum
|
||||||
*/
|
*/
|
||||||
class QuorumRpcReader(
|
class QuorumRpcReader(
|
||||||
private val apis: ApiSource,
|
private val apis: ApiSource,
|
||||||
private val quorum: CallQuorum
|
private val quorum: CallQuorum
|
||||||
) : Reader<JsonRpcRequest, QuorumRpcReader.Result> {
|
) : Reader<JsonRpcRequest, QuorumRpcReader.Result> {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -58,8 +58,8 @@ class QuorumRpcReader(
|
|||||||
val defaultResult: Mono<Result> = Mono.just(quorum).flatMap { q ->
|
val defaultResult: Mono<Result> = Mono.just(quorum).flatMap { q ->
|
||||||
if (q.isFailed()) {
|
if (q.isFailed()) {
|
||||||
Mono.error<Result>(
|
Mono.error<Result>(
|
||||||
q.getError()?.asException(JsonRpcResponse.NumberId(1))
|
q.getError()?.asException(JsonRpcResponse.NumberId(1))
|
||||||
?: RpcException(-32000, "Unknown Upstream error")
|
?: RpcException(-32000, "Unknown Upstream error")
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
log.warn("Did not get any result from upstream. Method [${key.method}] using [$q]")
|
log.warn("Did not get any result from upstream. Method [${key.method}] using [$q]")
|
||||||
@@ -68,72 +68,71 @@ class QuorumRpcReader(
|
|||||||
}
|
}
|
||||||
|
|
||||||
return Flux.from(apis)
|
return Flux.from(apis)
|
||||||
.takeUntil {
|
.takeUntil {
|
||||||
quorum.isFailed() || quorum.isResolved()
|
quorum.isFailed() || quorum.isResolved()
|
||||||
}
|
}
|
||||||
.flatMap { api ->
|
.flatMap { api ->
|
||||||
api.getApi()
|
api.getApi()
|
||||||
.read(key)
|
.read(key)
|
||||||
.flatMap { response ->
|
.flatMap { response ->
|
||||||
response.requireResult()
|
response.requireResult()
|
||||||
.onErrorResume { err ->
|
.onErrorResume { err ->
|
||||||
if (err is RpcException || err is JsonRpcException) {
|
if (err is RpcException || err is JsonRpcException) {
|
||||||
// on error notify quorum, it may use error message or other details
|
// on error notify quorum, it may use error message or other details
|
||||||
val cleanErr: JsonRpcException = when (err) {
|
val cleanErr: JsonRpcException = when (err) {
|
||||||
is RpcException -> JsonRpcException.from(err)
|
is RpcException -> JsonRpcException.from(err)
|
||||||
is JsonRpcException -> err
|
is JsonRpcException -> err
|
||||||
else -> throw IllegalStateException("Cannot convert from exception", err)
|
else -> throw IllegalStateException("Cannot convert from exception", err)
|
||||||
}
|
}
|
||||||
quorum.record(cleanErr, api)
|
quorum.record(cleanErr, api)
|
||||||
// it it's failed after that, then we don't need more calls, stop api source
|
// it it's failed after that, then we don't need more calls, stop api source
|
||||||
if (quorum.isFailed()) {
|
if (quorum.isFailed()) {
|
||||||
apis.resolve()
|
apis.resolve()
|
||||||
} else {
|
} else {
|
||||||
apis.request(1)
|
apis.request(1)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log.warn("Result processing error", err)
|
log.warn("Result processing error", err)
|
||||||
}
|
}
|
||||||
Mono.empty()
|
Mono.empty()
|
||||||
}
|
|
||||||
}
|
}
|
||||||
.map { Tuples.of(it, api) }
|
|
||||||
}
|
|
||||||
.retryWhen(retrySpec)
|
|
||||||
// record all correct responses until quorum reached
|
|
||||||
.reduce(quorum, { res, a ->
|
|
||||||
if (res.record(a.t1, a.t2)) {
|
|
||||||
apis.resolve()
|
|
||||||
} else {
|
|
||||||
apis.request(1)
|
|
||||||
}
|
|
||||||
res
|
|
||||||
})
|
|
||||||
// if last call resulted in error it's still possible that request was resolved correctly. i.e. for BroadcastQuorum
|
|
||||||
.onErrorResume { err ->
|
|
||||||
if (quorum.isResolved()) {
|
|
||||||
Mono.just(quorum)
|
|
||||||
} else {
|
|
||||||
Mono.error(err)
|
|
||||||
}
|
}
|
||||||
|
.map { Tuples.of(it, api) }
|
||||||
|
}
|
||||||
|
.retryWhen(retrySpec)
|
||||||
|
// record all correct responses until quorum reached
|
||||||
|
.reduce(quorum, { res, a ->
|
||||||
|
if (res.record(a.t1, a.t2)) {
|
||||||
|
apis.resolve()
|
||||||
|
} else {
|
||||||
|
apis.request(1)
|
||||||
}
|
}
|
||||||
.doOnNext {
|
res
|
||||||
if (!it.isResolved() && !it.isFailed()) {
|
})
|
||||||
log.debug("No quorum for ${key.method} using [${quorum}]. Error: ${it.getError()?.message ?: ""}")
|
// if last call resulted in error it's still possible that request was resolved correctly. i.e. for BroadcastQuorum
|
||||||
}
|
.onErrorResume { err ->
|
||||||
|
if (quorum.isResolved()) {
|
||||||
|
Mono.just(quorum)
|
||||||
|
} else {
|
||||||
|
Mono.error(err)
|
||||||
}
|
}
|
||||||
// return nothing if not resolved
|
}
|
||||||
.filter { it.isResolved() }
|
.doOnNext {
|
||||||
.map {
|
if (!it.isResolved() && !it.isFailed()) {
|
||||||
// TODO find actual quorum number
|
log.debug("No quorum for ${key.method} using [$quorum]. Error: ${it.getError()?.message ?: ""}")
|
||||||
QuorumRpcReader.Result(it.getResult()!!, 1)
|
|
||||||
}
|
}
|
||||||
.switchIfEmpty(defaultResult)
|
}
|
||||||
|
// return nothing if not resolved
|
||||||
|
.filter { it.isResolved() }
|
||||||
|
.map {
|
||||||
|
// TODO find actual quorum number
|
||||||
|
QuorumRpcReader.Result(it.getResult()!!, 1)
|
||||||
|
}
|
||||||
|
.switchIfEmpty(defaultResult)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class Result(
|
class Result(
|
||||||
val value: ByteArray,
|
val value: ByteArray,
|
||||||
val quorum: Int
|
val quorum: Int
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,18 +16,16 @@
|
|||||||
*/
|
*/
|
||||||
package io.emeraldpay.dshackle.quorum
|
package io.emeraldpay.dshackle.quorum
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper
|
|
||||||
import io.emeraldpay.dshackle.Global
|
import io.emeraldpay.dshackle.Global
|
||||||
import io.emeraldpay.dshackle.upstream.Upstream
|
import io.emeraldpay.dshackle.upstream.Upstream
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
||||||
import io.emeraldpay.etherjar.rpc.JacksonRpcConverter
|
|
||||||
import io.emeraldpay.etherjar.rpc.RpcException
|
import io.emeraldpay.etherjar.rpc.RpcException
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
|
|
||||||
abstract class ValueAwareQuorum<T>(
|
abstract class ValueAwareQuorum<T>(
|
||||||
val clazz: Class<T>
|
val clazz: Class<T>
|
||||||
): CallQuorum {
|
) : CallQuorum {
|
||||||
|
|
||||||
private val log = LoggerFactory.getLogger(ValueAwareQuorum::class.java)
|
private val log = LoggerFactory.getLogger(ValueAwareQuorum::class.java)
|
||||||
private var rpcError: JsonRpcError? = null
|
private var rpcError: JsonRpcError? = null
|
||||||
@@ -45,7 +43,7 @@ abstract class ValueAwareQuorum<T>(
|
|||||||
} catch (e: Exception) {
|
} catch (e: Exception) {
|
||||||
recordError(response, e.message, upstream)
|
recordError(response, e.message, upstream)
|
||||||
}
|
}
|
||||||
return isResolved();
|
return isResolved()
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun record(error: JsonRpcException, upstream: Upstream) {
|
override fun record(error: JsonRpcException, upstream: Upstream) {
|
||||||
@@ -60,4 +58,4 @@ abstract class ValueAwareQuorum<T>(
|
|||||||
override fun getError(): JsonRpcError? {
|
override fun getError(): JsonRpcError? {
|
||||||
return rpcError
|
return rpcError
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,15 +20,14 @@ import io.emeraldpay.dshackle.Defaults
|
|||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import reactor.core.publisher.Flux
|
import reactor.core.publisher.Flux
|
||||||
import reactor.core.publisher.Mono
|
import reactor.core.publisher.Mono
|
||||||
import java.time.Duration
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Composition of multiple readers.
|
* Composition of multiple readers.
|
||||||
* Reader returns first value returned by any of the source readers by checking one by one until one of them returns a non-empty result.
|
* Reader returns first value returned by any of the source readers by checking one by one until one of them returns a non-empty result.
|
||||||
*/
|
*/
|
||||||
class CompoundReader<K, D>(
|
class CompoundReader<K, D>(
|
||||||
private vararg val readers: Reader<K, D>
|
private vararg val readers: Reader<K, D>
|
||||||
): Reader<K, D> {
|
) : Reader<K, D> {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private val log = LoggerFactory.getLogger(CompoundReader::class.java)
|
private val log = LoggerFactory.getLogger(CompoundReader::class.java)
|
||||||
@@ -39,13 +38,12 @@ class CompoundReader<K, D>(
|
|||||||
return Mono.empty()
|
return Mono.empty()
|
||||||
}
|
}
|
||||||
return Flux.fromIterable(readers.asIterable())
|
return Flux.fromIterable(readers.asIterable())
|
||||||
.flatMap({ rdr ->
|
.flatMap({ rdr ->
|
||||||
rdr.read(key)
|
rdr.read(key)
|
||||||
.timeout(Defaults.timeoutInternal, Mono.empty())
|
.timeout(Defaults.timeoutInternal, Mono.empty())
|
||||||
.doOnError { t -> log.warn("Failed to read from $rdr", t) }
|
.doOnError { t -> log.warn("Failed to read from $rdr", t) }
|
||||||
.onErrorResume { Mono.empty() }
|
.onErrorResume { Mono.empty() }
|
||||||
}, 1)
|
}, 1)
|
||||||
.next()
|
.next()
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -18,9 +18,9 @@ package io.emeraldpay.dshackle.reader
|
|||||||
|
|
||||||
import reactor.core.publisher.Mono
|
import reactor.core.publisher.Mono
|
||||||
|
|
||||||
class EmptyReader<K, D>: Reader<K, D> {
|
class EmptyReader<K, D> : Reader<K, D> {
|
||||||
|
|
||||||
override fun read(key: K): Mono<D> {
|
override fun read(key: K): Mono<D> {
|
||||||
return Mono.empty()
|
return Mono.empty()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,5 +21,4 @@ import reactor.core.publisher.Mono
|
|||||||
interface Reader<in K, D> {
|
interface Reader<in K, D> {
|
||||||
|
|
||||||
fun read(key: K): Mono<D>
|
fun read(key: K): Mono<D>
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -22,22 +22,21 @@ import java.util.function.Function
|
|||||||
* Reader wrapper that maps the input key from ne value to another (ex. convert from Long to String)
|
* Reader wrapper that maps the input key from ne value to another (ex. convert from Long to String)
|
||||||
*/
|
*/
|
||||||
class RekeyingReader<K, K1, D>(
|
class RekeyingReader<K, K1, D>(
|
||||||
/**
|
/**
|
||||||
* Mapping between original Key and Key supported by the reader
|
* Mapping between original Key and Key supported by the reader
|
||||||
*/
|
*/
|
||||||
private val rekey: Function<K, K1>,
|
private val rekey: Function<K, K1>,
|
||||||
/**
|
/**
|
||||||
* Actual reader
|
* Actual reader
|
||||||
*/
|
*/
|
||||||
private val reader: Reader<K1, D>
|
private val reader: Reader<K1, D>
|
||||||
) : Reader<K, D> {
|
) : Reader<K, D> {
|
||||||
|
|
||||||
override fun read(key: K): Mono<D> {
|
override fun read(key: K): Mono<D> {
|
||||||
return Mono.just(key)
|
return Mono.just(key)
|
||||||
.map(rekey)
|
.map(rekey)
|
||||||
.flatMap {
|
.flatMap {
|
||||||
reader.read(it)
|
reader.read(it)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -25,8 +25,8 @@ import reactor.core.publisher.Mono
|
|||||||
* Reader that requests data through upstream RPC using provided JSON RPC request builder
|
* Reader that requests data through upstream RPC using provided JSON RPC request builder
|
||||||
*/
|
*/
|
||||||
class RpcReader<T>(
|
class RpcReader<T>(
|
||||||
private val up: Multistream,
|
private val up: Multistream,
|
||||||
private val paramsBuilder: (T) -> JsonRpcRequest
|
private val paramsBuilder: (T) -> JsonRpcRequest
|
||||||
) : Reader<T, ByteArray> {
|
) : Reader<T, ByteArray> {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -45,11 +45,10 @@ class RpcReader<T>(
|
|||||||
|
|
||||||
override fun read(key: T): Mono<ByteArray> {
|
override fun read(key: T): Mono<ByteArray> {
|
||||||
return up.getDirectApi(Selector.empty)
|
return up.getDirectApi(Selector.empty)
|
||||||
.flatMap { rdr ->
|
.flatMap { rdr ->
|
||||||
rdr.read(paramsBuilder(key)).flatMap {
|
rdr.read(paramsBuilder(key)).flatMap {
|
||||||
it.requireResult()
|
it.requireResult()
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -22,18 +22,17 @@ import java.util.function.Function
|
|||||||
* Reader wrapper that transforms output of the reader to a different format
|
* Reader wrapper that transforms output of the reader to a different format
|
||||||
*/
|
*/
|
||||||
class TransformingReader<K, D0, D>(
|
class TransformingReader<K, D0, D>(
|
||||||
/**
|
/**
|
||||||
* Actual reader
|
* Actual reader
|
||||||
*/
|
*/
|
||||||
private val reader: Reader<K, D0>,
|
private val reader: Reader<K, D0>,
|
||||||
/**
|
/**
|
||||||
* Result transformation
|
* Result transformation
|
||||||
*/
|
*/
|
||||||
private val transformer: Function<in D0, out D>
|
private val transformer: Function<in D0, out D>
|
||||||
) : Reader<K, D> {
|
) : Reader<K, D> {
|
||||||
|
|
||||||
override fun read(key: K): Mono<D> {
|
override fun read(key: K): Mono<D> {
|
||||||
return reader.read(key).map(transformer)
|
return reader.read(key).map(transformer)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -31,44 +31,45 @@ import org.springframework.context.annotation.DependsOn
|
|||||||
import org.springframework.stereotype.Service
|
import org.springframework.stereotype.Service
|
||||||
import reactor.core.publisher.Flux
|
import reactor.core.publisher.Flux
|
||||||
import reactor.core.publisher.Mono
|
import reactor.core.publisher.Mono
|
||||||
import java.util.*
|
import java.util.Locale
|
||||||
import java.util.concurrent.TimeUnit
|
import java.util.concurrent.TimeUnit
|
||||||
|
|
||||||
@Service @DependsOn("monitoringSetup")
|
@Service
|
||||||
|
@DependsOn("monitoringSetup")
|
||||||
class BlockchainRpc(
|
class BlockchainRpc(
|
||||||
@Autowired private val nativeCall: NativeCall,
|
@Autowired private val nativeCall: NativeCall,
|
||||||
@Autowired private val nativeSubscribe: NativeSubscribe,
|
@Autowired private val nativeSubscribe: NativeSubscribe,
|
||||||
@Autowired private val streamHead: StreamHead,
|
@Autowired private val streamHead: StreamHead,
|
||||||
@Autowired private val trackTx: List<TrackTx>,
|
@Autowired private val trackTx: List<TrackTx>,
|
||||||
@Autowired private val trackAddress: List<TrackAddress>,
|
@Autowired private val trackAddress: List<TrackAddress>,
|
||||||
@Autowired private val describe: Describe,
|
@Autowired private val describe: Describe,
|
||||||
@Autowired private val subscribeStatus: SubscribeStatus
|
@Autowired private val subscribeStatus: SubscribeStatus
|
||||||
): ReactorBlockchainGrpc.BlockchainImplBase() {
|
) : ReactorBlockchainGrpc.BlockchainImplBase() {
|
||||||
|
|
||||||
private val log = LoggerFactory.getLogger(BlockchainRpc::class.java)
|
private val log = LoggerFactory.getLogger(BlockchainRpc::class.java)
|
||||||
|
|
||||||
private val describeMetric = Counter.builder("request.grpc.request")
|
private val describeMetric = Counter.builder("request.grpc.request")
|
||||||
.tag("type", "describe")
|
.tag("type", "describe")
|
||||||
.tag("chain", "NA")
|
.tag("chain", "NA")
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
private val subscribeStatusMetric = Counter.builder("request.grpc.request")
|
private val subscribeStatusMetric = Counter.builder("request.grpc.request")
|
||||||
.tag("type", "subscribeStatus")
|
.tag("type", "subscribeStatus")
|
||||||
.tag("chain", "NA")
|
.tag("chain", "NA")
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
private val errorMetric = Counter.builder("request.grpc.err")
|
private val errorMetric = Counter.builder("request.grpc.err")
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
private val chainMetrics = ChainValue { chain -> RequestMetrics(chain) }
|
private val chainMetrics = ChainValue { chain -> RequestMetrics(chain) }
|
||||||
|
|
||||||
override fun nativeCall(request: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<BlockchainOuterClass.NativeCallReplyItem> {
|
override fun nativeCall(request: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<BlockchainOuterClass.NativeCallReplyItem> {
|
||||||
var startTime = 0L
|
var startTime = 0L
|
||||||
var metrics: RequestMetrics? = null
|
var metrics: RequestMetrics? = null
|
||||||
return nativeCall.nativeCall(
|
return nativeCall.nativeCall(
|
||||||
request
|
request
|
||||||
.doOnNext {
|
.doOnNext {
|
||||||
metrics = chainMetrics.get(it.chain)
|
metrics = chainMetrics.get(it.chain)
|
||||||
metrics!!.nativeCallMetric.increment()
|
metrics!!.nativeCallMetric.increment()
|
||||||
startTime = System.currentTimeMillis()
|
startTime = System.currentTimeMillis()
|
||||||
}
|
}
|
||||||
).doOnNext {
|
).doOnNext {
|
||||||
metrics?.nativeCallRespMetric?.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS)
|
metrics?.nativeCallRespMetric?.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS)
|
||||||
}.doOnError { errorMetric.increment() }
|
}.doOnError { errorMetric.increment() }
|
||||||
@@ -77,11 +78,11 @@ class BlockchainRpc(
|
|||||||
override fun nativeSubscribe(request: Mono<BlockchainOuterClass.NativeSubscribeRequest>): Flux<BlockchainOuterClass.NativeSubscribeReplyItem> {
|
override fun nativeSubscribe(request: Mono<BlockchainOuterClass.NativeSubscribeRequest>): Flux<BlockchainOuterClass.NativeSubscribeReplyItem> {
|
||||||
var metrics: RequestMetrics? = null
|
var metrics: RequestMetrics? = null
|
||||||
return nativeSubscribe.nativeSubscribe(
|
return nativeSubscribe.nativeSubscribe(
|
||||||
request
|
request
|
||||||
.doOnNext {
|
.doOnNext {
|
||||||
metrics = chainMetrics.get(it.chain)
|
metrics = chainMetrics.get(it.chain)
|
||||||
metrics!!.nativeSubscribeMetric.increment()
|
metrics!!.nativeSubscribeMetric.increment()
|
||||||
}
|
}
|
||||||
).doOnNext {
|
).doOnNext {
|
||||||
metrics?.nativeSubscribeRespMetric?.increment()
|
metrics?.nativeSubscribeRespMetric?.increment()
|
||||||
}.doOnError { errorMetric.increment() }
|
}.doOnError { errorMetric.increment() }
|
||||||
@@ -89,8 +90,8 @@ class BlockchainRpc(
|
|||||||
|
|
||||||
override fun subscribeHead(request: Mono<Common.Chain>): Flux<BlockchainOuterClass.ChainHead> {
|
override fun subscribeHead(request: Mono<Common.Chain>): Flux<BlockchainOuterClass.ChainHead> {
|
||||||
return streamHead.add(
|
return streamHead.add(
|
||||||
request
|
request
|
||||||
.doOnNext { chainMetrics.get(it.type).subscribeHeadMetric.increment() }
|
.doOnNext { chainMetrics.get(it.type).subscribeHeadMetric.increment() }
|
||||||
).doOnError { errorMetric.increment() }
|
).doOnError { errorMetric.increment() }
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -102,8 +103,8 @@ class BlockchainRpc(
|
|||||||
try {
|
try {
|
||||||
trackTx.find { it.isSupported(chain) }?.let { track ->
|
trackTx.find { it.isSupported(chain) }?.let { track ->
|
||||||
track.subscribe(request)
|
track.subscribe(request)
|
||||||
.doOnNext { metrics.subscribeHeadRespMetric.increment() }
|
.doOnNext { metrics.subscribeHeadRespMetric.increment() }
|
||||||
.doOnError { errorMetric.increment() }
|
.doOnError { errorMetric.increment() }
|
||||||
} ?: Flux.error(SilentException.UnsupportedBlockchain(chain))
|
} ?: Flux.error(SilentException.UnsupportedBlockchain(chain))
|
||||||
} catch (t: Throwable) {
|
} catch (t: Throwable) {
|
||||||
log.error("Internal error during Tx Subscription", t)
|
log.error("Internal error during Tx Subscription", t)
|
||||||
@@ -122,12 +123,12 @@ class BlockchainRpc(
|
|||||||
try {
|
try {
|
||||||
trackAddress.find { it.isSupported(chain, asset) }?.let { track ->
|
trackAddress.find { it.isSupported(chain, asset) }?.let { track ->
|
||||||
track.subscribe(request)
|
track.subscribe(request)
|
||||||
.doOnNext { metrics.subscribeBalanceRespMetric.increment() }
|
.doOnNext { metrics.subscribeBalanceRespMetric.increment() }
|
||||||
.doOnError { errorMetric.increment() }
|
.doOnError { errorMetric.increment() }
|
||||||
} ?: Flux.error<BlockchainOuterClass.AddressBalance>(SilentException.UnsupportedBlockchain(chain))
|
} ?: Flux.error<BlockchainOuterClass.AddressBalance>(SilentException.UnsupportedBlockchain(chain))
|
||||||
.doOnSubscribe {
|
.doOnSubscribe {
|
||||||
log.error("Balance for $chain:$asset is not supported")
|
log.error("Balance for $chain:$asset is not supported")
|
||||||
}
|
}
|
||||||
} catch (t: Throwable) {
|
} catch (t: Throwable) {
|
||||||
log.error("Internal error during Balance Subscription", t)
|
log.error("Internal error during Balance Subscription", t)
|
||||||
errorMetric.increment()
|
errorMetric.increment()
|
||||||
@@ -146,13 +147,16 @@ class BlockchainRpc(
|
|||||||
try {
|
try {
|
||||||
trackAddress.find { it.isSupported(chain, asset) }?.let { track ->
|
trackAddress.find { it.isSupported(chain, asset) }?.let { track ->
|
||||||
track.getBalance(request)
|
track.getBalance(request)
|
||||||
.doOnNext {
|
.doOnNext {
|
||||||
metrics.getBalanceRespMetric.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS)
|
metrics.getBalanceRespMetric.record(
|
||||||
}
|
System.currentTimeMillis() - startTime,
|
||||||
|
TimeUnit.MILLISECONDS
|
||||||
|
)
|
||||||
|
}
|
||||||
} ?: Flux.error<BlockchainOuterClass.AddressBalance>(SilentException.UnsupportedBlockchain(chain))
|
} ?: Flux.error<BlockchainOuterClass.AddressBalance>(SilentException.UnsupportedBlockchain(chain))
|
||||||
.doOnSubscribe {
|
.doOnSubscribe {
|
||||||
log.error("Balance for $chain:$asset is not supported")
|
log.error("Balance for $chain:$asset is not supported")
|
||||||
}
|
}
|
||||||
} catch (t: Throwable) {
|
} catch (t: Throwable) {
|
||||||
log.error("Internal error during Balance Request", t)
|
log.error("Internal error during Balance Request", t)
|
||||||
errorMetric.increment()
|
errorMetric.increment()
|
||||||
@@ -164,61 +168,61 @@ class BlockchainRpc(
|
|||||||
override fun describe(request: Mono<BlockchainOuterClass.DescribeRequest>): Mono<BlockchainOuterClass.DescribeResponse> {
|
override fun describe(request: Mono<BlockchainOuterClass.DescribeRequest>): Mono<BlockchainOuterClass.DescribeResponse> {
|
||||||
describeMetric.increment()
|
describeMetric.increment()
|
||||||
return describe.describe(request)
|
return describe.describe(request)
|
||||||
.doOnError { errorMetric.increment() }
|
.doOnError { errorMetric.increment() }
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun subscribeStatus(request: Mono<BlockchainOuterClass.StatusRequest>): Flux<BlockchainOuterClass.ChainStatus> {
|
override fun subscribeStatus(request: Mono<BlockchainOuterClass.StatusRequest>): Flux<BlockchainOuterClass.ChainStatus> {
|
||||||
subscribeStatusMetric.increment()
|
subscribeStatusMetric.increment()
|
||||||
return subscribeStatus.subscribeStatus(request)
|
return subscribeStatus.subscribeStatus(request)
|
||||||
.doOnError { errorMetric.increment() }
|
.doOnError { errorMetric.increment() }
|
||||||
}
|
}
|
||||||
|
|
||||||
class RequestMetrics(chain: Chain) {
|
class RequestMetrics(chain: Chain) {
|
||||||
val nativeCallMetric = Counter.builder("request.grpc.request")
|
val nativeCallMetric = Counter.builder("request.grpc.request")
|
||||||
.tag("type", "nativeCall")
|
.tag("type", "nativeCall")
|
||||||
.tag("chain", chain.chainCode)
|
.tag("chain", chain.chainCode)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
val nativeCallRespMetric = Timer.builder("request.grpc.response")
|
val nativeCallRespMetric = Timer.builder("request.grpc.response")
|
||||||
.tag("type", "nativeCall")
|
.tag("type", "nativeCall")
|
||||||
.tag("chain", chain.chainCode)
|
.tag("chain", chain.chainCode)
|
||||||
.publishPercentileHistogram()
|
.publishPercentileHistogram()
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
val nativeSubscribeMetric = Counter.builder("request.grpc.request")
|
val nativeSubscribeMetric = Counter.builder("request.grpc.request")
|
||||||
.tag("type", "nativeSubscribe")
|
.tag("type", "nativeSubscribe")
|
||||||
.tag("chain", chain.chainCode)
|
.tag("chain", chain.chainCode)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
val nativeSubscribeRespMetric = Counter.builder("request.grpc.response")
|
val nativeSubscribeRespMetric = Counter.builder("request.grpc.response")
|
||||||
.tag("type", "nativeSubscribe")
|
.tag("type", "nativeSubscribe")
|
||||||
.tag("chain", chain.chainCode)
|
.tag("chain", chain.chainCode)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
val subscribeHeadMetric = Counter.builder("request.grpc.request")
|
val subscribeHeadMetric = Counter.builder("request.grpc.request")
|
||||||
.tag("type", "subscribeHead")
|
.tag("type", "subscribeHead")
|
||||||
.tag("chain", chain.chainCode)
|
.tag("chain", chain.chainCode)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
val subscribeHeadRespMetric = Counter.builder("request.grpc.reply")
|
val subscribeHeadRespMetric = Counter.builder("request.grpc.reply")
|
||||||
.tag("type", "subscribeHead")
|
.tag("type", "subscribeHead")
|
||||||
.tag("chain", chain.chainCode)
|
.tag("chain", chain.chainCode)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
val subscribeTxMetric = Counter.builder("request.grpc.request")
|
val subscribeTxMetric = Counter.builder("request.grpc.request")
|
||||||
.tag("type", "subscribeTx")
|
.tag("type", "subscribeTx")
|
||||||
.tag("chain", chain.chainCode)
|
.tag("chain", chain.chainCode)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
val subscribeBalanceMetric = Counter.builder("request.grpc.request")
|
val subscribeBalanceMetric = Counter.builder("request.grpc.request")
|
||||||
.tag("type", "subscribeBalance")
|
.tag("type", "subscribeBalance")
|
||||||
.tag("chain", chain.chainCode)
|
.tag("chain", chain.chainCode)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
val subscribeBalanceRespMetric = Counter.builder("request.grpc.reply")
|
val subscribeBalanceRespMetric = Counter.builder("request.grpc.reply")
|
||||||
.tag("type", "subscribeBalance")
|
.tag("type", "subscribeBalance")
|
||||||
.tag("chain", chain.chainCode)
|
.tag("chain", chain.chainCode)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
val getBalanceMetric = Counter.builder("request.grpc.request")
|
val getBalanceMetric = Counter.builder("request.grpc.request")
|
||||||
.tag("type", "getBalance")
|
.tag("type", "getBalance")
|
||||||
.tag("chain", chain.chainCode)
|
.tag("chain", chain.chainCode)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
val getBalanceRespMetric = Timer.builder("request.grpc.response")
|
val getBalanceRespMetric = Timer.builder("request.grpc.response")
|
||||||
.tag("type", "getBalance")
|
.tag("type", "getBalance")
|
||||||
.tag("chain", chain.chainCode)
|
.tag("chain", chain.chainCode)
|
||||||
.publishPercentileHistogram()
|
.publishPercentileHistogram()
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,15 +19,17 @@ package io.emeraldpay.dshackle.rpc
|
|||||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||||
import io.emeraldpay.api.proto.Common
|
import io.emeraldpay.api.proto.Common
|
||||||
import io.emeraldpay.dshackle.startup.QuorumForLabels
|
import io.emeraldpay.dshackle.startup.QuorumForLabels
|
||||||
import io.emeraldpay.dshackle.upstream.*
|
import io.emeraldpay.dshackle.upstream.Capability
|
||||||
|
import io.emeraldpay.dshackle.upstream.DefaultUpstream
|
||||||
|
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
import org.springframework.stereotype.Service
|
import org.springframework.stereotype.Service
|
||||||
import reactor.core.publisher.Mono
|
import reactor.core.publisher.Mono
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
class Describe(
|
class Describe(
|
||||||
@Autowired private val multistreamHolder: MultistreamHolder,
|
@Autowired private val multistreamHolder: MultistreamHolder,
|
||||||
@Autowired private val subscribeStatus: SubscribeStatus
|
@Autowired private val subscribeStatus: SubscribeStatus
|
||||||
) {
|
) {
|
||||||
|
|
||||||
fun describe(requestMono: Mono<BlockchainOuterClass.DescribeRequest>): Mono<BlockchainOuterClass.DescribeResponse> {
|
fun describe(requestMono: Mono<BlockchainOuterClass.DescribeRequest>): Mono<BlockchainOuterClass.DescribeResponse> {
|
||||||
@@ -39,9 +41,9 @@ class Describe(
|
|||||||
val targets = chainUpstreams.getMethods().getSupportedMethods()
|
val targets = chainUpstreams.getMethods().getSupportedMethods()
|
||||||
val capabilities: MutableSet<Capability> = mutableSetOf()
|
val capabilities: MutableSet<Capability> = mutableSetOf()
|
||||||
val chainDescription = BlockchainOuterClass.DescribeChain.newBuilder()
|
val chainDescription = BlockchainOuterClass.DescribeChain.newBuilder()
|
||||||
.setChain(Common.ChainRef.forNumber(chain.id))
|
.setChain(Common.ChainRef.forNumber(chain.id))
|
||||||
.addAllSupportedMethods(targets)
|
.addAllSupportedMethods(targets)
|
||||||
.setStatus(status)
|
.setStatus(status)
|
||||||
chainUpstreams.getAll().let { ups ->
|
chainUpstreams.getAll().let { ups ->
|
||||||
ups.forEach { up ->
|
ups.forEach { up ->
|
||||||
val nodes = QuorumForLabels()
|
val nodes = QuorumForLabels()
|
||||||
@@ -50,25 +52,27 @@ class Describe(
|
|||||||
}
|
}
|
||||||
nodes.getAll().forEach { node ->
|
nodes.getAll().forEach { node ->
|
||||||
val nodeDetails = BlockchainOuterClass.NodeDetails.newBuilder()
|
val nodeDetails = BlockchainOuterClass.NodeDetails.newBuilder()
|
||||||
.setQuorum(node.quorum)
|
.setQuorum(node.quorum)
|
||||||
.addAllLabels(node.labels.entries.map { label ->
|
.addAllLabels(
|
||||||
|
node.labels.entries.map { label ->
|
||||||
BlockchainOuterClass.Label.newBuilder()
|
BlockchainOuterClass.Label.newBuilder()
|
||||||
.setName(label.key)
|
.setName(label.key)
|
||||||
.setValue(label.value)
|
.setValue(label.value)
|
||||||
.build()
|
.build()
|
||||||
})
|
}
|
||||||
|
)
|
||||||
chainDescription.addNodes(nodeDetails)
|
chainDescription.addNodes(nodeDetails)
|
||||||
}
|
}
|
||||||
capabilities.addAll(up.getCapabilities())
|
capabilities.addAll(up.getCapabilities())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
chainDescription.addAllCapabilities(
|
chainDescription.addAllCapabilities(
|
||||||
capabilities.map {
|
capabilities.map {
|
||||||
when (it) {
|
when (it) {
|
||||||
Capability.RPC -> BlockchainOuterClass.Capabilities.CAP_CALLS
|
Capability.RPC -> BlockchainOuterClass.Capabilities.CAP_CALLS
|
||||||
Capability.BALANCE -> BlockchainOuterClass.Capabilities.CAP_BALANCE
|
Capability.BALANCE -> BlockchainOuterClass.Capabilities.CAP_BALANCE
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
)
|
)
|
||||||
resp.addChains(chainDescription.build())
|
resp.addChains(chainDescription.build())
|
||||||
}
|
}
|
||||||
@@ -76,5 +80,4 @@ class Describe(
|
|||||||
resp.build()
|
resp.build()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -17,12 +17,11 @@ class EthereumAddresses {
|
|||||||
Flux.just(Address.from(addresses.addressSingle.address))
|
Flux.just(Address.from(addresses.addressSingle.address))
|
||||||
Common.AnyAddress.AddrTypeCase.ADDRESS_MULTI ->
|
Common.AnyAddress.AddrTypeCase.ADDRESS_MULTI ->
|
||||||
Flux.fromIterable(addresses.addressMulti.addressesList)
|
Flux.fromIterable(addresses.addressMulti.addressesList)
|
||||||
.map { Address.from(it.address) }
|
.map { Address.from(it.address) }
|
||||||
else -> {
|
else -> {
|
||||||
log.error("Unsupported address type: ${addresses.addrTypeCase}")
|
log.error("Unsupported address type: ${addresses.addrTypeCase}")
|
||||||
Flux.empty()
|
Flux.empty()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -21,33 +21,35 @@ import com.google.protobuf.ByteString
|
|||||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||||
import io.emeraldpay.dshackle.Global
|
import io.emeraldpay.dshackle.Global
|
||||||
import io.emeraldpay.dshackle.SilentException
|
import io.emeraldpay.dshackle.SilentException
|
||||||
import io.emeraldpay.dshackle.upstream.*
|
|
||||||
import io.emeraldpay.dshackle.quorum.AlwaysQuorum
|
|
||||||
import io.emeraldpay.dshackle.quorum.CallQuorum
|
import io.emeraldpay.dshackle.quorum.CallQuorum
|
||||||
import io.emeraldpay.dshackle.quorum.NotLaggingQuorum
|
import io.emeraldpay.dshackle.quorum.NotLaggingQuorum
|
||||||
import io.emeraldpay.dshackle.quorum.QuorumReaderFactory
|
import io.emeraldpay.dshackle.quorum.QuorumReaderFactory
|
||||||
|
import io.emeraldpay.dshackle.upstream.ApiSource
|
||||||
|
import io.emeraldpay.dshackle.upstream.Multistream
|
||||||
|
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
||||||
|
import io.emeraldpay.dshackle.upstream.Selector
|
||||||
import io.emeraldpay.dshackle.upstream.calls.EthereumCallSelector
|
import io.emeraldpay.dshackle.upstream.calls.EthereumCallSelector
|
||||||
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
|
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
||||||
import io.emeraldpay.grpc.BlockchainType
|
|
||||||
import io.emeraldpay.grpc.Chain
|
|
||||||
import io.emeraldpay.etherjar.rpc.RpcException
|
import io.emeraldpay.etherjar.rpc.RpcException
|
||||||
import io.emeraldpay.etherjar.rpc.RpcResponseError
|
import io.emeraldpay.etherjar.rpc.RpcResponseError
|
||||||
|
import io.emeraldpay.grpc.BlockchainType
|
||||||
|
import io.emeraldpay.grpc.Chain
|
||||||
import org.apache.commons.lang3.StringUtils
|
import org.apache.commons.lang3.StringUtils
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
import org.springframework.stereotype.Service
|
import org.springframework.stereotype.Service
|
||||||
import reactor.core.publisher.*
|
import reactor.core.publisher.Flux
|
||||||
|
import reactor.core.publisher.Mono
|
||||||
import reactor.kotlin.core.publisher.toMono
|
import reactor.kotlin.core.publisher.toMono
|
||||||
import java.lang.Exception
|
import java.util.EnumMap
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
open class NativeCall(
|
open class NativeCall(
|
||||||
@Autowired private val multistreamHolder: MultistreamHolder
|
@Autowired private val multistreamHolder: MultistreamHolder
|
||||||
) {
|
) {
|
||||||
|
|
||||||
private val log = LoggerFactory.getLogger(NativeCall::class.java)
|
private val log = LoggerFactory.getLogger(NativeCall::class.java)
|
||||||
@@ -69,19 +71,19 @@ open class NativeCall(
|
|||||||
|
|
||||||
open fun nativeCall(requestMono: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<BlockchainOuterClass.NativeCallReplyItem> {
|
open fun nativeCall(requestMono: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<BlockchainOuterClass.NativeCallReplyItem> {
|
||||||
return nativeCallResult(requestMono)
|
return nativeCallResult(requestMono)
|
||||||
.map(this::buildResponse)
|
.map(this::buildResponse)
|
||||||
.onErrorResume(this::processException)
|
.onErrorResume(this::processException)
|
||||||
}
|
}
|
||||||
|
|
||||||
open fun nativeCallResult(requestMono: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<CallResult> {
|
open fun nativeCallResult(requestMono: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<CallResult> {
|
||||||
return requestMono.flatMapMany(this::prepareCall)
|
return requestMono.flatMapMany(this::prepareCall)
|
||||||
.map(this::parseParams)
|
.map(this::parseParams)
|
||||||
.parallel()
|
.parallel()
|
||||||
.flatMap {
|
.flatMap {
|
||||||
this.fetch(it)
|
this.fetch(it)
|
||||||
.doOnError { e -> log.warn("Error during native call: ${e.message}") }
|
.doOnError { e -> log.warn("Error during native call: ${e.message}") }
|
||||||
}
|
}
|
||||||
.sequential()
|
.sequential()
|
||||||
}
|
}
|
||||||
|
|
||||||
fun parseParams(it: CallContext<RawCallDetails>): CallContext<ParsedCallDetails> {
|
fun parseParams(it: CallContext<RawCallDetails>): CallContext<ParsedCallDetails> {
|
||||||
@@ -91,14 +93,14 @@ open class NativeCall(
|
|||||||
|
|
||||||
fun buildResponse(it: CallResult): BlockchainOuterClass.NativeCallReplyItem {
|
fun buildResponse(it: CallResult): BlockchainOuterClass.NativeCallReplyItem {
|
||||||
val result = BlockchainOuterClass.NativeCallReplyItem.newBuilder()
|
val result = BlockchainOuterClass.NativeCallReplyItem.newBuilder()
|
||||||
.setSucceed(!it.isError())
|
.setSucceed(!it.isError())
|
||||||
.setId(it.id)
|
.setId(it.id)
|
||||||
if (it.isError()) {
|
if (it.isError()) {
|
||||||
it.error?.let { error ->
|
it.error?.let { error ->
|
||||||
result.setErrorMessage(error.message)
|
result.setErrorMessage(error.message)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
result.setPayload(ByteString.copyFrom(it.result))
|
result.payload = ByteString.copyFrom(it.result)
|
||||||
}
|
}
|
||||||
|
|
||||||
return result.build()
|
return result.build()
|
||||||
@@ -112,11 +114,11 @@ open class NativeCall(
|
|||||||
0
|
0
|
||||||
}
|
}
|
||||||
return BlockchainOuterClass.NativeCallReplyItem.newBuilder()
|
return BlockchainOuterClass.NativeCallReplyItem.newBuilder()
|
||||||
.setSucceed(false)
|
.setSucceed(false)
|
||||||
.setErrorMessage(it?.message ?: "Internal error")
|
.setErrorMessage(it?.message ?: "Internal error")
|
||||||
.setId(id)
|
.setId(id)
|
||||||
.build()
|
.build()
|
||||||
.toMono()
|
.toMono()
|
||||||
}
|
}
|
||||||
|
|
||||||
fun prepareCall(request: BlockchainOuterClass.NativeCallRequest): Flux<CallContext<RawCallDetails>> {
|
fun prepareCall(request: BlockchainOuterClass.NativeCallRequest): Flux<CallContext<RawCallDetails>> {
|
||||||
@@ -130,31 +132,35 @@ open class NativeCall(
|
|||||||
}
|
}
|
||||||
|
|
||||||
val upstream = multistreamHolder.getUpstream(chain)
|
val upstream = multistreamHolder.getUpstream(chain)
|
||||||
?: return Flux.error(CallFailure(0, SilentException.UnsupportedBlockchain(chain)))
|
?: return Flux.error(CallFailure(0, SilentException.UnsupportedBlockchain(chain)))
|
||||||
|
|
||||||
return prepareCall(request, upstream)
|
return prepareCall(request, upstream)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun prepareCall(request: BlockchainOuterClass.NativeCallRequest, upstream: Multistream): Flux<CallContext<RawCallDetails>> {
|
fun prepareCall(
|
||||||
|
request: BlockchainOuterClass.NativeCallRequest,
|
||||||
|
upstream: Multistream
|
||||||
|
): Flux<CallContext<RawCallDetails>> {
|
||||||
val chain = Chain.byId(request.chainValue)
|
val chain = Chain.byId(request.chainValue)
|
||||||
return Flux.fromIterable(request.itemsList).flatMap {
|
return Flux.fromIterable(request.itemsList).flatMap {
|
||||||
val method = it.method
|
val method = it.method
|
||||||
val params = it.payload.toStringUtf8()
|
val params = it.payload.toStringUtf8()
|
||||||
|
|
||||||
// for ethereum the actual block needed for the call may be specified in the call parameters
|
// for ethereum the actual block needed for the call may be specified in the call parameters
|
||||||
val callSpecificMatcher: Mono<Selector.Matcher> = if (BlockchainType.from(upstream.chain) == BlockchainType.ETHEREUM) {
|
val callSpecificMatcher: Mono<Selector.Matcher> =
|
||||||
ethereumCallSelectors[chain]?.getMatcher(method, params, upstream.getHead())
|
if (BlockchainType.from(upstream.chain) == BlockchainType.ETHEREUM) {
|
||||||
} else {
|
ethereumCallSelectors[chain]?.getMatcher(method, params, upstream.getHead())
|
||||||
null
|
} else {
|
||||||
} ?: Mono.empty()
|
null
|
||||||
|
} ?: Mono.empty()
|
||||||
|
|
||||||
callSpecificMatcher.defaultIfEmpty(Selector.empty).map { csm ->
|
callSpecificMatcher.defaultIfEmpty(Selector.empty).map { csm ->
|
||||||
val matcher = Selector.Builder()
|
val matcher = Selector.Builder()
|
||||||
.withMatcher(csm)
|
.withMatcher(csm)
|
||||||
.forMethod(method)
|
.forMethod(method)
|
||||||
.forLabels(Selector.convertToMatcher(request.selector))
|
.forLabels(Selector.convertToMatcher(request.selector))
|
||||||
|
|
||||||
val callQuorum = upstream.getMethods().getQuorumFor(method) ?: AlwaysQuorum() // can be null in tests
|
val callQuorum = upstream.getMethods().getQuorumFor(method) // can be null in tests
|
||||||
callQuorum.init(upstream.getHead())
|
callQuorum.init(upstream.getHead())
|
||||||
|
|
||||||
// for NotLaggingQuorum it makes sense to select compatible upstreams before the call
|
// for NotLaggingQuorum it makes sense to select compatible upstreams before the call
|
||||||
@@ -172,22 +178,22 @@ open class NativeCall(
|
|||||||
|
|
||||||
fun fetch(ctx: CallContext<ParsedCallDetails>): Mono<CallResult> {
|
fun fetch(ctx: CallContext<ParsedCallDetails>): Mono<CallResult> {
|
||||||
return ctx.upstream.getRoutedApi(ctx.matcher)
|
return ctx.upstream.getRoutedApi(ctx.matcher)
|
||||||
.flatMap { api ->
|
.flatMap { api ->
|
||||||
api.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
|
api.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
|
||||||
.flatMap(JsonRpcResponse::requireResult)
|
.flatMap(JsonRpcResponse::requireResult)
|
||||||
.map {
|
.map {
|
||||||
CallResult.ok(ctx.id, it)
|
CallResult.ok(ctx.id, it)
|
||||||
}
|
|
||||||
}.switchIfEmpty(
|
|
||||||
Mono.just(ctx).flatMap(this::executeOnRemote)
|
|
||||||
)
|
|
||||||
.onErrorResume {
|
|
||||||
if (it is CallFailure) {
|
|
||||||
Mono.just(CallResult.fail(it.id, it.reason))
|
|
||||||
} else {
|
|
||||||
Mono.just(CallResult.fail(ctx.id, it))
|
|
||||||
}
|
}
|
||||||
|
}.switchIfEmpty(
|
||||||
|
Mono.just(ctx).flatMap(this::executeOnRemote)
|
||||||
|
)
|
||||||
|
.onErrorResume {
|
||||||
|
if (it is CallFailure) {
|
||||||
|
Mono.just(CallResult.fail(it.id, it.reason))
|
||||||
|
} else {
|
||||||
|
Mono.just(CallResult.fail(ctx.id, it))
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun executeOnRemote(ctx: CallContext<ParsedCallDetails>): Mono<CallResult> {
|
fun executeOnRemote(ctx: CallContext<ParsedCallDetails>): Mono<CallResult> {
|
||||||
@@ -196,21 +202,21 @@ open class NativeCall(
|
|||||||
}
|
}
|
||||||
val reader = quorumReaderFactory.create(ctx.getApis(), ctx.callQuorum)
|
val reader = quorumReaderFactory.create(ctx.getApis(), ctx.callQuorum)
|
||||||
return reader
|
return reader
|
||||||
.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
|
.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
|
||||||
.map {
|
.map {
|
||||||
CallResult(ctx.id, it.value, null)
|
CallResult(ctx.id, it.value, null)
|
||||||
|
}
|
||||||
|
.onErrorResume { t ->
|
||||||
|
val failure = if (t is CallFailure) {
|
||||||
|
CallResult.fail(t.id, t.reason)
|
||||||
|
} else {
|
||||||
|
CallResult.fail(ctx.id, t)
|
||||||
}
|
}
|
||||||
.onErrorResume { t ->
|
Mono.just(failure)
|
||||||
val failure = if (t is CallFailure) {
|
}
|
||||||
CallResult.fail(t.id, t.reason)
|
.switchIfEmpty(
|
||||||
} else {
|
Mono.just(CallResult.fail(ctx.id, 1, "No response or no available upstream for ${ctx.payload.method}"))
|
||||||
CallResult.fail(ctx.id, t)
|
)
|
||||||
}
|
|
||||||
Mono.just(failure)
|
|
||||||
}
|
|
||||||
.switchIfEmpty(
|
|
||||||
Mono.just(CallResult.fail(ctx.id, 1, "No response or no available upstream for ${ctx.payload.method}"))
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Suppress("UNCHECKED_CAST")
|
@Suppress("UNCHECKED_CAST")
|
||||||
@@ -222,11 +228,13 @@ open class NativeCall(
|
|||||||
return req as List<Any>
|
return req as List<Any>
|
||||||
}
|
}
|
||||||
|
|
||||||
open class CallContext<T>(val id: Int,
|
open class CallContext<T>(
|
||||||
val upstream: Multistream,
|
val id: Int,
|
||||||
val matcher: Selector.Matcher,
|
val upstream: Multistream,
|
||||||
val callQuorum: CallQuorum,
|
val matcher: Selector.Matcher,
|
||||||
val payload: T) {
|
val callQuorum: CallQuorum,
|
||||||
|
val payload: T
|
||||||
|
) {
|
||||||
fun <X> withPayload(payload: X): CallContext<X> {
|
fun <X> withPayload(payload: X): CallContext<X> {
|
||||||
return CallContext(id, upstream, matcher, callQuorum, payload)
|
return CallContext(id, upstream, matcher, callQuorum, payload)
|
||||||
}
|
}
|
||||||
@@ -273,4 +281,4 @@ open class NativeCall(
|
|||||||
|
|
||||||
class RawCallDetails(val method: String, val params: String)
|
class RawCallDetails(val method: String, val params: String)
|
||||||
class ParsedCallDetails(val method: String, val params: List<Any>)
|
class ParsedCallDetails(val method: String, val params: List<Any>)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ import reactor.core.publisher.Mono
|
|||||||
|
|
||||||
@Service
|
@Service
|
||||||
class NativeSubscribe(
|
class NativeSubscribe(
|
||||||
@Autowired private val multistreamHolder: MultistreamHolder
|
@Autowired private val multistreamHolder: MultistreamHolder
|
||||||
) {
|
) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -45,9 +45,9 @@ class NativeSubscribe(
|
|||||||
|
|
||||||
fun nativeSubscribe(request: Mono<BlockchainOuterClass.NativeSubscribeRequest>): Flux<BlockchainOuterClass.NativeSubscribeReplyItem> {
|
fun nativeSubscribe(request: Mono<BlockchainOuterClass.NativeSubscribeRequest>): Flux<BlockchainOuterClass.NativeSubscribeReplyItem> {
|
||||||
return request
|
return request
|
||||||
.flatMapMany(this@NativeSubscribe::start)
|
.flatMapMany(this@NativeSubscribe::start)
|
||||||
.map(this@NativeSubscribe::convertToProto)
|
.map(this@NativeSubscribe::convertToProto)
|
||||||
.onErrorMap(this@NativeSubscribe::convertToStatus)
|
.onErrorMap(this@NativeSubscribe::convertToStatus)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun start(it: BlockchainOuterClass.NativeSubscribeRequest): Publisher<out Any> {
|
fun start(it: BlockchainOuterClass.NativeSubscribeRequest): Publisher<out Any> {
|
||||||
@@ -68,15 +68,15 @@ class NativeSubscribe(
|
|||||||
|
|
||||||
fun convertToStatus(t: Throwable) = when (t) {
|
fun convertToStatus(t: Throwable) = when (t) {
|
||||||
is SilentException.UnsupportedBlockchain -> StatusException(
|
is SilentException.UnsupportedBlockchain -> StatusException(
|
||||||
Status.UNAVAILABLE.withDescription("BLOCKCHAIN UNAVAILABLE: ${t.blockchainId}")
|
Status.UNAVAILABLE.withDescription("BLOCKCHAIN UNAVAILABLE: ${t.blockchainId}")
|
||||||
)
|
)
|
||||||
is UnsupportedOperationException -> StatusException(
|
is UnsupportedOperationException -> StatusException(
|
||||||
Status.UNIMPLEMENTED.withDescription(t.message)
|
Status.UNIMPLEMENTED.withDescription(t.message)
|
||||||
)
|
)
|
||||||
else -> {
|
else -> {
|
||||||
log.warn("Unhandled error", t)
|
log.warn("Unhandled error", t)
|
||||||
StatusException(
|
StatusException(
|
||||||
Status.INTERNAL.withDescription(t.message)
|
Status.INTERNAL.withDescription(t.message)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -84,15 +84,14 @@ class NativeSubscribe(
|
|||||||
fun subscribe(chain: Chain, method: String, params: Any?): Flux<out Any> {
|
fun subscribe(chain: Chain, method: String, params: Any?): Flux<out Any> {
|
||||||
val up = multistreamHolder.getUpstream(chain) ?: return Flux.error(SilentException.UnsupportedBlockchain(chain))
|
val up = multistreamHolder.getUpstream(chain) ?: return Flux.error(SilentException.UnsupportedBlockchain(chain))
|
||||||
return (up as EthereumMultistream)
|
return (up as EthereumMultistream)
|
||||||
.getSubscribe()
|
.getSubscribe()
|
||||||
.subscribe(method, params)
|
.subscribe(method, params)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun convertToProto(value: Any): BlockchainOuterClass.NativeSubscribeReplyItem {
|
fun convertToProto(value: Any): BlockchainOuterClass.NativeSubscribeReplyItem {
|
||||||
val result = objectMapper.writeValueAsBytes(value)
|
val result = objectMapper.writeValueAsBytes(value)
|
||||||
return BlockchainOuterClass.NativeSubscribeReplyItem.newBuilder()
|
return BlockchainOuterClass.NativeSubscribeReplyItem.newBuilder()
|
||||||
.setPayload(ByteString.copyFrom(result))
|
.setPayload(ByteString.copyFrom(result))
|
||||||
.build()
|
.build()
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ import reactor.core.publisher.Mono
|
|||||||
|
|
||||||
@Service
|
@Service
|
||||||
class StreamHead(
|
class StreamHead(
|
||||||
@Autowired private val multistreamHolder: MultistreamHolder
|
@Autowired private val multistreamHolder: MultistreamHolder
|
||||||
) {
|
) {
|
||||||
|
|
||||||
private val log = LoggerFactory.getLogger(StreamHead::class.java)
|
private val log = LoggerFactory.getLogger(StreamHead::class.java)
|
||||||
@@ -40,24 +40,23 @@ class StreamHead(
|
|||||||
Chain.byId(request.type.number)
|
Chain.byId(request.type.number)
|
||||||
}.flatMapMany { chain ->
|
}.flatMapMany { chain ->
|
||||||
val up = multistreamHolder.getUpstream(chain)
|
val up = multistreamHolder.getUpstream(chain)
|
||||||
?: return@flatMapMany Flux.error<BlockchainOuterClass.ChainHead>(Exception("Unavailable chain: $chain"))
|
?: return@flatMapMany Flux.error<BlockchainOuterClass.ChainHead>(Exception("Unavailable chain: $chain"))
|
||||||
up.getHead()
|
up.getHead()
|
||||||
.getFlux()
|
.getFlux()
|
||||||
.map { asProto(chain, it!!) }
|
.map { asProto(chain, it!!) }
|
||||||
.onErrorContinue { t, _ ->
|
.onErrorContinue { t, _ ->
|
||||||
log.warn("Head subscription error: ${t.message}")
|
log.warn("Head subscription error: ${t.message}")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun asProto(chain: Chain, block: BlockContainer): BlockchainOuterClass.ChainHead {
|
fun asProto(chain: Chain, block: BlockContainer): BlockchainOuterClass.ChainHead {
|
||||||
return BlockchainOuterClass.ChainHead.newBuilder()
|
return BlockchainOuterClass.ChainHead.newBuilder()
|
||||||
.setChainValue(chain.id)
|
.setChainValue(chain.id)
|
||||||
.setHeight(block.height)
|
.setHeight(block.height)
|
||||||
.setTimestamp(block.timestamp.toEpochMilli())
|
.setTimestamp(block.timestamp.toEpochMilli())
|
||||||
.setWeight(ByteString.copyFrom(block.difficulty.toByteArray()))
|
.setWeight(ByteString.copyFrom(block.difficulty.toByteArray()))
|
||||||
.setBlockId(block.hash.toHex())
|
.setBlockId(block.hash.toHex())
|
||||||
.build()
|
.build()
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -18,7 +18,9 @@ package io.emeraldpay.dshackle.rpc
|
|||||||
|
|
||||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||||
import io.emeraldpay.api.proto.Common
|
import io.emeraldpay.api.proto.Common
|
||||||
import io.emeraldpay.dshackle.upstream.*
|
import io.emeraldpay.dshackle.upstream.Multistream
|
||||||
|
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
||||||
|
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
|
||||||
import io.emeraldpay.grpc.Chain
|
import io.emeraldpay.grpc.Chain
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
import org.springframework.stereotype.Service
|
import org.springframework.stereotype.Service
|
||||||
@@ -27,7 +29,7 @@ import reactor.core.publisher.Mono
|
|||||||
|
|
||||||
@Service
|
@Service
|
||||||
class SubscribeStatus(
|
class SubscribeStatus(
|
||||||
@Autowired private val multistreamHolder: MultistreamHolder
|
@Autowired private val multistreamHolder: MultistreamHolder
|
||||||
) {
|
) {
|
||||||
|
|
||||||
fun subscribeStatus(requestMono: Mono<BlockchainOuterClass.StatusRequest>): Flux<BlockchainOuterClass.ChainStatus> {
|
fun subscribeStatus(requestMono: Mono<BlockchainOuterClass.StatusRequest>): Flux<BlockchainOuterClass.ChainStatus> {
|
||||||
@@ -52,10 +54,10 @@ class SubscribeStatus(
|
|||||||
|
|
||||||
fun chainUnavailable(chain: Chain): BlockchainOuterClass.ChainStatus {
|
fun chainUnavailable(chain: Chain): BlockchainOuterClass.ChainStatus {
|
||||||
return BlockchainOuterClass.ChainStatus.newBuilder()
|
return BlockchainOuterClass.ChainStatus.newBuilder()
|
||||||
.setAvailability(BlockchainOuterClass.AvailabilityEnum.AVAIL_UNAVAILABLE)
|
.setAvailability(BlockchainOuterClass.AvailabilityEnum.AVAIL_UNAVAILABLE)
|
||||||
.setChain(Common.ChainRef.forNumber(chain.id))
|
.setChain(Common.ChainRef.forNumber(chain.id))
|
||||||
.setQuorum(0)
|
.setQuorum(0)
|
||||||
.build()
|
.build()
|
||||||
}
|
}
|
||||||
|
|
||||||
fun chainStatus(chain: Chain, available: UpstreamAvailability, ups: Multistream): BlockchainOuterClass.ChainStatus {
|
fun chainStatus(chain: Chain, available: UpstreamAvailability, ups: Multistream): BlockchainOuterClass.ChainStatus {
|
||||||
@@ -67,12 +69,11 @@ class SubscribeStatus(
|
|||||||
0
|
0
|
||||||
}
|
}
|
||||||
return BlockchainOuterClass.ChainStatus.newBuilder()
|
return BlockchainOuterClass.ChainStatus.newBuilder()
|
||||||
.setAvailability(BlockchainOuterClass.AvailabilityEnum.forNumber(available.grpcId))
|
.setAvailability(BlockchainOuterClass.AvailabilityEnum.forNumber(available.grpcId))
|
||||||
.setChain(Common.ChainRef.forNumber(chain.id))
|
.setChain(Common.ChainRef.forNumber(chain.id))
|
||||||
.setQuorum(quorum)
|
.setQuorum(quorum)
|
||||||
.build()
|
.build()
|
||||||
}
|
}
|
||||||
|
|
||||||
class ChainSubscription(val chain: Chain, val up: Multistream, val avail: UpstreamAvailability)
|
class ChainSubscription(val chain: Chain, val up: Multistream, val avail: UpstreamAvailability)
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ package io.emeraldpay.dshackle.rpc
|
|||||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||||
import io.emeraldpay.grpc.Chain
|
import io.emeraldpay.grpc.Chain
|
||||||
import reactor.core.publisher.Flux
|
import reactor.core.publisher.Flux
|
||||||
import reactor.core.publisher.Mono
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Base interface to tracking balance on a single blockchain
|
* Base interface to tracking balance on a single blockchain
|
||||||
@@ -28,5 +27,4 @@ interface TrackAddress {
|
|||||||
fun isSupported(chain: Chain, asset: String): Boolean
|
fun isSupported(chain: Chain, asset: String): Boolean
|
||||||
fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance>
|
fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance>
|
||||||
fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance>
|
fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance>
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ package io.emeraldpay.dshackle.rpc
|
|||||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||||
import io.emeraldpay.api.proto.Common
|
import io.emeraldpay.api.proto.Common
|
||||||
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
|
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
|
||||||
import io.emeraldpay.grpc.BlockchainType
|
|
||||||
import io.emeraldpay.dshackle.Defaults
|
import io.emeraldpay.dshackle.Defaults
|
||||||
import io.emeraldpay.dshackle.SilentException
|
import io.emeraldpay.dshackle.SilentException
|
||||||
import io.emeraldpay.dshackle.upstream.Capability
|
import io.emeraldpay.dshackle.upstream.Capability
|
||||||
@@ -27,6 +26,7 @@ import io.emeraldpay.dshackle.upstream.Selector
|
|||||||
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream
|
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream
|
||||||
import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent
|
import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent
|
||||||
import io.emeraldpay.dshackle.upstream.grpc.BitcoinGrpcUpstream
|
import io.emeraldpay.dshackle.upstream.grpc.BitcoinGrpcUpstream
|
||||||
|
import io.emeraldpay.grpc.BlockchainType
|
||||||
import io.emeraldpay.grpc.Chain
|
import io.emeraldpay.grpc.Chain
|
||||||
import org.apache.commons.lang3.StringUtils
|
import org.apache.commons.lang3.StringUtils
|
||||||
import org.bitcoinj.params.MainNetParams
|
import org.bitcoinj.params.MainNetParams
|
||||||
@@ -37,14 +37,12 @@ import org.springframework.stereotype.Service
|
|||||||
import reactor.core.publisher.Flux
|
import reactor.core.publisher.Flux
|
||||||
import reactor.core.publisher.Mono
|
import reactor.core.publisher.Mono
|
||||||
import java.math.BigInteger
|
import java.math.BigInteger
|
||||||
import java.time.Duration
|
|
||||||
import java.util.concurrent.ConcurrentHashMap
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
import javax.annotation.PostConstruct
|
import javax.annotation.PostConstruct
|
||||||
import kotlin.collections.HashMap
|
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
class TrackBitcoinAddress(
|
class TrackBitcoinAddress(
|
||||||
@Autowired private val multistreamHolder: MultistreamHolder
|
@Autowired private val multistreamHolder: MultistreamHolder
|
||||||
) : TrackAddress {
|
) : TrackAddress {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -52,8 +50,8 @@ class TrackBitcoinAddress(
|
|||||||
}
|
}
|
||||||
|
|
||||||
override fun isSupported(chain: Chain, asset: String): Boolean {
|
override fun isSupported(chain: Chain, asset: String): Boolean {
|
||||||
return (asset == "bitcoin" || asset == "btc" || asset == "satoshi")
|
return (asset == "bitcoin" || asset == "btc" || asset == "satoshi") &&
|
||||||
&& BlockchainType.from(chain) == BlockchainType.BITCOIN && multistreamHolder.isAvailable(chain)
|
BlockchainType.from(chain) == BlockchainType.BITCOIN && multistreamHolder.isAvailable(chain)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -65,8 +63,8 @@ class TrackBitcoinAddress(
|
|||||||
* Criteria for a remote grpc upstream that can provide a balance
|
* Criteria for a remote grpc upstream that can provide a balance
|
||||||
*/
|
*/
|
||||||
private val balanceUpstreamMatcher = Selector.LocalAndMatcher(
|
private val balanceUpstreamMatcher = Selector.LocalAndMatcher(
|
||||||
Selector.GrpcMatcher(),
|
Selector.GrpcMatcher(),
|
||||||
Selector.CapabilityMatcher(Capability.BALANCE)
|
Selector.CapabilityMatcher(Capability.BALANCE)
|
||||||
)
|
)
|
||||||
|
|
||||||
@PostConstruct
|
@PostConstruct
|
||||||
@@ -99,7 +97,7 @@ class TrackBitcoinAddress(
|
|||||||
return when {
|
return when {
|
||||||
request.address.hasAddressXpub() -> {
|
request.address.hasAddressXpub() -> {
|
||||||
val xpubAddresses = api.getXpubAddresses()
|
val xpubAddresses = api.getXpubAddresses()
|
||||||
?: return Flux.error(IllegalStateException("Xpub verification is not available"))
|
?: return Flux.error(IllegalStateException("Xpub verification is not available"))
|
||||||
|
|
||||||
val addressXpub = request.address.addressXpub
|
val addressXpub = request.address.addressXpub
|
||||||
if (StringUtils.isEmpty(addressXpub.xpub)) {
|
if (StringUtils.isEmpty(addressXpub.xpub)) {
|
||||||
@@ -109,45 +107,52 @@ class TrackBitcoinAddress(
|
|||||||
val start = Math.max(0, addressXpub.start).toInt()
|
val start = Math.max(0, addressXpub.start).toInt()
|
||||||
val limit = Math.min(100, Math.max(1, addressXpub.limit)).toInt()
|
val limit = Math.min(100, Math.max(1, addressXpub.limit)).toInt()
|
||||||
xpubAddresses.activeAddresses(xpub, start, limit)
|
xpubAddresses.activeAddresses(xpub, start, limit)
|
||||||
.map { it.toString() }
|
.map { it.toString() }
|
||||||
.doOnError { t -> log.error("Failed to process xpub. ${t.javaClass}:${t.message}") }
|
.doOnError { t -> log.error("Failed to process xpub. ${t.javaClass}:${t.message}") }
|
||||||
}
|
}
|
||||||
request.address.hasAddressSingle() -> {
|
request.address.hasAddressSingle() -> {
|
||||||
Flux.just(request.address.addressSingle.address)
|
Flux.just(request.address.addressSingle.address)
|
||||||
}
|
}
|
||||||
request.address.hasAddressMulti() -> {
|
request.address.hasAddressMulti() -> {
|
||||||
Flux.fromIterable(
|
Flux.fromIterable(
|
||||||
request.address.addressMulti.addressesList
|
request.address.addressMulti.addressesList
|
||||||
.map { addr -> addr.address }
|
.map { addr -> addr.address }
|
||||||
//TODO why sorted?
|
// TODO why sorted?
|
||||||
.sorted()
|
.sorted()
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
else -> Flux.error(IllegalArgumentException("Unsupported address type"))
|
else -> Flux.error(IllegalArgumentException("Unsupported address type"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun requestBalances(chain: Chain, api: BitcoinMultistream, addresses: Flux<String>, includeUtxo: Boolean): Flux<AddressBalance> {
|
fun requestBalances(
|
||||||
|
chain: Chain,
|
||||||
|
api: BitcoinMultistream,
|
||||||
|
addresses: Flux<String>,
|
||||||
|
includeUtxo: Boolean
|
||||||
|
): Flux<AddressBalance> {
|
||||||
return addresses
|
return addresses
|
||||||
.map { Address(chain, it) }
|
.map { Address(chain, it) }
|
||||||
.flatMap { address ->
|
.flatMap { address ->
|
||||||
balanceForAddress(api, address, includeUtxo)
|
balanceForAddress(api, address, includeUtxo)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun balanceForAddress(api: BitcoinMultistream, address: Address, includeUtxo: Boolean): Mono<AddressBalance> {
|
fun balanceForAddress(api: BitcoinMultistream, address: Address, includeUtxo: Boolean): Mono<AddressBalance> {
|
||||||
return api.getReader()
|
return api.getReader()
|
||||||
.listUnspent(address.bitcoinAddress)
|
.listUnspent(address.bitcoinAddress)
|
||||||
.map { unspent ->
|
.map { unspent ->
|
||||||
totalUnspent(address, includeUtxo, unspent)
|
totalUnspent(address, includeUtxo, unspent)
|
||||||
}
|
}
|
||||||
.switchIfEmpty(Mono.just(0).map {
|
.switchIfEmpty(
|
||||||
|
Mono.just(0).map {
|
||||||
AddressBalance(address, BigInteger.ZERO)
|
AddressBalance(address, BigInteger.ZERO)
|
||||||
})
|
|
||||||
.onErrorResume { t ->
|
|
||||||
log.error("Failed to get unspent", t)
|
|
||||||
Mono.empty()
|
|
||||||
}
|
}
|
||||||
|
)
|
||||||
|
.onErrorResume { t ->
|
||||||
|
log.error("Failed to get unspent", t)
|
||||||
|
Mono.empty()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun totalUnspent(address: Address, includeUtxo: Boolean, unspent: List<SimpleUnspent>): AddressBalance {
|
fun totalUnspent(address: Address, includeUtxo: Boolean, unspent: List<SimpleUnspent>): AddressBalance {
|
||||||
@@ -156,10 +161,10 @@ class TrackBitcoinAddress(
|
|||||||
} else {
|
} else {
|
||||||
unspent.map {
|
unspent.map {
|
||||||
AddressBalance(
|
AddressBalance(
|
||||||
address,
|
address,
|
||||||
BigInteger.valueOf(it.value),
|
BigInteger.valueOf(it.value),
|
||||||
if (includeUtxo) listOf(BalanceUtxo(it.txid, it.vout, it.value))
|
if (includeUtxo) listOf(BalanceUtxo(it.txid, it.vout, it.value))
|
||||||
else emptyList()
|
else emptyList()
|
||||||
)
|
)
|
||||||
}.reduce { a, b -> a.plus(b) }
|
}.reduce { a, b -> a.plus(b) }
|
||||||
}
|
}
|
||||||
@@ -169,26 +174,32 @@ class TrackBitcoinAddress(
|
|||||||
val ups = api.getApiSource(balanceUpstreamMatcher)
|
val ups = api.getApiSource(balanceUpstreamMatcher)
|
||||||
ups.request(1)
|
ups.request(1)
|
||||||
return Mono.from(ups)
|
return Mono.from(ups)
|
||||||
.map { up ->
|
.map { up ->
|
||||||
up.cast(BitcoinGrpcUpstream::class.java).remote
|
up.cast(BitcoinGrpcUpstream::class.java).remote
|
||||||
}
|
}
|
||||||
.timeout(Defaults.timeoutInternal, Mono.empty())
|
.timeout(Defaults.timeoutInternal, Mono.empty())
|
||||||
.switchIfEmpty(
|
.switchIfEmpty(
|
||||||
Mono.just(0)
|
Mono.just(0)
|
||||||
.doOnNext {
|
.doOnNext {
|
||||||
log.warn("No upstream providing balance for ${api.chain}")
|
log.warn("No upstream providing balance for ${api.chain}")
|
||||||
}
|
}
|
||||||
.then(Mono.error(SilentException.DataUnavailable("BALANCE")))
|
.then(Mono.error(SilentException.DataUnavailable("BALANCE")))
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun getRemoteBalance(api: BitcoinMultistream, request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
fun getRemoteBalance(
|
||||||
|
api: BitcoinMultistream,
|
||||||
|
request: BlockchainOuterClass.BalanceRequest
|
||||||
|
): Flux<BlockchainOuterClass.AddressBalance> {
|
||||||
return getBalanceGrpc(api).flatMapMany { remote ->
|
return getBalanceGrpc(api).flatMapMany { remote ->
|
||||||
remote.getBalance(request)
|
remote.getBalance(request)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun subscribeRemoteBalance(api: BitcoinMultistream, request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
fun subscribeRemoteBalance(
|
||||||
|
api: BitcoinMultistream,
|
||||||
|
request: BlockchainOuterClass.BalanceRequest
|
||||||
|
): Flux<BlockchainOuterClass.AddressBalance> {
|
||||||
return getBalanceGrpc(api).flatMapMany { remote ->
|
return getBalanceGrpc(api).flatMapMany { remote ->
|
||||||
remote.subscribeBalance(request)
|
remote.subscribeBalance(request)
|
||||||
}
|
}
|
||||||
@@ -197,43 +208,43 @@ class TrackBitcoinAddress(
|
|||||||
override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
||||||
val chain = Chain.byId(request.asset.chainValue)
|
val chain = Chain.byId(request.asset.chainValue)
|
||||||
val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java)
|
val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java)
|
||||||
?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue))
|
?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue))
|
||||||
return if (isBalanceAvailable(chain)) {
|
return if (isBalanceAvailable(chain)) {
|
||||||
val addresses = allAddresses(upstream, request)
|
val addresses = allAddresses(upstream, request)
|
||||||
requestBalances(chain, upstream, addresses, request.includeUtxo)
|
requestBalances(chain, upstream, addresses, request.includeUtxo)
|
||||||
.map(this@TrackBitcoinAddress::buildResponse)
|
.map(this@TrackBitcoinAddress::buildResponse)
|
||||||
.doOnError { t ->
|
.doOnError { t ->
|
||||||
log.error("Failed to get balance", t)
|
log.error("Failed to get balance", t)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
getRemoteBalance(upstream, request)
|
getRemoteBalance(upstream, request)
|
||||||
.doOnError { t ->
|
.doOnError { t ->
|
||||||
log.error("Failed to get balance from remote", t)
|
log.error("Failed to get balance from remote", t)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
||||||
val chain = Chain.byId(request.asset.chainValue)
|
val chain = Chain.byId(request.asset.chainValue)
|
||||||
val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java)
|
val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java)
|
||||||
?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue))
|
?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue))
|
||||||
if (isBalanceAvailable(chain)) {
|
if (isBalanceAvailable(chain)) {
|
||||||
val addresses = allAddresses(upstream, request).cache()
|
val addresses = allAddresses(upstream, request).cache()
|
||||||
val following = upstream.getHead().getFlux()
|
val following = upstream.getHead().getFlux()
|
||||||
.flatMap {
|
.flatMap {
|
||||||
requestBalances(chain, upstream, Flux.from(addresses), request.includeUtxo)
|
requestBalances(chain, upstream, Flux.from(addresses), request.includeUtxo)
|
||||||
}
|
}
|
||||||
val last = HashMap<String, BigInteger>()
|
val last = HashMap<String, BigInteger>()
|
||||||
val result = following
|
val result = following
|
||||||
.filter { curr ->
|
.filter { curr ->
|
||||||
val prev = last[curr.address.address]
|
val prev = last[curr.address.address]
|
||||||
//TODO utxo can change without changing balance
|
// TODO utxo can change without changing balance
|
||||||
val changed = prev == null || curr.balance != prev
|
val changed = prev == null || curr.balance != prev
|
||||||
if (changed) {
|
if (changed) {
|
||||||
last[curr.address.address] = curr.balance
|
last[curr.address.address] = curr.balance
|
||||||
}
|
|
||||||
changed
|
|
||||||
}
|
}
|
||||||
|
changed
|
||||||
|
}
|
||||||
|
|
||||||
return result.map(this@TrackBitcoinAddress::buildResponse)
|
return result.map(this@TrackBitcoinAddress::buildResponse)
|
||||||
} else {
|
} else {
|
||||||
@@ -243,24 +254,30 @@ class TrackBitcoinAddress(
|
|||||||
|
|
||||||
private fun buildResponse(address: AddressBalance): BlockchainOuterClass.AddressBalance {
|
private fun buildResponse(address: AddressBalance): BlockchainOuterClass.AddressBalance {
|
||||||
return BlockchainOuterClass.AddressBalance.newBuilder()
|
return BlockchainOuterClass.AddressBalance.newBuilder()
|
||||||
.setBalance(address.balance.toString(10))
|
.setBalance(address.balance.toString(10))
|
||||||
.setAsset(Common.Asset.newBuilder()
|
.setAsset(
|
||||||
.setChainValue(address.address.chain.id)
|
Common.Asset.newBuilder()
|
||||||
.setCode("BTC"))
|
.setChainValue(address.address.chain.id)
|
||||||
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.address))
|
.setCode("BTC")
|
||||||
.addAllUtxo(
|
)
|
||||||
address.utxo.map { utxo ->
|
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.address))
|
||||||
BlockchainOuterClass.Utxo.newBuilder()
|
.addAllUtxo(
|
||||||
.setBalance(utxo.value.toString())
|
address.utxo.map { utxo ->
|
||||||
.setIndex(utxo.vout.toLong())
|
BlockchainOuterClass.Utxo.newBuilder()
|
||||||
.setTxId(utxo.txid)
|
.setBalance(utxo.value.toString())
|
||||||
.build()
|
.setIndex(utxo.vout.toLong())
|
||||||
}
|
.setTxId(utxo.txid)
|
||||||
)
|
.build()
|
||||||
.build()
|
}
|
||||||
|
)
|
||||||
|
.build()
|
||||||
}
|
}
|
||||||
|
|
||||||
open class AddressBalance(val address: Address, var balance: BigInteger = BigInteger.ZERO, var utxo: List<BalanceUtxo> = emptyList()) {
|
open class AddressBalance(
|
||||||
|
val address: Address,
|
||||||
|
var balance: BigInteger = BigInteger.ZERO,
|
||||||
|
var utxo: List<BalanceUtxo> = emptyList()
|
||||||
|
) {
|
||||||
constructor(chain: Chain, address: String, balance: BigInteger) : this(Address(chain, address), balance)
|
constructor(chain: Chain, address: String, balance: BigInteger) : this(Address(chain, address), balance)
|
||||||
|
|
||||||
fun plus(other: AddressBalance) = AddressBalance(address, balance + other.balance, utxo.plus(other.utxo))
|
fun plus(other: AddressBalance) = AddressBalance(address, balance + other.balance, utxo.plus(other.utxo))
|
||||||
@@ -268,7 +285,7 @@ class TrackBitcoinAddress(
|
|||||||
|
|
||||||
open class BalanceUtxo(val txid: String, val vout: Int, val value: Long)
|
open class BalanceUtxo(val txid: String, val vout: Int, val value: Long)
|
||||||
|
|
||||||
//TODO use bitcoin class for address
|
// TODO use bitcoin class for address
|
||||||
class Address(val chain: Chain, val address: String) {
|
class Address(val chain: Chain, val address: String) {
|
||||||
val network = if (chain == Chain.BITCOIN) {
|
val network = if (chain == Chain.BITCOIN) {
|
||||||
MainNetParams()
|
MainNetParams()
|
||||||
@@ -276,7 +293,7 @@ class TrackBitcoinAddress(
|
|||||||
TestNet3Params()
|
TestNet3Params()
|
||||||
}
|
}
|
||||||
val bitcoinAddress = org.bitcoinj.core.Address.fromString(
|
val bitcoinAddress = org.bitcoinj.core.Address.fromString(
|
||||||
network, address
|
network, address
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,11 +18,11 @@ package io.emeraldpay.dshackle.rpc
|
|||||||
import com.google.protobuf.ByteString
|
import com.google.protobuf.ByteString
|
||||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||||
import io.emeraldpay.api.proto.Common
|
import io.emeraldpay.api.proto.Common
|
||||||
import io.emeraldpay.grpc.BlockchainType
|
|
||||||
import io.emeraldpay.dshackle.SilentException
|
import io.emeraldpay.dshackle.SilentException
|
||||||
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
||||||
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream
|
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream
|
||||||
import io.emeraldpay.dshackle.upstream.bitcoin.ExtractBlock
|
import io.emeraldpay.dshackle.upstream.bitcoin.ExtractBlock
|
||||||
|
import io.emeraldpay.grpc.BlockchainType
|
||||||
import io.emeraldpay.grpc.Chain
|
import io.emeraldpay.grpc.Chain
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
@@ -37,7 +37,7 @@ import kotlin.math.min
|
|||||||
|
|
||||||
@Service
|
@Service
|
||||||
class TrackBitcoinTx(
|
class TrackBitcoinTx(
|
||||||
@Autowired private val multistreamHolder: MultistreamHolder
|
@Autowired private val multistreamHolder: MultistreamHolder
|
||||||
) : TrackTx {
|
) : TrackTx {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -51,61 +51,69 @@ class TrackBitcoinTx(
|
|||||||
override fun subscribe(request: BlockchainOuterClass.TxStatusRequest): Flux<BlockchainOuterClass.TxStatus> {
|
override fun subscribe(request: BlockchainOuterClass.TxStatusRequest): Flux<BlockchainOuterClass.TxStatus> {
|
||||||
val chain = Chain.byId(request.chainValue)
|
val chain = Chain.byId(request.chainValue)
|
||||||
val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java)
|
val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java)
|
||||||
?: return Flux.error(SilentException.UnsupportedBlockchain(chain))
|
?: return Flux.error(SilentException.UnsupportedBlockchain(chain))
|
||||||
val txid = request.txId
|
val txid = request.txId
|
||||||
val confirmations = max(min(1, request.confirmationLimit), 12)
|
val confirmations = max(min(1, request.confirmationLimit), 12)
|
||||||
return subscribe(chain, upstream, txid)
|
return subscribe(chain, upstream, txid)
|
||||||
.takeUntil { tx ->
|
.takeUntil { tx ->
|
||||||
tx.confirmations >= confirmations
|
tx.confirmations >= confirmations
|
||||||
}.map(this::asProto)
|
}.map(this::asProto)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun subscribe(chain: Chain, upstream: BitcoinMultistream, txid: String): Flux<TxStatus> {
|
fun subscribe(chain: Chain, upstream: BitcoinMultistream, txid: String): Flux<TxStatus> {
|
||||||
return loadExisting(upstream, txid)
|
return loadExisting(upstream, txid)
|
||||||
.flatMapMany { status ->
|
.flatMapMany { status ->
|
||||||
if (status.mined) {
|
if (status.mined) {
|
||||||
//Head almost always knows the current height, so it can continue with calculating confirmations
|
// Head almost always knows the current height, so it can continue with calculating confirmations
|
||||||
//without publishing an empty TxStatus first
|
// without publishing an empty TxStatus first
|
||||||
continueWithMined(upstream, status)
|
continueWithMined(upstream, status)
|
||||||
} else {
|
} else {
|
||||||
loadMempool(upstream, txid)
|
loadMempool(upstream, txid)
|
||||||
.flatMapMany { tx ->
|
.flatMapMany { tx ->
|
||||||
val next = if (tx.found) {
|
val next = if (tx.found) {
|
||||||
untilMined(upstream, tx)
|
untilMined(upstream, tx)
|
||||||
} else {
|
} else {
|
||||||
untilFound(chain, upstream, txid)
|
untilFound(chain, upstream, txid)
|
||||||
}
|
}
|
||||||
//fist provide the current status, then updates
|
// fist provide the current status, then updates
|
||||||
Flux.concat(Mono.just(tx), next)
|
Flux.concat(Mono.just(tx), next)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun continueWithMined(upstream: BitcoinMultistream, status: TxStatus): Flux<TxStatus> {
|
fun continueWithMined(upstream: BitcoinMultistream, status: TxStatus): Flux<TxStatus> {
|
||||||
return upstream.getReader().getBlock(status.blockHash!!)
|
return upstream.getReader().getBlock(status.blockHash!!)
|
||||||
.map { block ->
|
.map { block ->
|
||||||
TxStatus(status.txid, true, ExtractBlock.getHeight(block), true, status.blockHash, ExtractBlock.getTime(block), ExtractBlock.getDifficulty(block))
|
TxStatus(
|
||||||
}.flatMapMany { tx ->
|
status.txid,
|
||||||
withConfirmations(upstream, tx)
|
true,
|
||||||
}
|
ExtractBlock.getHeight(block),
|
||||||
|
true,
|
||||||
|
status.blockHash,
|
||||||
|
ExtractBlock.getTime(block),
|
||||||
|
ExtractBlock.getDifficulty(block)
|
||||||
|
)
|
||||||
|
}.flatMapMany { tx ->
|
||||||
|
withConfirmations(upstream, tx)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun untilFound(chain: Chain, upstream: BitcoinMultistream, txid: String): Flux<TxStatus> {
|
fun untilFound(chain: Chain, upstream: BitcoinMultistream, txid: String): Flux<TxStatus> {
|
||||||
return Flux.interval(Duration.ofSeconds(1))
|
return Flux.interval(Duration.ofSeconds(1))
|
||||||
.take(Duration.ofMinutes(10))
|
.take(Duration.ofMinutes(10))
|
||||||
.flatMap { loadMempool(upstream, txid) }
|
.flatMap { loadMempool(upstream, txid) }
|
||||||
.skipUntil { it.found }
|
.skipUntil { it.found }
|
||||||
.flatMap { subscribe(chain, upstream, txid) }
|
.flatMap { subscribe(chain, upstream, txid) }
|
||||||
.doOnError { t ->
|
.doOnError { t ->
|
||||||
log.error("Failed to wait until found", t)
|
log.error("Failed to wait until found", t)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun untilMined(upstream: BitcoinMultistream, tx: TxStatus): Mono<TxStatus> {
|
fun untilMined(upstream: BitcoinMultistream, tx: TxStatus): Mono<TxStatus> {
|
||||||
return upstream.getHead().getFlux().flatMap {
|
return upstream.getHead().getFlux().flatMap {
|
||||||
loadExisting(upstream, tx.txid)
|
loadExisting(upstream, tx.txid)
|
||||||
.filter { it.mined }
|
.filter { it.mined }
|
||||||
}.single()
|
}.single()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -136,34 +144,36 @@ class TrackBitcoinTx(
|
|||||||
|
|
||||||
private fun asProto(tx: TxStatus): BlockchainOuterClass.TxStatus {
|
private fun asProto(tx: TxStatus): BlockchainOuterClass.TxStatus {
|
||||||
val data = BlockchainOuterClass.TxStatus.newBuilder()
|
val data = BlockchainOuterClass.TxStatus.newBuilder()
|
||||||
.setTxId(tx.txid)
|
.setTxId(tx.txid)
|
||||||
.setConfirmations(tx.confirmations.toInt())
|
.setConfirmations(tx.confirmations.toInt())
|
||||||
|
|
||||||
data.broadcasted = tx.found
|
data.broadcasted = tx.found
|
||||||
val isMined = tx.mined
|
val isMined = tx.mined
|
||||||
data.mined = isMined
|
data.mined = isMined
|
||||||
if (isMined) {
|
if (isMined) {
|
||||||
data.setBlock(
|
data.setBlock(
|
||||||
Common.BlockInfo.newBuilder()
|
Common.BlockInfo.newBuilder()
|
||||||
.setBlockId(tx.blockHash!!.substring(2))
|
.setBlockId(tx.blockHash!!.substring(2))
|
||||||
.setTimestamp(tx.blockTime!!.toEpochMilli())
|
.setTimestamp(tx.blockTime!!.toEpochMilli())
|
||||||
.setWeight(ByteString.copyFrom(tx.blockTotalDifficulty!!.toByteArray()))
|
.setWeight(ByteString.copyFrom(tx.blockTotalDifficulty!!.toByteArray()))
|
||||||
.setHeight(tx.height!!)
|
.setHeight(tx.height!!)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
return data.build()
|
return data.build()
|
||||||
}
|
}
|
||||||
|
|
||||||
class TxStatus(
|
class TxStatus(
|
||||||
val txid: String,
|
val txid: String,
|
||||||
val found: Boolean = false,
|
val found: Boolean = false,
|
||||||
val height: Long? = null,
|
val height: Long? = null,
|
||||||
val mined: Boolean = false,
|
val mined: Boolean = false,
|
||||||
val blockHash: String? = null,
|
val blockHash: String? = null,
|
||||||
val blockTime: Instant? = null,
|
val blockTime: Instant? = null,
|
||||||
val blockTotalDifficulty: BigInteger? = null,
|
val blockTotalDifficulty: BigInteger? = null,
|
||||||
val confirmations: Long = 0) {
|
val confirmations: Long = 0
|
||||||
|
) {
|
||||||
|
|
||||||
fun withHead(headHeight: Long) = TxStatus(txid, found, height, mined, blockHash, blockTime, blockTotalDifficulty, headHeight - height!! + 1)
|
fun withHead(headHeight: Long) =
|
||||||
|
TxStatus(txid, found, height, mined, blockHash, blockTime, blockTotalDifficulty, headHeight - height!! + 1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package io.emeraldpay.dshackle.rpc
|
|||||||
|
|
||||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||||
import io.emeraldpay.api.proto.Common
|
import io.emeraldpay.api.proto.Common
|
||||||
import io.emeraldpay.grpc.BlockchainType
|
|
||||||
import io.emeraldpay.dshackle.SilentException
|
import io.emeraldpay.dshackle.SilentException
|
||||||
import io.emeraldpay.dshackle.config.TokensConfig
|
import io.emeraldpay.dshackle.config.TokensConfig
|
||||||
import io.emeraldpay.dshackle.upstream.Head
|
import io.emeraldpay.dshackle.upstream.Head
|
||||||
@@ -11,25 +10,25 @@ import io.emeraldpay.dshackle.upstream.Selector
|
|||||||
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
|
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
||||||
import io.emeraldpay.grpc.Chain
|
|
||||||
import io.emeraldpay.etherjar.domain.Address
|
import io.emeraldpay.etherjar.domain.Address
|
||||||
import io.emeraldpay.etherjar.erc20.ERC20Token
|
import io.emeraldpay.etherjar.erc20.ERC20Token
|
||||||
import io.emeraldpay.etherjar.hex.Hex32
|
import io.emeraldpay.etherjar.hex.Hex32
|
||||||
import io.emeraldpay.etherjar.hex.HexQuantity
|
import io.emeraldpay.etherjar.hex.HexQuantity
|
||||||
|
import io.emeraldpay.grpc.BlockchainType
|
||||||
|
import io.emeraldpay.grpc.Chain
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
import org.springframework.stereotype.Service
|
import org.springframework.stereotype.Service
|
||||||
import reactor.core.publisher.Flux
|
import reactor.core.publisher.Flux
|
||||||
import reactor.core.publisher.Mono
|
import reactor.core.publisher.Mono
|
||||||
import java.math.BigInteger
|
import java.math.BigInteger
|
||||||
import java.util.*
|
import java.util.Locale
|
||||||
import javax.annotation.PostConstruct
|
import javax.annotation.PostConstruct
|
||||||
import kotlin.collections.HashMap
|
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
class TrackERC20Address(
|
class TrackERC20Address(
|
||||||
@Autowired private val multistreamHolder: MultistreamHolder,
|
@Autowired private val multistreamHolder: MultistreamHolder,
|
||||||
@Autowired private val tokensConfig: TokensConfig
|
@Autowired private val tokensConfig: TokensConfig
|
||||||
) : TrackAddress {
|
) : TrackAddress {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -46,8 +45,8 @@ class TrackERC20Address(
|
|||||||
val asset = token.name!!.lowercase(Locale.getDefault())
|
val asset = token.name!!.lowercase(Locale.getDefault())
|
||||||
val id = TokenId(chain, asset)
|
val id = TokenId(chain, asset)
|
||||||
val definition = TokenDefinition(
|
val definition = TokenDefinition(
|
||||||
chain, asset,
|
chain, asset,
|
||||||
ERC20Token(Address.from(token.address))
|
ERC20Token(Address.from(token.address))
|
||||||
)
|
)
|
||||||
tokens[id] = definition
|
tokens[id] = definition
|
||||||
log.info("Enable ERC20 balance for $chain:$asset")
|
log.info("Enable ERC20 balance for $chain:$asset")
|
||||||
@@ -56,7 +55,7 @@ class TrackERC20Address(
|
|||||||
|
|
||||||
override fun isSupported(chain: Chain, asset: String): Boolean {
|
override fun isSupported(chain: Chain, asset: String): Boolean {
|
||||||
return tokens.containsKey(TokenId(chain, asset.lowercase(Locale.getDefault()))) &&
|
return tokens.containsKey(TokenId(chain, asset.lowercase(Locale.getDefault()))) &&
|
||||||
BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain)
|
BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain)
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
||||||
@@ -64,9 +63,9 @@ class TrackERC20Address(
|
|||||||
val asset = request.asset.code.lowercase(Locale.getDefault())
|
val asset = request.asset.code.lowercase(Locale.getDefault())
|
||||||
val tokenDefinition = tokens[TokenId(chain, asset)] ?: return Flux.empty()
|
val tokenDefinition = tokens[TokenId(chain, asset)] ?: return Flux.empty()
|
||||||
return ethereumAddresses.extract(request.address)
|
return ethereumAddresses.extract(request.address)
|
||||||
.map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) }
|
.map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) }
|
||||||
.flatMap { addr -> getBalance(addr).map(addr::withBalance) }
|
.flatMap { addr -> getBalance(addr).map(addr::withBalance) }
|
||||||
.map { buildResponse(it) }
|
.map { buildResponse(it) }
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
||||||
@@ -76,63 +75,65 @@ class TrackERC20Address(
|
|||||||
val head = multistreamHolder.getUpstream(chain)?.getHead()?.getFlux() ?: Flux.empty()
|
val head = multistreamHolder.getUpstream(chain)?.getHead()?.getFlux() ?: Flux.empty()
|
||||||
|
|
||||||
return ethereumAddresses.extract(request.address)
|
return ethereumAddresses.extract(request.address)
|
||||||
.map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) }
|
.map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) }
|
||||||
.flatMap { addr ->
|
.flatMap { addr ->
|
||||||
val current = getBalance(addr)
|
val current = getBalance(addr)
|
||||||
val updates = head.flatMap { getBalance(addr) }
|
val updates = head.flatMap { getBalance(addr) }
|
||||||
Flux.concat(current, updates)
|
Flux.concat(current, updates)
|
||||||
.distinctUntilChanged()
|
.distinctUntilChanged()
|
||||||
.map { addr.withBalance(it) }
|
.map { addr.withBalance(it) }
|
||||||
}
|
}
|
||||||
.map { buildResponse(it) }
|
.map { buildResponse(it) }
|
||||||
}
|
}
|
||||||
|
|
||||||
fun getBalance(addr: TrackedAddress): Mono<BigInteger> {
|
fun getBalance(addr: TrackedAddress): Mono<BigInteger> {
|
||||||
val upstream = getUpstream(addr.chain)
|
val upstream = getUpstream(addr.chain)
|
||||||
return upstream
|
return upstream
|
||||||
.getDirectApi(Selector.empty)
|
.getDirectApi(Selector.empty)
|
||||||
.flatMap { api ->
|
.flatMap { api ->
|
||||||
api.read(prepareEthCall(addr.token, addr.address, upstream.getHead()))
|
api.read(prepareEthCall(addr.token, addr.address, upstream.getHead()))
|
||||||
.flatMap(JsonRpcResponse::requireStringResult)
|
.flatMap(JsonRpcResponse::requireStringResult)
|
||||||
.map {
|
.map {
|
||||||
Hex32.from(it).asQuantity().value
|
Hex32.from(it).asQuantity().value
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun prepareEthCall(token: ERC20Token, target: Address, head: Head): JsonRpcRequest {
|
fun prepareEthCall(token: ERC20Token, target: Address, head: Head): JsonRpcRequest {
|
||||||
val call = token
|
val call = token
|
||||||
.readBalanceOf(target)
|
.readBalanceOf(target)
|
||||||
.toJson()
|
.toJson()
|
||||||
val height = head.getCurrentHeight()?.let { HexQuantity.from(it).toHex() } ?: "latest"
|
val height = head.getCurrentHeight()?.let { HexQuantity.from(it).toHex() } ?: "latest"
|
||||||
return JsonRpcRequest("eth_call", listOf(call, height))
|
return JsonRpcRequest("eth_call", listOf(call, height))
|
||||||
}
|
}
|
||||||
|
|
||||||
fun getUpstream(chain: Chain): EthereumMultistream {
|
fun getUpstream(chain: Chain): EthereumMultistream {
|
||||||
return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java)
|
return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java)
|
||||||
?: throw SilentException.UnsupportedBlockchain(chain)
|
?: throw SilentException.UnsupportedBlockchain(chain)
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun buildResponse(address: TrackedAddress): BlockchainOuterClass.AddressBalance {
|
private fun buildResponse(address: TrackedAddress): BlockchainOuterClass.AddressBalance {
|
||||||
return BlockchainOuterClass.AddressBalance.newBuilder()
|
return BlockchainOuterClass.AddressBalance.newBuilder()
|
||||||
.setBalance(address.balance!!.toString(10))
|
.setBalance(address.balance!!.toString(10))
|
||||||
.setAsset(Common.Asset.newBuilder()
|
.setAsset(
|
||||||
.setChainValue(address.chain.id)
|
Common.Asset.newBuilder()
|
||||||
.setCode(address.tokenName.uppercase(Locale.getDefault())))
|
.setChainValue(address.chain.id)
|
||||||
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex()))
|
.setCode(address.tokenName.uppercase(Locale.getDefault()))
|
||||||
.build()
|
)
|
||||||
|
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex()))
|
||||||
|
.build()
|
||||||
}
|
}
|
||||||
|
|
||||||
class TrackedAddress(val chain: Chain,
|
class TrackedAddress(
|
||||||
val address: Address,
|
val chain: Chain,
|
||||||
val token: ERC20Token,
|
val address: Address,
|
||||||
val tokenName: String,
|
val token: ERC20Token,
|
||||||
val balance: BigInteger? = null
|
val tokenName: String,
|
||||||
|
val balance: BigInteger? = null
|
||||||
) {
|
) {
|
||||||
fun withBalance(balance: BigInteger) = TrackedAddress(chain, address, token, tokenName, balance)
|
fun withBalance(balance: BigInteger) = TrackedAddress(chain, address, token, tokenName, balance)
|
||||||
}
|
}
|
||||||
|
|
||||||
data class TokenId(val chain: Chain, val name: String)
|
data class TokenId(val chain: Chain, val name: String)
|
||||||
data class TokenDefinition(val chain: Chain, val name: String, val token: ERC20Token)
|
data class TokenDefinition(val chain: Chain, val name: String, val token: ERC20Token)
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -18,24 +18,24 @@ package io.emeraldpay.dshackle.rpc
|
|||||||
|
|
||||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||||
import io.emeraldpay.api.proto.Common
|
import io.emeraldpay.api.proto.Common
|
||||||
import io.emeraldpay.grpc.BlockchainType
|
|
||||||
import io.emeraldpay.dshackle.Defaults
|
import io.emeraldpay.dshackle.Defaults
|
||||||
import io.emeraldpay.dshackle.SilentException
|
import io.emeraldpay.dshackle.SilentException
|
||||||
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
||||||
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
|
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
|
||||||
import io.emeraldpay.grpc.Chain
|
|
||||||
import io.emeraldpay.etherjar.domain.Address
|
import io.emeraldpay.etherjar.domain.Address
|
||||||
import io.emeraldpay.etherjar.domain.Wei
|
import io.emeraldpay.etherjar.domain.Wei
|
||||||
|
import io.emeraldpay.grpc.BlockchainType
|
||||||
|
import io.emeraldpay.grpc.Chain
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
import org.springframework.stereotype.Service
|
import org.springframework.stereotype.Service
|
||||||
import reactor.core.publisher.Flux
|
import reactor.core.publisher.Flux
|
||||||
import reactor.core.publisher.Mono
|
import reactor.core.publisher.Mono
|
||||||
import java.util.*
|
import java.util.Locale
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
class TrackEthereumAddress(
|
class TrackEthereumAddress(
|
||||||
@Autowired private val multistreamHolder: MultistreamHolder
|
@Autowired private val multistreamHolder: MultistreamHolder
|
||||||
) : TrackAddress {
|
) : TrackAddress {
|
||||||
|
|
||||||
private val log = LoggerFactory.getLogger(TrackEthereumAddress::class.java)
|
private val log = LoggerFactory.getLogger(TrackEthereumAddress::class.java)
|
||||||
@@ -43,46 +43,46 @@ class TrackEthereumAddress(
|
|||||||
|
|
||||||
override fun isSupported(chain: Chain, asset: String): Boolean {
|
override fun isSupported(chain: Chain, asset: String): Boolean {
|
||||||
return asset == "ether" &&
|
return asset == "ether" &&
|
||||||
BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain)
|
BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain)
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
||||||
return initAddress(request)
|
return initAddress(request)
|
||||||
.flatMap { a -> getBalance(a).map { a.withBalance(it) } }
|
.flatMap { a -> getBalance(a).map { a.withBalance(it) } }
|
||||||
.map { buildResponse(it) }
|
.map { buildResponse(it) }
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
||||||
val chain = Chain.byId(request.asset.chainValue)
|
val chain = Chain.byId(request.asset.chainValue)
|
||||||
val head = multistreamHolder.getUpstream(chain)?.getHead()?.getFlux() ?: Flux.empty()
|
val head = multistreamHolder.getUpstream(chain)?.getHead()?.getFlux() ?: Flux.empty()
|
||||||
val balances = initAddress(request)
|
val balances = initAddress(request)
|
||||||
.flatMap { tracked ->
|
.flatMap { tracked ->
|
||||||
val current = getBalance(tracked)
|
val current = getBalance(tracked)
|
||||||
.map {
|
.map {
|
||||||
tracked.withBalance(it)
|
tracked.withBalance(it)
|
||||||
}
|
|
||||||
val updates = head
|
|
||||||
.flatMap {
|
|
||||||
getBalance(tracked)
|
|
||||||
}.map {
|
|
||||||
tracked.withBalance(it)
|
|
||||||
}
|
|
||||||
|
|
||||||
Flux.concat(current, updates)
|
|
||||||
.distinctUntilChanged {
|
|
||||||
it.balance ?: Wei.ZERO
|
|
||||||
}
|
|
||||||
}
|
|
||||||
.doOnError { t ->
|
|
||||||
if (t is SilentException) {
|
|
||||||
if (t is SilentException.UnsupportedBlockchain) {
|
|
||||||
log.warn("Unsupported blockchain: ${t.blockchainId}")
|
|
||||||
}
|
|
||||||
log.debug("Failed to process subscription", t)
|
|
||||||
} else {
|
|
||||||
log.warn("Failed to process subscription", t)
|
|
||||||
}
|
}
|
||||||
|
val updates = head
|
||||||
|
.flatMap {
|
||||||
|
getBalance(tracked)
|
||||||
|
}.map {
|
||||||
|
tracked.withBalance(it)
|
||||||
|
}
|
||||||
|
|
||||||
|
Flux.concat(current, updates)
|
||||||
|
.distinctUntilChanged {
|
||||||
|
it.balance ?: Wei.ZERO
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.doOnError { t ->
|
||||||
|
if (t is SilentException) {
|
||||||
|
if (t is SilentException.UnsupportedBlockchain) {
|
||||||
|
log.warn("Unsupported blockchain: ${t.blockchainId}")
|
||||||
|
}
|
||||||
|
log.debug("Failed to process subscription", t)
|
||||||
|
} else {
|
||||||
|
log.warn("Failed to process subscription", t)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return balances.map {
|
return balances.map {
|
||||||
buildResponse(it)
|
buildResponse(it)
|
||||||
@@ -91,7 +91,7 @@ class TrackEthereumAddress(
|
|||||||
|
|
||||||
fun getUpstream(chain: Chain): EthereumMultistream {
|
fun getUpstream(chain: Chain): EthereumMultistream {
|
||||||
return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java)
|
return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java)
|
||||||
?: throw SilentException.UnsupportedBlockchain(chain)
|
?: throw SilentException.UnsupportedBlockchain(chain)
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun initAddress(request: BlockchainOuterClass.BalanceRequest): Flux<TrackedAddress> {
|
private fun initAddress(request: BlockchainOuterClass.BalanceRequest): Flux<TrackedAddress> {
|
||||||
@@ -110,33 +110,36 @@ class TrackEthereumAddress(
|
|||||||
private fun createAddress(address: Common.SingleAddress, chain: Chain): TrackedAddress {
|
private fun createAddress(address: Common.SingleAddress, chain: Chain): TrackedAddress {
|
||||||
val addressParsed = Address.from(address.address)
|
val addressParsed = Address.from(address.address)
|
||||||
return TrackedAddress(
|
return TrackedAddress(
|
||||||
chain,
|
chain,
|
||||||
addressParsed
|
addressParsed
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun getBalance(addr: TrackedAddress): Mono<Wei> {
|
fun getBalance(addr: TrackedAddress): Mono<Wei> {
|
||||||
return getUpstream(addr.chain)
|
return getUpstream(addr.chain)
|
||||||
.getReader()
|
.getReader()
|
||||||
.balance()
|
.balance()
|
||||||
.read(addr.address)
|
.read(addr.address)
|
||||||
.timeout(Defaults.timeout)
|
.timeout(Defaults.timeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun buildResponse(address: TrackedAddress): BlockchainOuterClass.AddressBalance {
|
private fun buildResponse(address: TrackedAddress): BlockchainOuterClass.AddressBalance {
|
||||||
return BlockchainOuterClass.AddressBalance.newBuilder()
|
return BlockchainOuterClass.AddressBalance.newBuilder()
|
||||||
.setBalance(address.balance!!.amount!!.toString(10))
|
.setBalance(address.balance!!.amount!!.toString(10))
|
||||||
.setAsset(Common.Asset.newBuilder()
|
.setAsset(
|
||||||
.setChainValue(address.chain.id)
|
Common.Asset.newBuilder()
|
||||||
.setCode("ETHER"))
|
.setChainValue(address.chain.id)
|
||||||
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex()))
|
.setCode("ETHER")
|
||||||
.build()
|
)
|
||||||
|
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex()))
|
||||||
|
.build()
|
||||||
}
|
}
|
||||||
|
|
||||||
class TrackedAddress(val chain: Chain,
|
class TrackedAddress(
|
||||||
val address: Address,
|
val chain: Chain,
|
||||||
val balance: Wei? = null
|
val address: Address,
|
||||||
|
val balance: Wei? = null
|
||||||
) {
|
) {
|
||||||
fun withBalance(balance: Wei) = TrackedAddress(chain, address, balance)
|
fun withBalance(balance: Wei) = TrackedAddress(chain, address, balance)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,19 +19,19 @@ package io.emeraldpay.dshackle.rpc
|
|||||||
import com.google.protobuf.ByteString
|
import com.google.protobuf.ByteString
|
||||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||||
import io.emeraldpay.api.proto.Common
|
import io.emeraldpay.api.proto.Common
|
||||||
import io.emeraldpay.grpc.BlockchainType
|
|
||||||
import io.emeraldpay.dshackle.SilentException
|
import io.emeraldpay.dshackle.SilentException
|
||||||
import io.emeraldpay.dshackle.data.BlockContainer
|
import io.emeraldpay.dshackle.data.BlockContainer
|
||||||
import io.emeraldpay.dshackle.data.TxId
|
import io.emeraldpay.dshackle.data.TxId
|
||||||
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
||||||
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
|
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
|
||||||
import io.emeraldpay.grpc.Chain
|
|
||||||
import io.emeraldpay.etherjar.domain.BlockHash
|
import io.emeraldpay.etherjar.domain.BlockHash
|
||||||
import io.emeraldpay.etherjar.domain.TransactionId
|
import io.emeraldpay.etherjar.domain.TransactionId
|
||||||
import io.emeraldpay.etherjar.rpc.RpcException
|
import io.emeraldpay.etherjar.rpc.RpcException
|
||||||
import io.emeraldpay.etherjar.rpc.json.BlockJson
|
import io.emeraldpay.etherjar.rpc.json.BlockJson
|
||||||
import io.emeraldpay.etherjar.rpc.json.TransactionJson
|
import io.emeraldpay.etherjar.rpc.json.TransactionJson
|
||||||
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
|
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
|
||||||
|
import io.emeraldpay.grpc.BlockchainType
|
||||||
|
import io.emeraldpay.grpc.Chain
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
import org.springframework.stereotype.Service
|
import org.springframework.stereotype.Service
|
||||||
@@ -48,7 +48,7 @@ import kotlin.math.min
|
|||||||
|
|
||||||
@Service
|
@Service
|
||||||
class TrackEthereumTx(
|
class TrackEthereumTx(
|
||||||
@Autowired private val multistreamHolder: MultistreamHolder
|
@Autowired private val multistreamHolder: MultistreamHolder
|
||||||
) : TrackTx {
|
) : TrackTx {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -70,63 +70,62 @@ class TrackEthereumTx(
|
|||||||
val base = prepareTracking(request)
|
val base = prepareTracking(request)
|
||||||
val up = getUpstream(base.chain)
|
val up = getUpstream(base.chain)
|
||||||
return update(base)
|
return update(base)
|
||||||
.defaultIfEmpty(base)
|
.defaultIfEmpty(base)
|
||||||
.flatMapMany {
|
.flatMapMany {
|
||||||
Flux.concat(Mono.just(it), subscribe(it, up))
|
Flux.concat(Mono.just(it), subscribe(it, up))
|
||||||
.distinctUntilChanged(TxDetails::status)
|
.distinctUntilChanged(TxDetails::status)
|
||||||
.map(this@TrackEthereumTx::asProto)
|
.map(this@TrackEthereumTx::asProto)
|
||||||
.subscribeOn(scheduler)
|
.subscribeOn(scheduler)
|
||||||
}
|
}
|
||||||
.doOnError { t ->
|
.doOnError { t ->
|
||||||
log.error("Subscription error", t)
|
log.error("Subscription error", t)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
fun getUpstream(chain: Chain): EthereumMultistream {
|
fun getUpstream(chain: Chain): EthereumMultistream {
|
||||||
return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java)
|
return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java)
|
||||||
?: throw SilentException.UnsupportedBlockchain(chain)
|
?: throw SilentException.UnsupportedBlockchain(chain)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun subscribe(base: TxDetails, up: EthereumMultistream): Flux<TxDetails> {
|
fun subscribe(base: TxDetails, up: EthereumMultistream): Flux<TxDetails> {
|
||||||
var latestTx = base
|
var latestTx = base
|
||||||
|
|
||||||
val untilFound = Mono.just(latestTx)
|
val untilFound = Mono.just(latestTx)
|
||||||
.subscribeOn(scheduler)
|
.subscribeOn(scheduler)
|
||||||
.map {
|
.map {
|
||||||
//replace with the latest value, it may be already found
|
// replace with the latest value, it may be already found
|
||||||
latestTx
|
latestTx
|
||||||
|
}
|
||||||
|
.flatMap { latest ->
|
||||||
|
if (!latest.status.found) {
|
||||||
|
update(latest).defaultIfEmpty(latestTx)
|
||||||
|
} else {
|
||||||
|
Mono.just(latest)
|
||||||
}
|
}
|
||||||
.flatMap { latest ->
|
}
|
||||||
if (!latest.status.found) {
|
.flatMap { received ->
|
||||||
update(latest).defaultIfEmpty(latestTx)
|
if (!received.status.found) {
|
||||||
} else {
|
Mono.error(SilentException("Retry not found"))
|
||||||
Mono.just(latest)
|
} else {
|
||||||
}
|
Mono.just(received)
|
||||||
}
|
}
|
||||||
.flatMap { received ->
|
}
|
||||||
if (!received.status.found) {
|
.retryWhen(
|
||||||
Mono.error(SilentException("Retry not found"))
|
Retry.fixedDelay(10, Duration.ofSeconds(2))
|
||||||
} else {
|
)
|
||||||
Mono.just(received)
|
.onErrorResume { Mono.empty() }
|
||||||
}
|
|
||||||
}
|
|
||||||
.retryWhen(
|
|
||||||
Retry.fixedDelay(10, Duration.ofSeconds(2))
|
|
||||||
)
|
|
||||||
.onErrorResume { Mono.empty() }
|
|
||||||
|
|
||||||
val inBlocks = up.getHead().getFlux()
|
val inBlocks = up.getHead().getFlux()
|
||||||
.subscribeOn(scheduler)
|
.subscribeOn(scheduler)
|
||||||
.flatMap { block ->
|
.flatMap { block ->
|
||||||
onNewBlock(latestTx, block)
|
onNewBlock(latestTx, block)
|
||||||
}
|
}
|
||||||
|
|
||||||
return Flux.merge(untilFound, inBlocks)
|
return Flux.merge(untilFound, inBlocks)
|
||||||
.takeUntil(TxDetails::shouldClose)
|
.takeUntil(TxDetails::shouldClose)
|
||||||
.doOnNext { newTx ->
|
.doOnNext { newTx ->
|
||||||
latestTx = newTx
|
latestTx = newTx
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun onNewBlock(tx: TxDetails, block: BlockContainer): Mono<TxDetails> {
|
fun onNewBlock(tx: TxDetails, block: BlockContainer): Mono<TxDetails> {
|
||||||
@@ -134,7 +133,8 @@ class TrackEthereumTx(
|
|||||||
if (!tx.status.mined) {
|
if (!tx.status.mined) {
|
||||||
val justMined = block.transactions.contains(txid)
|
val justMined = block.transactions.contains(txid)
|
||||||
return if (justMined) {
|
return if (justMined) {
|
||||||
Mono.just(tx.withStatus(
|
Mono.just(
|
||||||
|
tx.withStatus(
|
||||||
mined = true,
|
mined = true,
|
||||||
found = true,
|
found = true,
|
||||||
confirmations = 1,
|
confirmations = 1,
|
||||||
@@ -142,13 +142,14 @@ class TrackEthereumTx(
|
|||||||
blockTime = block.timestamp,
|
blockTime = block.timestamp,
|
||||||
blockTotalDifficulty = block.difficulty,
|
blockTotalDifficulty = block.difficulty,
|
||||||
blockHash = BlockHash(block.hash.value)
|
blockHash = BlockHash(block.hash.value)
|
||||||
))
|
)
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
update(tx)
|
update(tx)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
//verify if it's still on chain
|
// verify if it's still on chain
|
||||||
//TODO head is supposed to erase block when it was replaced, so can safely recalc here
|
// TODO head is supposed to erase block when it was replaced, so can safely recalc here
|
||||||
return update(tx)
|
return update(tx)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -157,19 +158,19 @@ class TrackEthereumTx(
|
|||||||
val initialStatus = tx.status
|
val initialStatus = tx.status
|
||||||
val upstream = getUpstream(tx.chain)
|
val upstream = getUpstream(tx.chain)
|
||||||
return upstream.getReader()
|
return upstream.getReader()
|
||||||
.txByHash().read(tx.txid)
|
.txByHash().read(tx.txid)
|
||||||
.onErrorResume(RpcException::class.java) { t ->
|
.onErrorResume(RpcException::class.java) { t ->
|
||||||
log.warn("Upstream error, ignoring. {}", t.rpcMessage)
|
log.warn("Upstream error, ignoring. {}", t.rpcMessage)
|
||||||
Mono.empty<TransactionJson>()
|
Mono.empty<TransactionJson>()
|
||||||
}
|
}
|
||||||
.flatMap { updateFromBlock(upstream, tx, it) }
|
.flatMap { updateFromBlock(upstream, tx, it) }
|
||||||
.doOnError { t ->
|
.doOnError { t ->
|
||||||
log.error("Failed to load tx block", t)
|
log.error("Failed to load tx block", t)
|
||||||
}
|
}
|
||||||
.switchIfEmpty(Mono.just(tx.withStatus(found = false)))
|
.switchIfEmpty(Mono.just(tx.withStatus(found = false)))
|
||||||
.filter { current ->
|
.filter { current ->
|
||||||
initialStatus != current.status || current.shouldClose()
|
initialStatus != current.status || current.shouldClose()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun prepareTracking(request: BlockchainOuterClass.TxStatusRequest): TxDetails {
|
fun prepareTracking(request: BlockchainOuterClass.TxStatusRequest): TxDetails {
|
||||||
@@ -178,10 +179,10 @@ class TrackEthereumTx(
|
|||||||
throw SilentException.UnsupportedBlockchain(request.chainValue)
|
throw SilentException.UnsupportedBlockchain(request.chainValue)
|
||||||
}
|
}
|
||||||
val details = TxDetails(
|
val details = TxDetails(
|
||||||
chain,
|
chain,
|
||||||
Instant.now(),
|
Instant.now(),
|
||||||
TransactionId.from(request.txId),
|
TransactionId.from(request.txId),
|
||||||
min(max(1, request.confirmationLimit), 100)
|
min(max(1, request.confirmationLimit), 100)
|
||||||
)
|
)
|
||||||
return details
|
return details
|
||||||
}
|
}
|
||||||
@@ -189,12 +190,12 @@ class TrackEthereumTx(
|
|||||||
fun setBlockDetails(tx: TxDetails, block: BlockJson<TransactionRefJson>): TxDetails {
|
fun setBlockDetails(tx: TxDetails, block: BlockJson<TransactionRefJson>): TxDetails {
|
||||||
return if (block.number != null && block.totalDifficulty != null) {
|
return if (block.number != null && block.totalDifficulty != null) {
|
||||||
tx.withStatus(
|
tx.withStatus(
|
||||||
blockTotalDifficulty = block.totalDifficulty,
|
blockTotalDifficulty = block.totalDifficulty,
|
||||||
blockTime = block.timestamp
|
blockTime = block.timestamp
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
tx.withStatus(
|
tx.withStatus(
|
||||||
mined = false
|
mined = false
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -205,22 +206,22 @@ class TrackEthereumTx(
|
|||||||
return Mono.empty()
|
return Mono.empty()
|
||||||
}
|
}
|
||||||
return upstream.getReader()
|
return upstream.getReader()
|
||||||
.blocksByHashParsed().read(tx.status.blockHash)
|
.blocksByHashParsed().read(tx.status.blockHash)
|
||||||
.map { block ->
|
.map { block ->
|
||||||
setBlockDetails(tx, block)
|
setBlockDetails(tx, block)
|
||||||
}.doOnError { t ->
|
}.doOnError { t ->
|
||||||
log.warn("Failed to update weight", t)
|
log.warn("Failed to update weight", t)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun updateFromBlock(upstream: EthereumMultistream, tx: TxDetails, blockTx: TransactionJson): Mono<TxDetails> {
|
fun updateFromBlock(upstream: EthereumMultistream, tx: TxDetails, blockTx: TransactionJson): Mono<TxDetails> {
|
||||||
return if (blockTx.blockNumber != null && blockTx.blockHash != null && blockTx.blockHash != ZERO_BLOCK) {
|
return if (blockTx.blockNumber != null && blockTx.blockHash != null && blockTx.blockHash != ZERO_BLOCK) {
|
||||||
val updated = tx.withStatus(
|
val updated = tx.withStatus(
|
||||||
blockHash = blockTx.blockHash,
|
blockHash = blockTx.blockHash,
|
||||||
height = blockTx.blockNumber,
|
height = blockTx.blockNumber,
|
||||||
found = true,
|
found = true,
|
||||||
mined = true,
|
mined = true,
|
||||||
confirmations = 1
|
confirmations = 1
|
||||||
)
|
)
|
||||||
upstream.getHead().getFlux().next().map { head ->
|
upstream.getHead().getFlux().next().map { head ->
|
||||||
val height = updated.status.height
|
val height = updated.status.height
|
||||||
@@ -228,72 +229,89 @@ class TrackEthereumTx(
|
|||||||
updated
|
updated
|
||||||
} else {
|
} else {
|
||||||
updated.withStatus(
|
updated.withStatus(
|
||||||
confirmations = head.height - height + 1
|
confirmations = head.height - height + 1
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}.doOnError { t ->
|
}.doOnError { t ->
|
||||||
log.error("Unable to load head details", t)
|
log.error("Unable to load head details", t)
|
||||||
}.flatMap(this::loadWeight)
|
}.flatMap(this::loadWeight)
|
||||||
} else {
|
} else {
|
||||||
Mono.just(tx.withStatus(
|
Mono.just(
|
||||||
|
tx.withStatus(
|
||||||
found = true,
|
found = true,
|
||||||
mined = false
|
mined = false
|
||||||
))
|
)
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun asProto(tx: TxDetails): BlockchainOuterClass.TxStatus {
|
private fun asProto(tx: TxDetails): BlockchainOuterClass.TxStatus {
|
||||||
val data = BlockchainOuterClass.TxStatus.newBuilder()
|
val data = BlockchainOuterClass.TxStatus.newBuilder()
|
||||||
.setTxId(tx.txid.toHex())
|
.setTxId(tx.txid.toHex())
|
||||||
.setConfirmations(tx.status.confirmations.toInt())
|
.setConfirmations(tx.status.confirmations.toInt())
|
||||||
|
|
||||||
data.broadcasted = tx.status.found
|
data.broadcasted = tx.status.found
|
||||||
val isMined = tx.status.mined
|
val isMined = tx.status.mined
|
||||||
data.mined = isMined
|
data.mined = isMined
|
||||||
if (isMined) {
|
if (isMined) {
|
||||||
data.setBlock(
|
data.setBlock(
|
||||||
Common.BlockInfo.newBuilder()
|
Common.BlockInfo.newBuilder()
|
||||||
.setBlockId(tx.status.blockHash!!.toHex().substring(2))
|
.setBlockId(tx.status.blockHash!!.toHex().substring(2))
|
||||||
.setTimestamp(tx.status.blockTime!!.toEpochMilli())
|
.setTimestamp(tx.status.blockTime!!.toEpochMilli())
|
||||||
.setWeight(ByteString.copyFrom(tx.status.blockTotalDifficulty!!.toByteArray()))
|
.setWeight(ByteString.copyFrom(tx.status.blockTotalDifficulty!!.toByteArray()))
|
||||||
.setHeight(tx.status.height!!)
|
.setHeight(tx.status.height!!)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
return data.build()
|
return data.build()
|
||||||
}
|
}
|
||||||
|
|
||||||
class TxDetails(val chain: Chain,
|
class TxDetails(
|
||||||
val since: Instant,
|
val chain: Chain,
|
||||||
val txid: TransactionId,
|
val since: Instant,
|
||||||
val maxConfirmations: Int,
|
val txid: TransactionId,
|
||||||
val status: TxStatus
|
val maxConfirmations: Int,
|
||||||
|
val status: TxStatus
|
||||||
) {
|
) {
|
||||||
|
|
||||||
constructor(chain: Chain,
|
constructor(
|
||||||
since: Instant,
|
chain: Chain,
|
||||||
txid: TransactionId,
|
since: Instant,
|
||||||
maxConfirmations: Int) : this(chain, since, txid, maxConfirmations, TxStatus())
|
txid: TransactionId,
|
||||||
|
maxConfirmations: Int
|
||||||
|
) : this(chain, since, txid, maxConfirmations, TxStatus())
|
||||||
|
|
||||||
fun copy(
|
fun copy(
|
||||||
since: Instant = this.since,
|
since: Instant = this.since,
|
||||||
status: TxStatus = this.status
|
status: TxStatus = this.status
|
||||||
) = TxDetails(chain, since, txid, maxConfirmations, status)
|
) = TxDetails(chain, since, txid, maxConfirmations, status)
|
||||||
|
|
||||||
fun withStatus(found: Boolean = this.status.found,
|
fun withStatus(
|
||||||
height: Long? = this.status.height,
|
found: Boolean = this.status.found,
|
||||||
mined: Boolean = this.status.mined,
|
height: Long? = this.status.height,
|
||||||
blockHash: BlockHash? = this.status.blockHash,
|
mined: Boolean = this.status.mined,
|
||||||
blockTime: Instant? = this.status.blockTime,
|
blockHash: BlockHash? = this.status.blockHash,
|
||||||
blockTotalDifficulty: BigInteger? = this.status.blockTotalDifficulty,
|
blockTime: Instant? = this.status.blockTime,
|
||||||
confirmations: Long = this.status.confirmations): TxDetails {
|
blockTotalDifficulty: BigInteger? = this.status.blockTotalDifficulty,
|
||||||
return copy(status = this.status.copy(found, height, mined, blockHash, blockTime, blockTotalDifficulty, confirmations))
|
confirmations: Long = this.status.confirmations
|
||||||
|
): TxDetails {
|
||||||
|
return copy(
|
||||||
|
status = this.status.copy(
|
||||||
|
found,
|
||||||
|
height,
|
||||||
|
mined,
|
||||||
|
blockHash,
|
||||||
|
blockTime,
|
||||||
|
blockTotalDifficulty,
|
||||||
|
confirmations
|
||||||
|
)
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun shouldClose(): Boolean {
|
fun shouldClose(): Boolean {
|
||||||
return maxConfirmations <= this.status.confirmations
|
return maxConfirmations <= this.status.confirmations ||
|
||||||
|| since.isBefore(Instant.now().minus(TRACK_TTL))
|
since.isBefore(Instant.now().minus(TRACK_TTL)) ||
|
||||||
|| (!status.found && since.isBefore(Instant.now().minus(NOT_FOUND_TRACK_TTL)))
|
(!status.found && since.isBefore(Instant.now().minus(NOT_FOUND_TRACK_TTL))) ||
|
||||||
|| (!status.mined && since.isBefore(Instant.now().minus(NOT_MINED_TRACK_TTL)))
|
(!status.mined && since.isBefore(Instant.now().minus(NOT_MINED_TRACK_TTL)))
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun toString(): String {
|
override fun toString(): String {
|
||||||
@@ -320,25 +338,27 @@ class TrackEthereumTx(
|
|||||||
result = 31 * result + status.hashCode()
|
result = 31 * result + status.hashCode()
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
class TxStatus(val found: Boolean = false,
|
class TxStatus(
|
||||||
val height: Long? = null,
|
val found: Boolean = false,
|
||||||
val mined: Boolean = false,
|
val height: Long? = null,
|
||||||
val blockHash: BlockHash? = null,
|
val mined: Boolean = false,
|
||||||
val blockTime: Instant? = null,
|
val blockHash: BlockHash? = null,
|
||||||
val blockTotalDifficulty: BigInteger? = null,
|
val blockTime: Instant? = null,
|
||||||
val confirmations: Long = 0) {
|
val blockTotalDifficulty: BigInteger? = null,
|
||||||
|
val confirmations: Long = 0
|
||||||
|
) {
|
||||||
|
|
||||||
fun copy(found: Boolean = this.found,
|
fun copy(
|
||||||
height: Long? = this.height,
|
found: Boolean = this.found,
|
||||||
mined: Boolean = this.mined,
|
height: Long? = this.height,
|
||||||
blockHash: BlockHash? = this.blockHash,
|
mined: Boolean = this.mined,
|
||||||
blockTime: Instant? = this.blockTime,
|
blockHash: BlockHash? = this.blockHash,
|
||||||
blockTotalDifficulty: BigInteger? = this.blockTotalDifficulty,
|
blockTime: Instant? = this.blockTime,
|
||||||
confirmation: Long = this.confirmations)
|
blockTotalDifficulty: BigInteger? = this.blockTotalDifficulty,
|
||||||
= TxStatus(found, height, mined, blockHash, blockTime, blockTotalDifficulty, confirmation)
|
confirmation: Long = this.confirmations
|
||||||
|
) = TxStatus(found, height, mined, blockHash, blockTime, blockTotalDifficulty, confirmation)
|
||||||
|
|
||||||
fun clean() = TxStatus(false, null, false, null, null, null, 0)
|
fun clean() = TxStatus(false, null, false, null, null, null, 0)
|
||||||
|
|
||||||
@@ -369,7 +389,5 @@ class TrackEthereumTx(
|
|||||||
override fun toString(): String {
|
override fun toString(): String {
|
||||||
return "TxStatus(found=$found, height=$height, mined=$mined, blockHash=$blockHash, blockTime=$blockTime, blockTotalDifficulty=$blockTotalDifficulty, confirmations=$confirmations)"
|
return "TxStatus(found=$found, height=$height, mined=$mined, blockHash=$blockHash, blockTime=$blockTime, blockTotalDifficulty=$blockTotalDifficulty, confirmations=$confirmations)"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,4 +22,4 @@ import reactor.core.publisher.Flux
|
|||||||
interface TrackTx {
|
interface TrackTx {
|
||||||
fun isSupported(chain: Chain): Boolean
|
fun isSupported(chain: Chain): Boolean
|
||||||
fun subscribe(request: BlockchainOuterClass.TxStatusRequest): Flux<BlockchainOuterClass.TxStatus>
|
fun subscribe(request: BlockchainOuterClass.TxStatusRequest): Flux<BlockchainOuterClass.TxStatus>
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,7 +16,6 @@
|
|||||||
*/
|
*/
|
||||||
package io.emeraldpay.dshackle.startup
|
package io.emeraldpay.dshackle.startup
|
||||||
|
|
||||||
import io.emeraldpay.grpc.BlockchainType
|
|
||||||
import io.emeraldpay.dshackle.FileResolver
|
import io.emeraldpay.dshackle.FileResolver
|
||||||
import io.emeraldpay.dshackle.cache.CachesFactory
|
import io.emeraldpay.dshackle.cache.CachesFactory
|
||||||
import io.emeraldpay.dshackle.config.UpstreamsConfig
|
import io.emeraldpay.dshackle.config.UpstreamsConfig
|
||||||
@@ -34,49 +33,48 @@ import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcHttpClient
|
|||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
||||||
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
|
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
|
||||||
|
import io.emeraldpay.grpc.BlockchainType
|
||||||
import io.emeraldpay.grpc.Chain
|
import io.emeraldpay.grpc.Chain
|
||||||
import io.micrometer.core.instrument.Counter
|
import io.micrometer.core.instrument.Counter
|
||||||
import org.slf4j.LoggerFactory
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
|
||||||
import org.springframework.stereotype.Repository
|
|
||||||
import io.micrometer.core.instrument.Metrics
|
import io.micrometer.core.instrument.Metrics
|
||||||
import io.micrometer.core.instrument.Tag
|
import io.micrometer.core.instrument.Tag
|
||||||
import io.micrometer.core.instrument.Timer
|
import io.micrometer.core.instrument.Timer
|
||||||
|
import org.slf4j.LoggerFactory
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
|
import org.springframework.stereotype.Repository
|
||||||
import java.net.URI
|
import java.net.URI
|
||||||
import java.util.*
|
|
||||||
import java.util.concurrent.atomic.AtomicInteger
|
import java.util.concurrent.atomic.AtomicInteger
|
||||||
import javax.annotation.PostConstruct
|
import javax.annotation.PostConstruct
|
||||||
import kotlin.collections.HashMap
|
|
||||||
|
|
||||||
@Repository
|
@Repository
|
||||||
open class ConfiguredUpstreams(
|
open class ConfiguredUpstreams(
|
||||||
@Autowired private val currentUpstreams: CurrentMultistreamHolder,
|
@Autowired private val currentUpstreams: CurrentMultistreamHolder,
|
||||||
@Autowired private val fileResolver: FileResolver,
|
@Autowired private val fileResolver: FileResolver,
|
||||||
@Autowired private val config: UpstreamsConfig,
|
@Autowired private val config: UpstreamsConfig,
|
||||||
@Autowired private val cachesFactory: CachesFactory
|
@Autowired private val cachesFactory: CachesFactory
|
||||||
) {
|
) {
|
||||||
|
|
||||||
private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java)
|
private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java)
|
||||||
private var seq = AtomicInteger(0)
|
private var seq = AtomicInteger(0)
|
||||||
|
|
||||||
private val chainNames = mapOf(
|
private val chainNames = mapOf(
|
||||||
"ethereum" to Chain.ETHEREUM,
|
"ethereum" to Chain.ETHEREUM,
|
||||||
"ethereum-classic" to Chain.ETHEREUM_CLASSIC,
|
"ethereum-classic" to Chain.ETHEREUM_CLASSIC,
|
||||||
"eth" to Chain.ETHEREUM,
|
"eth" to Chain.ETHEREUM,
|
||||||
"polygon" to Chain.MATIC,
|
"polygon" to Chain.MATIC,
|
||||||
"matic" to Chain.MATIC,
|
"matic" to Chain.MATIC,
|
||||||
"etc" to Chain.ETHEREUM_CLASSIC,
|
"etc" to Chain.ETHEREUM_CLASSIC,
|
||||||
"morden" to Chain.TESTNET_MORDEN,
|
"morden" to Chain.TESTNET_MORDEN,
|
||||||
"kovan" to Chain.TESTNET_KOVAN,
|
"kovan" to Chain.TESTNET_KOVAN,
|
||||||
"kovan-testnet" to Chain.TESTNET_KOVAN,
|
"kovan-testnet" to Chain.TESTNET_KOVAN,
|
||||||
"goerli" to Chain.TESTNET_GOERLI,
|
"goerli" to Chain.TESTNET_GOERLI,
|
||||||
"goerli-testnet" to Chain.TESTNET_GOERLI,
|
"goerli-testnet" to Chain.TESTNET_GOERLI,
|
||||||
"rinkeby" to Chain.TESTNET_RINKEBY,
|
"rinkeby" to Chain.TESTNET_RINKEBY,
|
||||||
"rinkeby-testnet" to Chain.TESTNET_RINKEBY,
|
"rinkeby-testnet" to Chain.TESTNET_RINKEBY,
|
||||||
"ropsten" to Chain.TESTNET_ROPSTEN,
|
"ropsten" to Chain.TESTNET_ROPSTEN,
|
||||||
"ropsten-testnet" to Chain.TESTNET_ROPSTEN,
|
"ropsten-testnet" to Chain.TESTNET_ROPSTEN,
|
||||||
"bitcoin" to Chain.BITCOIN,
|
"bitcoin" to Chain.BITCOIN,
|
||||||
"bitcoin-testnet" to Chain.TESTNET_BITCOIN
|
"bitcoin-testnet" to Chain.TESTNET_BITCOIN
|
||||||
)
|
)
|
||||||
|
|
||||||
@PostConstruct
|
@PostConstruct
|
||||||
@@ -95,7 +93,7 @@ open class ConfiguredUpstreams(
|
|||||||
return@forEach
|
return@forEach
|
||||||
}
|
}
|
||||||
val options = (up.options ?: UpstreamsConfig.Options())
|
val options = (up.options ?: UpstreamsConfig.Options())
|
||||||
.merge(defaultOptions[chain] ?: UpstreamsConfig.Options.getDefaults())
|
.merge(defaultOptions[chain] ?: UpstreamsConfig.Options.getDefaults())
|
||||||
when (BlockchainType.from(chain)) {
|
when (BlockchainType.from(chain)) {
|
||||||
BlockchainType.ETHEREUM -> {
|
BlockchainType.ETHEREUM -> {
|
||||||
buildEthereumUpstream(up.cast(UpstreamsConfig.EthereumConnection::class.java), chain, options)
|
buildEthereumUpstream(up.cast(UpstreamsConfig.EthereumConnection::class.java), chain, options)
|
||||||
@@ -135,9 +133,10 @@ open class ConfiguredUpstreams(
|
|||||||
|
|
||||||
fun buildMethods(config: UpstreamsConfig.Upstream<*>, chain: Chain): CallMethods {
|
fun buildMethods(config: UpstreamsConfig.Upstream<*>, chain: Chain): CallMethods {
|
||||||
return if (config.methods != null) {
|
return if (config.methods != null) {
|
||||||
ManagedCallMethods(currentUpstreams.getDefaultMethods(chain),
|
ManagedCallMethods(
|
||||||
config.methods!!.enabled.map { it.name }.toSet(),
|
currentUpstreams.getDefaultMethods(chain),
|
||||||
config.methods!!.disabled.map { it.name }.toSet()
|
config.methods!!.enabled.map { it.name }.toSet(),
|
||||||
|
config.methods!!.disabled.map { it.name }.toSet()
|
||||||
).also {
|
).also {
|
||||||
config.methods!!.enabled.forEach { m ->
|
config.methods!!.enabled.forEach { m ->
|
||||||
if (m.quorum != null) {
|
if (m.quorum != null) {
|
||||||
@@ -150,9 +149,11 @@ open class ConfiguredUpstreams(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun buildBitcoinUpstream(config: UpstreamsConfig.Upstream<UpstreamsConfig.BitcoinConnection>,
|
private fun buildBitcoinUpstream(
|
||||||
chain: Chain,
|
config: UpstreamsConfig.Upstream<UpstreamsConfig.BitcoinConnection>,
|
||||||
options: UpstreamsConfig.Options) {
|
chain: Chain,
|
||||||
|
options: UpstreamsConfig.Options
|
||||||
|
) {
|
||||||
|
|
||||||
val conn = config.connection!!
|
val conn = config.connection!!
|
||||||
val directApi: Reader<JsonRpcRequest, JsonRpcResponse>? = buildHttpClient(config)
|
val directApi: Reader<JsonRpcRequest, JsonRpcResponse>? = buildHttpClient(config)
|
||||||
@@ -171,19 +172,24 @@ open class ConfiguredUpstreams(
|
|||||||
}
|
}
|
||||||
|
|
||||||
val methods = buildMethods(config, chain)
|
val methods = buildMethods(config, chain)
|
||||||
val upstream = BitcoinRpcUpstream(config.id
|
val upstream = BitcoinRpcUpstream(
|
||||||
?: "bitcoin-${seq.getAndIncrement()}", chain, directApi,
|
config.id
|
||||||
options, config.role,
|
?: "bitcoin-${seq.getAndIncrement()}",
|
||||||
QuorumForLabels.QuorumItem(1, config.labels),
|
chain, directApi,
|
||||||
methods, esplora)
|
options, config.role,
|
||||||
|
QuorumForLabels.QuorumItem(1, config.labels),
|
||||||
|
methods, esplora
|
||||||
|
)
|
||||||
|
|
||||||
upstream.start()
|
upstream.start()
|
||||||
currentUpstreams.update(UpstreamChange(chain, upstream, UpstreamChange.ChangeType.ADDED))
|
currentUpstreams.update(UpstreamChange(chain, upstream, UpstreamChange.ChangeType.ADDED))
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun buildEthereumUpstream(config: UpstreamsConfig.Upstream<UpstreamsConfig.EthereumConnection>,
|
private fun buildEthereumUpstream(
|
||||||
chain: Chain,
|
config: UpstreamsConfig.Upstream<UpstreamsConfig.EthereumConnection>,
|
||||||
options: UpstreamsConfig.Options) {
|
chain: Chain,
|
||||||
|
options: UpstreamsConfig.Options
|
||||||
|
) {
|
||||||
val conn = config.connection!!
|
val conn = config.connection!!
|
||||||
|
|
||||||
val urls = ArrayList<URI>()
|
val urls = ArrayList<URI>()
|
||||||
@@ -194,8 +200,8 @@ open class ConfiguredUpstreams(
|
|||||||
|
|
||||||
val wsFactoryApi: EthereumWsFactory? = conn.ws?.let { endpoint ->
|
val wsFactoryApi: EthereumWsFactory? = conn.ws?.let { endpoint ->
|
||||||
val wsApi = EthereumWsFactory(
|
val wsApi = EthereumWsFactory(
|
||||||
endpoint.url,
|
endpoint.url,
|
||||||
endpoint.origin ?: URI("http://localhost"),
|
endpoint.origin ?: URI("http://localhost"),
|
||||||
)
|
)
|
||||||
wsApi.config = endpoint
|
wsApi.config = endpoint
|
||||||
endpoint.basicAuth?.let { auth ->
|
endpoint.basicAuth?.let { auth ->
|
||||||
@@ -208,11 +214,11 @@ open class ConfiguredUpstreams(
|
|||||||
log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}")
|
log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}")
|
||||||
val ethereumUpstream = if (wsFactoryApi != null && !conn.preferHttp) {
|
val ethereumUpstream = if (wsFactoryApi != null && !conn.preferHttp) {
|
||||||
EthereumWsUpstream(
|
EthereumWsUpstream(
|
||||||
config.id!!,
|
config.id!!,
|
||||||
chain, wsFactoryApi,
|
chain, wsFactoryApi,
|
||||||
options, config.role,
|
options, config.role,
|
||||||
QuorumForLabels.QuorumItem(1, config.labels),
|
QuorumForLabels.QuorumItem(1, config.labels),
|
||||||
methods
|
methods
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
val directApi: Reader<JsonRpcRequest, JsonRpcResponse>? = buildHttpClient(config)
|
val directApi: Reader<JsonRpcRequest, JsonRpcResponse>? = buildHttpClient(config)
|
||||||
@@ -221,11 +227,11 @@ open class ConfiguredUpstreams(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
EthereumRpcUpstream(
|
EthereumRpcUpstream(
|
||||||
config.id!!,
|
config.id!!,
|
||||||
chain, directApi, wsFactoryApi,
|
chain, directApi, wsFactoryApi,
|
||||||
options, config.role,
|
options, config.role,
|
||||||
QuorumForLabels.QuorumItem(1, config.labels),
|
QuorumForLabels.QuorumItem(1, config.labels),
|
||||||
methods
|
methods
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -233,23 +239,26 @@ open class ConfiguredUpstreams(
|
|||||||
currentUpstreams.update(UpstreamChange(chain, ethereumUpstream, UpstreamChange.ChangeType.ADDED))
|
currentUpstreams.update(UpstreamChange(chain, ethereumUpstream, UpstreamChange.ChangeType.ADDED))
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun buildGrpcUpstream(config: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>, options: UpstreamsConfig.Options) {
|
private fun buildGrpcUpstream(
|
||||||
|
config: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>,
|
||||||
|
options: UpstreamsConfig.Options
|
||||||
|
) {
|
||||||
val endpoint = config.connection!!
|
val endpoint = config.connection!!
|
||||||
val ds = GrpcUpstreams(
|
val ds = GrpcUpstreams(
|
||||||
config.id!!,
|
config.id!!,
|
||||||
endpoint.host!!,
|
endpoint.host!!,
|
||||||
endpoint.port ?: 2449,
|
endpoint.port,
|
||||||
endpoint.auth,
|
endpoint.auth,
|
||||||
fileResolver
|
fileResolver
|
||||||
).apply {
|
).apply {
|
||||||
timeout = options.timeout
|
timeout = options.timeout
|
||||||
}
|
}
|
||||||
log.info("Using ALL CHAINS (gRPC) upstream, at ${endpoint.host}:${endpoint.port}")
|
log.info("Using ALL CHAINS (gRPC) upstream, at ${endpoint.host}:${endpoint.port}")
|
||||||
ds.start()
|
ds.start()
|
||||||
.doOnNext {
|
.doOnNext {
|
||||||
log.info("Chain ${it.chain} has ${it.type} through gRPC at ${endpoint.host}:${endpoint.port}")
|
log.info("Chain ${it.chain} has ${it.type} through gRPC at ${endpoint.host}:${endpoint.port}")
|
||||||
}
|
}
|
||||||
.subscribe(currentUpstreams::update)
|
.subscribe(currentUpstreams::update)
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun buildHttpClient(config: UpstreamsConfig.Upstream<out UpstreamsConfig.RpcConnection>): JsonRpcHttpClient? {
|
private fun buildHttpClient(config: UpstreamsConfig.Upstream<out UpstreamsConfig.RpcConnection>): JsonRpcHttpClient? {
|
||||||
@@ -262,29 +271,29 @@ open class ConfiguredUpstreams(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
val metricsTags = listOf(
|
val metricsTags = listOf(
|
||||||
// "unknown" is not supposed to happen
|
// "unknown" is not supposed to happen
|
||||||
Tag.of("upstream", config.id ?: "unknown"),
|
Tag.of("upstream", config.id ?: "unknown"),
|
||||||
// UNSPECIFIED shouldn't happen too
|
// UNSPECIFIED shouldn't happen too
|
||||||
Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED ).chainCode)
|
Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED).chainCode)
|
||||||
)
|
)
|
||||||
val metrics = RpcMetrics(
|
val metrics = RpcMetrics(
|
||||||
Timer.builder("upstream.rpc.conn")
|
Timer.builder("upstream.rpc.conn")
|
||||||
.description("Request time through a HTTP JSON RPC connection")
|
.description("Request time through a HTTP JSON RPC connection")
|
||||||
.tags(metricsTags)
|
.tags(metricsTags)
|
||||||
.publishPercentileHistogram()
|
.publishPercentileHistogram()
|
||||||
.register(Metrics.globalRegistry),
|
.register(Metrics.globalRegistry),
|
||||||
Counter.builder("upstream.rpc.err")
|
Counter.builder("upstream.rpc.err")
|
||||||
.description("Errors received on request through HTTP JSON RPC connection")
|
.description("Errors received on request through HTTP JSON RPC connection")
|
||||||
.tags(metricsTags)
|
.tags(metricsTags)
|
||||||
.register(Metrics.globalRegistry)
|
.register(Metrics.globalRegistry)
|
||||||
)
|
)
|
||||||
urls.add(endpoint.url)
|
urls.add(endpoint.url)
|
||||||
JsonRpcHttpClient(
|
JsonRpcHttpClient(
|
||||||
endpoint.url.toString(),
|
endpoint.url.toString(),
|
||||||
metrics,
|
metrics,
|
||||||
conn.rpc?.basicAuth,
|
conn.rpc?.basicAuth,
|
||||||
tls
|
tls
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,9 +17,8 @@
|
|||||||
package io.emeraldpay.dshackle.startup
|
package io.emeraldpay.dshackle.startup
|
||||||
|
|
||||||
import io.emeraldpay.dshackle.config.UpstreamsConfig
|
import io.emeraldpay.dshackle.config.UpstreamsConfig
|
||||||
import java.util.*
|
import java.util.Collections
|
||||||
import java.util.concurrent.locks.ReentrantReadWriteLock
|
import java.util.concurrent.locks.ReentrantReadWriteLock
|
||||||
import kotlin.collections.ArrayList
|
|
||||||
import kotlin.concurrent.read
|
import kotlin.concurrent.read
|
||||||
import kotlin.concurrent.write
|
import kotlin.concurrent.write
|
||||||
|
|
||||||
@@ -71,7 +70,6 @@ class QuorumForLabels() {
|
|||||||
return nodes.hashCode()
|
return nodes.hashCode()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Details for a single element (upstream, node or aggregation)
|
* Details for a single element (upstream, node or aggregation)
|
||||||
*/
|
*/
|
||||||
@@ -97,8 +95,5 @@ class QuorumForLabels() {
|
|||||||
result = 31 * result + labels.hashCode()
|
result = 31 * result + labels.hashCode()
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user