solution: kotlin code conventions
This commit is contained in:
8
.gitignore
vendored
8
.gitignore
vendored
@@ -1,7 +1,11 @@
|
||||
.gradle/
|
||||
build/
|
||||
out/
|
||||
*.iml
|
||||
./dshackle.yaml
|
||||
./upstream.yaml
|
||||
testsetup/
|
||||
testsetup/
|
||||
.idea/
|
||||
.idea_modules/
|
||||
*.iml
|
||||
*.ipr
|
||||
*.iws
|
||||
@@ -27,6 +27,7 @@ plugins {
|
||||
id 'io.spring.dependency-management' version '1.0.6.RELEASE'
|
||||
id 'com.palantir.git-version' version '0.12.2'
|
||||
id "com.google.protobuf" version "0.8.12"
|
||||
id "org.jlleitschuh.gradle.ktlint" version "10.2.0"
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
kotlin.code.style=official
|
||||
|
||||
# Languages
|
||||
groovyVersion=2.5.14
|
||||
kotlinVersion=1.5.30
|
||||
|
||||
@@ -17,7 +17,7 @@ package io.emeraldpay.dshackle
|
||||
|
||||
import io.emeraldpay.api.proto.Common
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import java.util.*
|
||||
import java.util.EnumMap
|
||||
import java.util.concurrent.locks.ReentrantLock
|
||||
import kotlin.concurrent.withLock
|
||||
|
||||
@@ -25,7 +25,7 @@ import kotlin.concurrent.withLock
|
||||
* Keeps a lazily created value associated with a Chain
|
||||
*/
|
||||
class ChainValue<V>(
|
||||
private val factory: (chain: Chain) -> V
|
||||
private val factory: (chain: Chain) -> V
|
||||
) {
|
||||
|
||||
private val values = EnumMap<Chain, V>(Chain::class.java)
|
||||
@@ -52,4 +52,4 @@ class ChainValue<V>(
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,26 +16,26 @@
|
||||
*/
|
||||
package io.emeraldpay.dshackle
|
||||
|
||||
import com.fasterxml.jackson.core.Version
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import com.fasterxml.jackson.databind.module.SimpleModule
|
||||
import io.emeraldpay.dshackle.config.*
|
||||
import io.emeraldpay.dshackle.config.CacheConfig
|
||||
import io.emeraldpay.dshackle.config.MainConfig
|
||||
import io.emeraldpay.dshackle.config.MainConfigReader
|
||||
import io.emeraldpay.dshackle.config.MonitoringConfig
|
||||
import io.emeraldpay.dshackle.config.TokensConfig
|
||||
import io.emeraldpay.dshackle.config.UpstreamsConfig
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.beans.factory.annotation.Qualifier
|
||||
import org.springframework.boot.ExitCodeGenerator
|
||||
import org.springframework.boot.SpringApplication
|
||||
import org.springframework.context.ApplicationContext
|
||||
import org.springframework.context.annotation.*
|
||||
import org.springframework.context.annotation.Bean
|
||||
import org.springframework.context.annotation.Configuration
|
||||
import org.springframework.core.env.Environment
|
||||
import org.springframework.scheduling.annotation.EnableAsync
|
||||
import org.springframework.scheduling.annotation.EnableScheduling
|
||||
import reactor.core.scheduler.Scheduler
|
||||
import reactor.core.scheduler.Schedulers
|
||||
import java.io.File
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.*
|
||||
import java.util.concurrent.Executors
|
||||
import kotlin.system.exitProcess
|
||||
|
||||
@@ -43,8 +43,8 @@ import kotlin.system.exitProcess
|
||||
@EnableScheduling
|
||||
@EnableAsync
|
||||
open class Config(
|
||||
@Autowired private val env: Environment,
|
||||
@Autowired private val ctx: ApplicationContext
|
||||
@Autowired private val env: Environment,
|
||||
@Autowired private val ctx: ApplicationContext
|
||||
) {
|
||||
|
||||
companion object {
|
||||
@@ -68,14 +68,15 @@ open class Config(
|
||||
if (!FileResolver.isAccessible(target)) {
|
||||
target = File(LOCAL_CONFIG)
|
||||
if (!FileResolver.isAccessible(target)) {
|
||||
throw IllegalStateException("Configuration is not found neither at ${DEFAULT_CONFIG} nor ${LOCAL_CONFIG}")
|
||||
throw IllegalStateException("Configuration is not found neither at $DEFAULT_CONFIG nor $LOCAL_CONFIG")
|
||||
}
|
||||
}
|
||||
target = target.normalize()
|
||||
return target
|
||||
}
|
||||
|
||||
@Bean @Qualifier("upstreamScheduler")
|
||||
@Bean
|
||||
@Qualifier("upstreamScheduler")
|
||||
open fun upstreamScheduler(): Scheduler {
|
||||
return Schedulers.fromExecutorService(Executors.newFixedThreadPool(16))
|
||||
}
|
||||
@@ -91,7 +92,7 @@ open class Config(
|
||||
}
|
||||
val reader = MainConfigReader(fileResolver)
|
||||
return reader.read(f.inputStream())
|
||||
?: throw IllegalStateException("Config is not available at ${f.absolutePath}")
|
||||
?: throw IllegalStateException("Config is not available at ${f.absolutePath}")
|
||||
}
|
||||
|
||||
@Bean
|
||||
@@ -119,5 +120,4 @@ open class Config(
|
||||
open fun monitoringConfig(@Autowired mainConfig: MainConfig): MonitoringConfig {
|
||||
return mainConfig.monitoring
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,4 +25,4 @@ class Defaults {
|
||||
val timeoutInternal: Duration = timeout.dividedBy(4)
|
||||
val retryConnection: Duration = Duration.ofSeconds(10)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ package io.emeraldpay.dshackle
|
||||
import java.io.File
|
||||
|
||||
open class FileResolver(
|
||||
private val baseDir: File
|
||||
private val baseDir: File
|
||||
) {
|
||||
|
||||
companion object {
|
||||
@@ -35,5 +35,4 @@ open class FileResolver(
|
||||
}
|
||||
return File(baseDir, path)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspentDeserializer
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.*
|
||||
import java.util.TimeZone
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.ScheduledExecutorService
|
||||
|
||||
@@ -57,12 +57,10 @@ class Global {
|
||||
objectMapper.registerModule(JavaTimeModule())
|
||||
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
|
||||
objectMapper
|
||||
.setDateFormat(SimpleDateFormat("yyyy-MM-dd\'T\'HH:mm:ss.SSS"))
|
||||
.setTimeZone(TimeZone.getTimeZone("UTC"))
|
||||
.setDateFormat(SimpleDateFormat("yyyy-MM-dd\'T\'HH:mm:ss.SSS"))
|
||||
.setTimeZone(TimeZone.getTimeZone("UTC"))
|
||||
|
||||
return objectMapper
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ package io.emeraldpay.dshackle
|
||||
|
||||
import io.emeraldpay.dshackle.config.MainConfig
|
||||
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerGrpc
|
||||
import io.grpc.*
|
||||
import io.grpc.Server
|
||||
import io.grpc.netty.NettyServerBuilder
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
@@ -29,15 +29,15 @@ import javax.annotation.PreDestroy
|
||||
|
||||
@Service
|
||||
open class GrpcServer(
|
||||
@Autowired val rpcs: List<io.grpc.BindableService>,
|
||||
@Autowired val mainConfig: MainConfig,
|
||||
@Autowired val tlsSetup: TlsSetup,
|
||||
@Autowired val accessHandler: AccessHandlerGrpc
|
||||
@Autowired val rpcs: List<io.grpc.BindableService>,
|
||||
@Autowired val mainConfig: MainConfig,
|
||||
@Autowired val tlsSetup: TlsSetup,
|
||||
@Autowired val accessHandler: AccessHandlerGrpc
|
||||
) {
|
||||
|
||||
private val log = LoggerFactory.getLogger(GrpcServer::class.java)
|
||||
|
||||
private var server: Server? = null;
|
||||
private var server: Server? = null
|
||||
|
||||
@PostConstruct
|
||||
fun start() {
|
||||
@@ -45,14 +45,14 @@ open class GrpcServer(
|
||||
log.debug("Running with DEBUG LOGGING")
|
||||
log.info("Listening Native gRPC on ${mainConfig.host}:${mainConfig.port}")
|
||||
val serverBuilder = NettyServerBuilder
|
||||
.forAddress(InetSocketAddress(mainConfig.host, mainConfig.port))
|
||||
.let {
|
||||
if (mainConfig.accessLogConfig.enabled) {
|
||||
it.intercept(accessHandler)
|
||||
} else {
|
||||
it
|
||||
}
|
||||
.forAddress(InetSocketAddress(mainConfig.host, mainConfig.port))
|
||||
.let {
|
||||
if (mainConfig.accessLogConfig.enabled) {
|
||||
it.intercept(accessHandler)
|
||||
} else {
|
||||
it
|
||||
}
|
||||
}
|
||||
|
||||
tlsSetup.setupServer("Native gRPC", mainConfig.tls, true)?.let {
|
||||
serverBuilder.sslContext(it)
|
||||
@@ -75,5 +75,4 @@ open class GrpcServer(
|
||||
server?.shutdownNow()
|
||||
log.info("GRPC Server shot down")
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
package io.emeraldpay.dshackle
|
||||
|
||||
import io.emeraldpay.dshackle.config.MainConfig
|
||||
import io.emeraldpay.dshackle.config.ProxyConfig
|
||||
import io.emeraldpay.dshackle.monitoring.MonitoringSetup
|
||||
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp
|
||||
import io.emeraldpay.dshackle.proxy.ProxyServer
|
||||
@@ -26,8 +25,6 @@ import io.emeraldpay.dshackle.proxy.WriteRpcJson
|
||||
import io.emeraldpay.dshackle.rpc.NativeCall
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.context.annotation.DependsOn
|
||||
import org.springframework.core.env.Environment
|
||||
import org.springframework.stereotype.Service
|
||||
import javax.annotation.PostConstruct
|
||||
|
||||
@@ -36,14 +33,14 @@ import javax.annotation.PostConstruct
|
||||
*/
|
||||
@Service
|
||||
class ProxyStarter(
|
||||
@Autowired private val mainConfig: MainConfig,
|
||||
@Autowired private val readRpcJson: ReadRpcJson,
|
||||
@Autowired private val writeRpcJson: WriteRpcJson,
|
||||
@Autowired private val nativeCall: NativeCall,
|
||||
@Autowired private val tlsSetup: TlsSetup,
|
||||
@Autowired private val accessHandlerHttp: AccessHandlerHttp,
|
||||
// depend on Monitoring, declared here just to ensure it's properly initialized before the Proxy
|
||||
@Autowired private val monitoringSetup: MonitoringSetup
|
||||
@Autowired private val mainConfig: MainConfig,
|
||||
@Autowired private val readRpcJson: ReadRpcJson,
|
||||
@Autowired private val writeRpcJson: WriteRpcJson,
|
||||
@Autowired private val nativeCall: NativeCall,
|
||||
@Autowired private val tlsSetup: TlsSetup,
|
||||
@Autowired private val accessHandlerHttp: AccessHandlerHttp,
|
||||
// depend on Monitoring, declared here just to ensure it's properly initialized before the Proxy
|
||||
@Autowired private val monitoringSetup: MonitoringSetup
|
||||
) {
|
||||
|
||||
companion object {
|
||||
@@ -60,5 +57,4 @@ class ProxyStarter(
|
||||
val server = ProxyServer(config, readRpcJson, writeRpcJson, nativeCall, tlsSetup, accessHandlerHttp.factory)
|
||||
server.start()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,7 +22,6 @@ import io.emeraldpay.grpc.Chain
|
||||
*/
|
||||
open class SilentException(message: String) : Exception(message) {
|
||||
|
||||
|
||||
/**
|
||||
* Blockchain is not available or not supported by current instance of the Dshackle
|
||||
*/
|
||||
@@ -31,4 +30,4 @@ open class SilentException(message: String) : Exception(message) {
|
||||
}
|
||||
|
||||
class DataUnavailable(val code: String) : SilentException("Data is unavailable: $code")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ import org.springframework.context.annotation.Import
|
||||
import org.springframework.core.io.ClassPathResource
|
||||
import org.springframework.core.io.support.ResourcePropertySource
|
||||
|
||||
@SpringBootApplication(scanBasePackages = [ "io.emeraldpay.dshackle" ])
|
||||
@SpringBootApplication(scanBasePackages = ["io.emeraldpay.dshackle"])
|
||||
@Import(Config::class)
|
||||
open class Starter
|
||||
|
||||
@@ -35,4 +35,4 @@ fun main(args: Array<String>) {
|
||||
app.setDefaultProperties(ResourcePropertySource("version.properties").source)
|
||||
app.setBanner(ResourceBanner(ClassPathResource("banner.txt")))
|
||||
app.run(*args)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
open class TlsSetup(
|
||||
@Autowired val fileResolver: FileResolver
|
||||
@Autowired val fileResolver: FileResolver
|
||||
) {
|
||||
|
||||
companion object {
|
||||
@@ -68,19 +68,19 @@ open class TlsSetup(
|
||||
|
||||
val sslContextBuilder = if (grpc) {
|
||||
GrpcSslContexts.forServer(
|
||||
fileResolver.resolve(config.certificate!!),
|
||||
fileResolver.resolve(config.key!!)
|
||||
fileResolver.resolve(config.certificate!!),
|
||||
fileResolver.resolve(config.key!!)
|
||||
)
|
||||
} else {
|
||||
SslContextBuilder.forServer(
|
||||
fileResolver.resolve(config.certificate!!),
|
||||
fileResolver.resolve(config.key!!)
|
||||
fileResolver.resolve(config.certificate!!),
|
||||
fileResolver.resolve(config.key!!)
|
||||
)
|
||||
}
|
||||
if (StringUtils.isNotEmpty(config.clientCa)) {
|
||||
log.info("Using TLS for client authentication for $category")
|
||||
sslContextBuilder.trustManager(
|
||||
fileResolver.resolve(config.clientCa!!)
|
||||
fileResolver.resolve(config.clientCa!!)
|
||||
)
|
||||
if (config.clientRequire != null && config.clientRequire!!) {
|
||||
sslContextBuilder.clientAuth(ClientAuth.REQUIRE)
|
||||
@@ -96,5 +96,4 @@ open class TlsSetup(
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,8 +25,8 @@ import reactor.core.publisher.Mono
|
||||
* Connects two caches to read through them. First is cache height->hash, second is hash->block.
|
||||
*/
|
||||
open class BlockByHeight(
|
||||
private val heights: Reader<Long, BlockId>,
|
||||
private val blocks: Reader<BlockId, BlockContainer>
|
||||
private val heights: Reader<Long, BlockId>,
|
||||
private val blocks: Reader<BlockId, BlockContainer>
|
||||
) : Reader<Long, BlockContainer> {
|
||||
|
||||
companion object {
|
||||
@@ -35,7 +35,6 @@ open class BlockByHeight(
|
||||
|
||||
override fun read(key: Long): Mono<BlockContainer> {
|
||||
return heights.read(key)
|
||||
.flatMap { blocks.read(it) }
|
||||
.flatMap { blocks.read(it) }
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,12 +23,12 @@ import io.emeraldpay.dshackle.reader.Reader
|
||||
import reactor.core.publisher.Mono
|
||||
|
||||
open class BlocksMemCache(
|
||||
maxSize: Int = 64
|
||||
maxSize: Int = 64
|
||||
) : Reader<BlockId, BlockContainer> {
|
||||
|
||||
private val mapping = Caffeine.newBuilder()
|
||||
.maximumSize(maxSize.toLong())
|
||||
.build<BlockId, BlockContainer>()
|
||||
.maximumSize(maxSize.toLong())
|
||||
.build<BlockId, BlockContainer>()
|
||||
|
||||
override fun read(key: BlockId): Mono<BlockContainer> {
|
||||
return Mono.justOrEmpty(get(key))
|
||||
@@ -45,4 +45,4 @@ open class BlocksMemCache(
|
||||
open fun purge() {
|
||||
mapping.cleanUp()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,10 +32,10 @@ import java.time.Instant
|
||||
* Cache blocks in Redis database
|
||||
*/
|
||||
class BlocksRedisCache(
|
||||
redis: RedisReactiveCommands<String, ByteArray>,
|
||||
chain: Chain
|
||||
redis: RedisReactiveCommands<String, ByteArray>,
|
||||
chain: Chain
|
||||
) : Reader<BlockId, BlockContainer>,
|
||||
OnBlockRedisCache<BlockContainer>(redis, chain, CachesProto.ValueContainer.ValueType.BLOCK) {
|
||||
OnBlockRedisCache<BlockContainer>(redis, chain, CachesProto.ValueContainer.ValueType.BLOCK) {
|
||||
|
||||
companion object {
|
||||
private val log = LoggerFactory.getLogger(BlocksRedisCache::class.java)
|
||||
@@ -59,21 +59,21 @@ class BlocksRedisCache(
|
||||
}
|
||||
val meta = value.blockMeta
|
||||
return BlockContainer(
|
||||
meta.height,
|
||||
BlockId(meta.hash.toByteArray()),
|
||||
BigInteger(meta.difficulty.toByteArray()),
|
||||
Instant.ofEpochMilli(meta.timestamp),
|
||||
false,
|
||||
value.value.toByteArray(),
|
||||
null,
|
||||
meta.txHashesList.map {
|
||||
TxId(it.toByteArray())
|
||||
}
|
||||
meta.height,
|
||||
BlockId(meta.hash.toByteArray()),
|
||||
BigInteger(meta.difficulty.toByteArray()),
|
||||
Instant.ofEpochMilli(meta.timestamp),
|
||||
false,
|
||||
value.value.toByteArray(),
|
||||
null,
|
||||
meta.txHashesList.map {
|
||||
TxId(it.toByteArray())
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
fun add(block: BlockContainer): Mono<Void> {
|
||||
if (block.timestamp == null || block.hash == null) { //null in unit tests
|
||||
if (block.timestamp == null || block.hash == null) { // null in unit tests
|
||||
return Mono.empty()
|
||||
}
|
||||
if (block.full) {
|
||||
@@ -81,5 +81,4 @@ class BlocksRedisCache(
|
||||
}
|
||||
return super.add(block, block)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,9 +16,12 @@
|
||||
package io.emeraldpay.dshackle.cache
|
||||
|
||||
import io.emeraldpay.dshackle.Global
|
||||
import io.emeraldpay.dshackle.data.*
|
||||
import io.emeraldpay.dshackle.data.BlockContainer
|
||||
import io.emeraldpay.dshackle.data.BlockId
|
||||
import io.emeraldpay.dshackle.data.DefaultContainer
|
||||
import io.emeraldpay.dshackle.data.TxContainer
|
||||
import io.emeraldpay.dshackle.data.TxId
|
||||
import io.emeraldpay.dshackle.reader.CompoundReader
|
||||
import io.emeraldpay.dshackle.reader.EmptyReader
|
||||
import io.emeraldpay.dshackle.reader.Reader
|
||||
import io.emeraldpay.dshackle.upstream.Head
|
||||
import io.emeraldpay.dshackle.upstream.ethereum.EthereumFullBlocksReader
|
||||
@@ -30,14 +33,14 @@ import reactor.core.publisher.Flux
|
||||
import reactor.core.publisher.Mono
|
||||
|
||||
open class Caches(
|
||||
private val memBlocksByHash: BlocksMemCache,
|
||||
private val blocksByHeight: HeightCache,
|
||||
private val memTxsByHash: TxMemCache,
|
||||
private val memReceipts: ReceiptMemCache,
|
||||
private val redisBlocksByHash: BlocksRedisCache?,
|
||||
private val redisTxsByHash: TxRedisCache?,
|
||||
private val redisReceipts: ReceiptRedisCache?,
|
||||
private val redisHeightByHashCache: HeightByHashRedisCache?
|
||||
private val memBlocksByHash: BlocksMemCache,
|
||||
private val blocksByHeight: HeightCache,
|
||||
private val memTxsByHash: TxMemCache,
|
||||
private val memReceipts: ReceiptMemCache,
|
||||
private val redisBlocksByHash: BlocksRedisCache?,
|
||||
private val redisTxsByHash: TxRedisCache?,
|
||||
private val redisReceipts: ReceiptRedisCache?,
|
||||
private val redisHeightByHashCache: HeightByHashRedisCache?
|
||||
) {
|
||||
|
||||
companion object {
|
||||
@@ -91,17 +94,17 @@ open class Caches(
|
||||
if (currentHeight != null && data.height != null && memReceipts.acceptsRecentBlocks(currentHeight - data.height)) {
|
||||
memReceipts.add(data)
|
||||
}
|
||||
//TODO move subscription to the caller
|
||||
// TODO move subscription to the caller
|
||||
redisReceipts?.add(data)?.subscribe()
|
||||
}
|
||||
|
||||
fun cache(tag: Tag, tx: TxContainer) {
|
||||
//do not cache transactions that are not in a block yet
|
||||
// do not cache transactions that are not in a block yet
|
||||
if (tx.blockId == null) {
|
||||
return
|
||||
}
|
||||
memTxsByHash.add(tx)
|
||||
//TODO move subscription to the caller
|
||||
// TODO move subscription to the caller
|
||||
getBlocksByHash().read(tx.blockId).flatMap { block ->
|
||||
redisTxsByHash?.add(tx, block) ?: Mono.empty()
|
||||
}.subscribe()
|
||||
@@ -113,14 +116,14 @@ open class Caches(
|
||||
redisHeightByHashCache?.add(block)?.let(job::add)
|
||||
|
||||
if (tag == Tag.LATEST) {
|
||||
//for LATEST data cache it in memory, it may be short living so better to avoid Redis
|
||||
// for LATEST data cache it in memory, it may be short living so better to avoid Redis
|
||||
memoizeBlock(block)
|
||||
} else if (tag == Tag.REQUESTED) {
|
||||
val blockOnlyContainer: BlockContainer?
|
||||
var jsonValue: BlockJson<*>? = null
|
||||
if (block.full) {
|
||||
jsonValue = Global.objectMapper.readValue<BlockJson<*>>(block.json, BlockJson::class.java)
|
||||
//shouldn't cache block json with transactions, separate txes and blocks with refs
|
||||
// shouldn't cache block json with transactions, separate txes and blocks with refs
|
||||
val blockOnly = jsonValue.withoutTransactionDetails()
|
||||
blockOnlyContainer = BlockContainer.from(blockOnly)
|
||||
} else {
|
||||
@@ -138,15 +141,17 @@ open class Caches(
|
||||
TxContainer.from(tx)
|
||||
}
|
||||
if (redisTxsByHash != null) {
|
||||
job.add(Flux.fromIterable(transactions)
|
||||
job.add(
|
||||
Flux.fromIterable(transactions)
|
||||
.doOnNext { memTxsByHash.add(it) }
|
||||
.flatMap { redisTxsByHash.add(it, block) }
|
||||
.then())
|
||||
.then()
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Flux.fromIterable(job).flatMap { it }.subscribe() //TODO move out to a caller
|
||||
Flux.fromIterable(job).flatMap { it }.subscribe() // TODO move out to a caller
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -156,7 +161,7 @@ open class Caches(
|
||||
memBlocksByHash.add(block)
|
||||
memHeightByHash.add(block)
|
||||
val replaced = blocksByHeight.add(block)
|
||||
//evict cached transactions if an existing block was updated
|
||||
// evict cached transactions if an existing block was updated
|
||||
replaced?.let { evict(it) }
|
||||
}
|
||||
|
||||
@@ -222,7 +227,7 @@ open class Caches(
|
||||
REQUESTED
|
||||
}
|
||||
|
||||
class Builder() {
|
||||
class Builder {
|
||||
private var blocksByHash: BlocksMemCache? = null
|
||||
private var blocksByHeight: HeightCache? = null
|
||||
private var txsByHash: TxMemCache? = null
|
||||
@@ -285,8 +290,10 @@ open class Caches(
|
||||
if (receipts == null) {
|
||||
receipts = ReceiptMemCache()
|
||||
}
|
||||
return Caches(blocksByHash!!, blocksByHeight!!, txsByHash!!, receipts!!,
|
||||
redisBlocksByHash, redisTxsByHash, redisReceiptCache, redisHeightByHashCache)
|
||||
return Caches(
|
||||
blocksByHash!!, blocksByHeight!!, txsByHash!!, receipts!!,
|
||||
redisBlocksByHash, redisTxsByHash, redisReceiptCache, redisHeightByHashCache
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,5 +21,4 @@ package io.emeraldpay.dshackle.cache
|
||||
interface CachesEnabled {
|
||||
|
||||
fun setCaches(caches: Caches)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,14 +27,13 @@ import io.lettuce.core.codec.StringCodec
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Repository
|
||||
import java.util.*
|
||||
import java.util.EnumMap
|
||||
import javax.annotation.PostConstruct
|
||||
import kotlin.system.exitProcess
|
||||
|
||||
|
||||
@Repository
|
||||
open class CachesFactory(
|
||||
@Autowired private val cacheConfig: CacheConfig
|
||||
@Autowired private val cacheConfig: CacheConfig
|
||||
) {
|
||||
|
||||
companion object {
|
||||
@@ -50,14 +49,14 @@ open class CachesFactory(
|
||||
val redisConfig = cacheConfig.redis ?: return
|
||||
|
||||
var uri = RedisURI.builder()
|
||||
.withHost(redisConfig.host)
|
||||
.withPort(redisConfig.port)
|
||||
.withHost(redisConfig.host)
|
||||
.withPort(redisConfig.port)
|
||||
|
||||
redisConfig.db?.let { value ->
|
||||
uri = uri.withDatabase(value)
|
||||
}
|
||||
|
||||
//log URI _before_ adding a password, to avoid leaking it to the log
|
||||
// log URI _before_ adding a password, to avoid leaking it to the log
|
||||
log.info("Use Redis cache at: ${uri.build().toURI()}")
|
||||
|
||||
redisConfig.password?.let { value ->
|
||||
@@ -113,4 +112,4 @@ open class CachesFactory(
|
||||
}
|
||||
return existing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,5 +20,4 @@ class CurrentBlockCache<K, D> : Reader<K, D> {
|
||||
fun evict() {
|
||||
cache.set(ConcurrentHashMap())
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,9 +29,9 @@ import reactor.core.publisher.Mono
|
||||
*
|
||||
*/
|
||||
class HeightByHashAdding(
|
||||
private val mem: Reader<BlockId, Long>,
|
||||
private val redis: HeightByHashCache?,
|
||||
private val upstreamReader: Reader<BlockId, BlockContainer>
|
||||
private val mem: Reader<BlockId, Long>,
|
||||
private val redis: HeightByHashCache?,
|
||||
private val upstreamReader: Reader<BlockId, BlockContainer>
|
||||
) : Reader<BlockId, Long> {
|
||||
|
||||
companion object {
|
||||
@@ -39,7 +39,7 @@ class HeightByHashAdding(
|
||||
}
|
||||
|
||||
constructor(caches: Caches, upstreamReader: Reader<BlockId, BlockContainer>) :
|
||||
this(caches.getLastHeightByHash(), caches.getRedisHeightByHash(), upstreamReader)
|
||||
this(caches.getLastHeightByHash(), caches.getRedisHeightByHash(), upstreamReader)
|
||||
|
||||
private val delegate: Reader<BlockId, Long>
|
||||
|
||||
@@ -48,26 +48,26 @@ class HeightByHashAdding(
|
||||
delegate = object : Reader<BlockId, Long> {
|
||||
override fun read(key: BlockId): Mono<Long> {
|
||||
return mem.read(key)
|
||||
.switchIfEmpty(
|
||||
Mono.just(key)
|
||||
.flatMap { redis.read(it) }
|
||||
)
|
||||
.switchIfEmpty(
|
||||
Mono.just(key)
|
||||
.flatMap { upstreamReader.read(it) }
|
||||
.flatMap { redis.add(it).then(Mono.just(it.height)) }
|
||||
)
|
||||
.switchIfEmpty(
|
||||
Mono.just(key)
|
||||
.flatMap { redis.read(it) }
|
||||
)
|
||||
.switchIfEmpty(
|
||||
Mono.just(key)
|
||||
.flatMap { upstreamReader.read(it) }
|
||||
.flatMap { redis.add(it).then(Mono.just(it.height)) }
|
||||
)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
delegate = object : Reader<BlockId, Long> {
|
||||
override fun read(key: BlockId): Mono<Long> {
|
||||
return mem.read(key)
|
||||
.switchIfEmpty(
|
||||
Mono.just(key)
|
||||
.flatMap { upstreamReader.read(it) }
|
||||
.map { it.height }
|
||||
)
|
||||
.switchIfEmpty(
|
||||
Mono.just(key)
|
||||
.flatMap { upstreamReader.read(it) }
|
||||
.map { it.height }
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -76,5 +76,4 @@ class HeightByHashAdding(
|
||||
override fun read(key: BlockId): Mono<Long> {
|
||||
return delegate.read(key)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,4 +23,4 @@ import reactor.core.publisher.Mono
|
||||
interface HeightByHashCache : Reader<BlockId, Long> {
|
||||
|
||||
fun add(block: BlockContainer): Mono<Void>
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ import org.slf4j.LoggerFactory
|
||||
import reactor.core.publisher.Mono
|
||||
|
||||
class HeightByHashMemCache(
|
||||
maxSize: Int = 256
|
||||
maxSize: Int = 256
|
||||
) : Reader<BlockId, Long> {
|
||||
|
||||
companion object {
|
||||
@@ -31,8 +31,8 @@ class HeightByHashMemCache(
|
||||
}
|
||||
|
||||
private val heights = Caffeine.newBuilder()
|
||||
.maximumSize(maxSize.toLong())
|
||||
.build<BlockId, Long>()
|
||||
.maximumSize(maxSize.toLong())
|
||||
.build<BlockId, Long>()
|
||||
|
||||
override fun read(key: BlockId): Mono<Long> {
|
||||
return Mono.justOrEmpty(heights.getIfPresent(key))
|
||||
@@ -41,4 +41,4 @@ class HeightByHashMemCache(
|
||||
fun add(block: BlockContainer) {
|
||||
heights.put(block.hash, block.height)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,8 +32,8 @@ import java.util.concurrent.TimeUnit
|
||||
* reader would use full block's cache to find out height).
|
||||
*/
|
||||
class HeightByHashRedisCache(
|
||||
private val redis: RedisReactiveCommands<String, ByteArray>,
|
||||
private val chain: Chain
|
||||
private val redis: RedisReactiveCommands<String, ByteArray>,
|
||||
private val chain: Chain
|
||||
) : Reader<BlockId, Long>, HeightByHashCache {
|
||||
|
||||
companion object {
|
||||
@@ -45,41 +45,41 @@ class HeightByHashRedisCache(
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
override fun read(key: BlockId): Mono<Long> {
|
||||
return redis.get(key(key))
|
||||
.flatMap { data ->
|
||||
Mono.justOrEmpty(fromBytes(data)) as Mono<Long>
|
||||
}.onErrorResume {
|
||||
log.warn("Failed to read Block Height. ${it.javaClass}:${it.message}")
|
||||
Mono.empty()
|
||||
}
|
||||
.flatMap { data ->
|
||||
Mono.justOrEmpty(fromBytes(data)) as Mono<Long>
|
||||
}.onErrorResume {
|
||||
log.warn("Failed to read Block Height. ${it.javaClass}:${it.message}")
|
||||
Mono.empty()
|
||||
}
|
||||
}
|
||||
|
||||
override fun add(block: BlockContainer): Mono<Void> {
|
||||
return Mono.just(block)
|
||||
.flatMap { blockData ->
|
||||
// even if block replaced, the mapping hash-long is still valid, so can be cached for long time
|
||||
// even for fresh blocks
|
||||
val ttl = TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES)
|
||||
.flatMap { blockData ->
|
||||
// even if block replaced, the mapping hash-long is still valid, so can be cached for long time
|
||||
// even for fresh blocks
|
||||
val ttl = TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES)
|
||||
|
||||
val key = key(blockData.hash)
|
||||
val value = asBytes(blockData.height)
|
||||
redis.setex(key, ttl, value)
|
||||
}
|
||||
.doOnError {
|
||||
log.warn("Failed to save Block Height. ${it.javaClass}:${it.message}")
|
||||
}
|
||||
//if failed to cache, just continue without it
|
||||
.onErrorResume {
|
||||
Mono.empty()
|
||||
}
|
||||
.then()
|
||||
val key = key(blockData.hash)
|
||||
val value = asBytes(blockData.height)
|
||||
redis.setex(key, ttl, value)
|
||||
}
|
||||
.doOnError {
|
||||
log.warn("Failed to save Block Height. ${it.javaClass}:${it.message}")
|
||||
}
|
||||
// if failed to cache, just continue without it
|
||||
.onErrorResume {
|
||||
Mono.empty()
|
||||
}
|
||||
.then()
|
||||
}
|
||||
|
||||
fun asBytes(value: Long): ByteArray {
|
||||
val result = ByteArray(8)
|
||||
val bb = ByteBuffer.allocate(8)
|
||||
.order(ByteOrder.BIG_ENDIAN)
|
||||
.order(ByteOrder.BIG_ENDIAN)
|
||||
bb.asLongBuffer()
|
||||
.put(value)
|
||||
.put(value)
|
||||
bb.get(result)
|
||||
return result
|
||||
}
|
||||
@@ -89,9 +89,9 @@ class HeightByHashRedisCache(
|
||||
return null
|
||||
}
|
||||
return ByteBuffer.wrap(value)
|
||||
.order(ByteOrder.BIG_ENDIAN)
|
||||
.asLongBuffer()
|
||||
.get()
|
||||
.order(ByteOrder.BIG_ENDIAN)
|
||||
.asLongBuffer()
|
||||
.get()
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -100,4 +100,4 @@ class HeightByHashRedisCache(
|
||||
fun key(hash: BlockId): String {
|
||||
return "height:${chain.id}:${hash.toHex()}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,12 +25,12 @@ import reactor.core.publisher.Mono
|
||||
* Memory cache for blocks heights, keeps mapping height->hash.
|
||||
*/
|
||||
open class HeightCache(
|
||||
maxSize: Int = 512
|
||||
maxSize: Int = 512
|
||||
) : Reader<Long, BlockId> {
|
||||
|
||||
private val heights = Caffeine.newBuilder()
|
||||
.maximumSize(maxSize.toLong())
|
||||
.build<Long, BlockId>()
|
||||
.maximumSize(maxSize.toLong())
|
||||
.build<Long, BlockId>()
|
||||
|
||||
override fun read(key: Long): Mono<BlockId> {
|
||||
return Mono.justOrEmpty(heights.getIfPresent(key))
|
||||
@@ -45,5 +45,4 @@ open class HeightCache(
|
||||
fun purge() {
|
||||
heights.cleanUp()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,9 +30,9 @@ import java.util.concurrent.TimeUnit
|
||||
import kotlin.math.min
|
||||
|
||||
abstract class OnBlockRedisCache<T>(
|
||||
private val redis: RedisReactiveCommands<String, ByteArray>,
|
||||
private val chain: Chain,
|
||||
private val valueType: ValueContainer.ValueType
|
||||
private val redis: RedisReactiveCommands<String, ByteArray>,
|
||||
private val chain: Chain,
|
||||
private val valueType: ValueContainer.ValueType
|
||||
) : Reader<BlockId, T> {
|
||||
|
||||
companion object {
|
||||
@@ -51,18 +51,18 @@ abstract class OnBlockRedisCache<T>(
|
||||
|
||||
fun toProto(block: BlockContainer, value: T): ValueContainer {
|
||||
return ValueContainer.newBuilder()
|
||||
.setType(valueType)
|
||||
.setValue(ByteString.copyFrom(serializeValue(value)))
|
||||
.setBlockMeta(buildMeta(block))
|
||||
.build()
|
||||
.setType(valueType)
|
||||
.setValue(ByteString.copyFrom(serializeValue(value)))
|
||||
.setBlockMeta(buildMeta(block))
|
||||
.build()
|
||||
}
|
||||
|
||||
open fun buildMeta(block: BlockContainer): CachesProto.BlockMeta.Builder {
|
||||
return CachesProto.BlockMeta.newBuilder()
|
||||
.setHash(ByteString.copyFrom(block.hash.value))
|
||||
.setHeight(block.height)
|
||||
.setDifficulty(ByteString.copyFrom(block.difficulty.toByteArray()))
|
||||
.setTimestamp(block.timestamp.toEpochMilli())
|
||||
.setHash(ByteString.copyFrom(block.hash.value))
|
||||
.setHeight(block.height)
|
||||
.setDifficulty(ByteString.copyFrom(block.difficulty.toByteArray()))
|
||||
.setTimestamp(block.timestamp.toEpochMilli())
|
||||
}
|
||||
|
||||
abstract fun serializeValue(value: T): ByteArray
|
||||
@@ -83,7 +83,7 @@ abstract class OnBlockRedisCache<T>(
|
||||
* Key in Redis
|
||||
*/
|
||||
fun key(hash: BlockId): String {
|
||||
return "${prefix}:${chain.id}:${hash.toHex()}"
|
||||
return "$prefix:${chain.id}:${hash.toHex()}"
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -92,51 +92,51 @@ abstract class OnBlockRedisCache<T>(
|
||||
*/
|
||||
open fun add(container: BlockContainer, value: T): Mono<Void> {
|
||||
return Mono.just(container)
|
||||
.flatMap { block ->
|
||||
val ttl = cachingTime(block.timestamp)
|
||||
if (ttl > MIN_CACHE_TIME_SECONDS) {
|
||||
val key = key(block.hash)
|
||||
val proto = toProto(block, value)
|
||||
redis.setex(key, ttl, proto.toByteArray())
|
||||
} else {
|
||||
Mono.empty()
|
||||
}
|
||||
}
|
||||
.doOnError {
|
||||
log.warn("Failed to save Block to Redis: ${it.message}")
|
||||
}
|
||||
//if failed to cache, just continue without it
|
||||
.onErrorResume {
|
||||
.flatMap { block ->
|
||||
val ttl = cachingTime(block.timestamp)
|
||||
if (ttl > MIN_CACHE_TIME_SECONDS) {
|
||||
val key = key(block.hash)
|
||||
val proto = toProto(block, value)
|
||||
redis.setex(key, ttl, proto.toByteArray())
|
||||
} else {
|
||||
Mono.empty()
|
||||
}
|
||||
.then()
|
||||
}
|
||||
.doOnError {
|
||||
log.warn("Failed to save Block to Redis: ${it.message}")
|
||||
}
|
||||
// if failed to cache, just continue without it
|
||||
.onErrorResume {
|
||||
Mono.empty()
|
||||
}
|
||||
.then()
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate time to cache the value
|
||||
*/
|
||||
fun cachingTime(blockTime: Instant): Long {
|
||||
//default caching time is age of the block, i.e. block create hour ago
|
||||
//keep for hour, but block created 10 seconds ago cache only for 10 seconds, because it
|
||||
//still can be replaced in the blockchain
|
||||
// default caching time is age of the block, i.e. block create hour ago
|
||||
// keep for hour, but block created 10 seconds ago cache only for 10 seconds, because it
|
||||
// still can be replaced in the blockchain
|
||||
val age = Instant.now().epochSecond - blockTime.epochSecond
|
||||
return min(age, TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES))
|
||||
}
|
||||
|
||||
fun evict(id: BlockId): Mono<Void> {
|
||||
return Mono.just(id)
|
||||
.flatMap {
|
||||
redis.del(key(it))
|
||||
}
|
||||
.then()
|
||||
.flatMap {
|
||||
redis.del(key(it))
|
||||
}
|
||||
.then()
|
||||
}
|
||||
|
||||
override fun read(key: BlockId): Mono<T> {
|
||||
return redis.get(key(key))
|
||||
.map { data ->
|
||||
fromProto(data)
|
||||
}.onErrorResume {
|
||||
Mono.empty()
|
||||
}
|
||||
.map { data ->
|
||||
fromProto(data)
|
||||
}.onErrorResume {
|
||||
Mono.empty()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,9 +30,9 @@ import java.util.concurrent.TimeUnit
|
||||
import kotlin.math.min
|
||||
|
||||
abstract class OnTxRedisCache<T>(
|
||||
private val redis: RedisReactiveCommands<String, ByteArray>,
|
||||
private val chain: Chain,
|
||||
private val valueType: CachesProto.ValueContainer.ValueType
|
||||
private val redis: RedisReactiveCommands<String, ByteArray>,
|
||||
private val chain: Chain,
|
||||
private val valueType: CachesProto.ValueContainer.ValueType
|
||||
) : Reader<TxId, T> {
|
||||
|
||||
companion object {
|
||||
@@ -56,42 +56,42 @@ abstract class OnTxRedisCache<T>(
|
||||
* Key in Redis
|
||||
*/
|
||||
fun key(hash: TxId): String {
|
||||
return "${prefix}:${chain.id}:${hash.toHex()}"
|
||||
return "$prefix:${chain.id}:${hash.toHex()}"
|
||||
}
|
||||
|
||||
fun evict(container: BlockContainer): Mono<Void> {
|
||||
return Mono.just(container)
|
||||
.map { block ->
|
||||
block.transactions.map {
|
||||
key(it)
|
||||
}.toTypedArray()
|
||||
}.flatMap { keys ->
|
||||
redis.del(*keys)
|
||||
}.then()
|
||||
.map { block ->
|
||||
block.transactions.map {
|
||||
key(it)
|
||||
}.toTypedArray()
|
||||
}.flatMap { keys ->
|
||||
redis.del(*keys)
|
||||
}.then()
|
||||
}
|
||||
|
||||
fun evict(id: TxId): Mono<Void> {
|
||||
return Mono.just(id)
|
||||
.flatMap {
|
||||
redis.del(key(it))
|
||||
}
|
||||
.then()
|
||||
.flatMap {
|
||||
redis.del(key(it))
|
||||
}
|
||||
.then()
|
||||
}
|
||||
|
||||
fun toProto(id: TxId, value: T): ByteArray {
|
||||
val meta = buildMeta(id, value)
|
||||
|
||||
return CachesProto.ValueContainer.newBuilder()
|
||||
.setType(valueType)
|
||||
.setValue(ByteString.copyFrom(serializeValue(value)))
|
||||
.setTxMeta(meta)
|
||||
.build()
|
||||
.toByteArray()
|
||||
.setType(valueType)
|
||||
.setValue(ByteString.copyFrom(serializeValue(value)))
|
||||
.setTxMeta(meta)
|
||||
.build()
|
||||
.toByteArray()
|
||||
}
|
||||
|
||||
open fun buildMeta(id: TxId, value: T): CachesProto.TxMeta.Builder {
|
||||
return CachesProto.TxMeta.newBuilder()
|
||||
.setHash(ByteString.copyFrom(id.value))
|
||||
.setHash(ByteString.copyFrom(id.value))
|
||||
}
|
||||
|
||||
abstract fun serializeValue(value: T): ByteArray
|
||||
@@ -110,43 +110,43 @@ abstract class OnTxRedisCache<T>(
|
||||
|
||||
override fun read(key: TxId): Mono<T> {
|
||||
return redis.get(key(key))
|
||||
.map { data ->
|
||||
fromProto(data)
|
||||
}.onErrorResume {
|
||||
Mono.empty()
|
||||
}
|
||||
.map { data ->
|
||||
fromProto(data)
|
||||
}.onErrorResume {
|
||||
Mono.empty()
|
||||
}
|
||||
}
|
||||
|
||||
open fun add(id: TxId, value: T, block: BlockContainer?, blockHeight: Long?): Mono<Void> {
|
||||
return Mono.just(id)
|
||||
.flatMap {
|
||||
val key = key(it)
|
||||
val encodedValue = toProto(it, value)
|
||||
val ttl = if (block?.timestamp != null) {
|
||||
cachingTime(block.timestamp)
|
||||
} else {
|
||||
cachingTime(blockHeight)
|
||||
}
|
||||
//store
|
||||
redis.setex(key, ttl, encodedValue)
|
||||
.flatMap {
|
||||
val key = key(it)
|
||||
val encodedValue = toProto(it, value)
|
||||
val ttl = if (block?.timestamp != null) {
|
||||
cachingTime(block.timestamp)
|
||||
} else {
|
||||
cachingTime(blockHeight)
|
||||
}
|
||||
.doOnError {
|
||||
log.warn("Failed to save TX to Redis: ${it.message}", it)
|
||||
}
|
||||
//if failed to cache, just continue without it
|
||||
.onErrorResume {
|
||||
Mono.empty()
|
||||
}
|
||||
.then()
|
||||
// store
|
||||
redis.setex(key, ttl, encodedValue)
|
||||
}
|
||||
.doOnError {
|
||||
log.warn("Failed to save TX to Redis: ${it.message}", it)
|
||||
}
|
||||
// if failed to cache, just continue without it
|
||||
.onErrorResume {
|
||||
Mono.empty()
|
||||
}
|
||||
.then()
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate time to cache the value, based on block time
|
||||
*/
|
||||
fun cachingTime(blockTime: Instant): Long {
|
||||
//default caching time is age of the block, i.e. block create hour ago
|
||||
//keep for hour, but block create 10 seconds ago cache for 10 seconds, as it
|
||||
//still can be replaced in the blockchain
|
||||
// default caching time is age of the block, i.e. block create hour ago
|
||||
// keep for hour, but block create 10 seconds ago cache for 10 seconds, as it
|
||||
// still can be replaced in the blockchain
|
||||
val age = Instant.now().epochSecond - blockTime.epochSecond
|
||||
return min(age, TimeUnit.HOURS.toSeconds(MAX_CACHE_TIME_HOURS))
|
||||
}
|
||||
@@ -165,4 +165,4 @@ abstract class OnTxRedisCache<T>(
|
||||
}
|
||||
return min(confirmations * BLOCK_TIME_SECONDS, TimeUnit.HOURS.toSeconds(MAX_CACHE_TIME_HOURS))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,8 +28,8 @@ import reactor.core.publisher.Mono
|
||||
* Keeps receipts for recent blocks in memory
|
||||
*/
|
||||
open class ReceiptMemCache(
|
||||
// how many blocks to keeps in memory
|
||||
val blocks: Int = 6
|
||||
// how many blocks to keeps in memory
|
||||
val blocks: Int = 6
|
||||
) : Reader<TxId, ByteArray> {
|
||||
|
||||
companion object {
|
||||
@@ -37,8 +37,8 @@ open class ReceiptMemCache(
|
||||
}
|
||||
|
||||
private val mapping = Caffeine.newBuilder()
|
||||
.maximumSize(blocks * 200L)
|
||||
.build<TxId, ByteArray>()
|
||||
.maximumSize(blocks * 200L)
|
||||
.build<TxId, ByteArray>()
|
||||
|
||||
open fun evict(block: BlockContainer) {
|
||||
block.transactions.forEach {
|
||||
@@ -60,5 +60,4 @@ open class ReceiptMemCache(
|
||||
open fun acceptsRecentBlocks(heightDelta: Long): Boolean {
|
||||
return blocks <= heightDelta && heightDelta >= 0
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,16 +16,15 @@
|
||||
package io.emeraldpay.dshackle.cache
|
||||
|
||||
import io.emeraldpay.dshackle.data.DefaultContainer
|
||||
import io.emeraldpay.dshackle.data.TxId
|
||||
import io.emeraldpay.dshackle.proto.CachesProto
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import io.emeraldpay.etherjar.rpc.json.TransactionReceiptJson
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import io.lettuce.core.api.reactive.RedisReactiveCommands
|
||||
import reactor.core.publisher.Mono
|
||||
|
||||
open class ReceiptRedisCache(
|
||||
redis: RedisReactiveCommands<String, ByteArray>,
|
||||
chain: Chain
|
||||
redis: RedisReactiveCommands<String, ByteArray>,
|
||||
chain: Chain
|
||||
) : OnTxRedisCache<ByteArray>(redis, chain, CachesProto.ValueContainer.ValueType.TX_RECEIPT) {
|
||||
|
||||
override fun deserializeValue(value: CachesProto.ValueContainer): ByteArray {
|
||||
@@ -39,4 +38,4 @@ open class ReceiptRedisCache(
|
||||
fun add(json: DefaultContainer<TransactionReceiptJson>): Mono<Void> {
|
||||
return super.add(json.txId!!, json.json!!, null, json.height)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,8 +28,8 @@ import reactor.core.publisher.Mono
|
||||
* Memory cache for transactions
|
||||
*/
|
||||
open class TxMemCache(
|
||||
// usually there is 100-150 tx per block on Ethereum, we keep data for about 32 blocks by default
|
||||
private val maxSize: Int = 125 * 32
|
||||
// usually there is 100-150 tx per block on Ethereum, we keep data for about 32 blocks by default
|
||||
private val maxSize: Int = 125 * 32
|
||||
) : Reader<TxId, TxContainer> {
|
||||
|
||||
companion object {
|
||||
@@ -37,8 +37,8 @@ open class TxMemCache(
|
||||
}
|
||||
|
||||
private val mapping = Caffeine.newBuilder()
|
||||
.maximumSize(maxSize.toLong())
|
||||
.build<TxId, TxContainer>()
|
||||
.maximumSize(maxSize.toLong())
|
||||
.build<TxId, TxContainer>()
|
||||
|
||||
override fun read(key: TxId): Mono<TxContainer> {
|
||||
return Mono.justOrEmpty(mapping.getIfPresent(key))
|
||||
@@ -52,13 +52,13 @@ open class TxMemCache(
|
||||
|
||||
open fun evict(block: BlockId) {
|
||||
val ids = mapping.asMap()
|
||||
.filter { it.value.blockId == block }
|
||||
.map { it.key }
|
||||
.filter { it.value.blockId == block }
|
||||
.map { it.key }
|
||||
mapping.invalidateAll(ids)
|
||||
}
|
||||
|
||||
open fun add(tx: TxContainer) {
|
||||
//do not cache fresh transactions
|
||||
// do not cache fresh transactions
|
||||
if (tx.blockId == null) {
|
||||
return
|
||||
}
|
||||
@@ -68,4 +68,4 @@ open class TxMemCache(
|
||||
open fun purge() {
|
||||
mapping.cleanUp()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,31 +15,26 @@
|
||||
*/
|
||||
package io.emeraldpay.dshackle.cache
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import com.google.protobuf.ByteString
|
||||
import io.emeraldpay.dshackle.data.BlockContainer
|
||||
import io.emeraldpay.dshackle.data.BlockId
|
||||
import io.emeraldpay.dshackle.data.TxContainer
|
||||
import io.emeraldpay.dshackle.data.TxId
|
||||
import io.emeraldpay.dshackle.proto.CachesProto
|
||||
import io.emeraldpay.dshackle.reader.Reader
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import io.emeraldpay.dshackle.proto.CachesProto
|
||||
import io.lettuce.core.api.reactive.RedisReactiveCommands
|
||||
import org.slf4j.LoggerFactory
|
||||
import reactor.core.publisher.Mono
|
||||
import reactor.util.function.Tuples
|
||||
import java.time.Instant
|
||||
import java.util.concurrent.TimeUnit
|
||||
import kotlin.math.min
|
||||
|
||||
/**
|
||||
* Cache transactions in Redis, up to 24 hours.
|
||||
*/
|
||||
open class TxRedisCache(
|
||||
private val redis: RedisReactiveCommands<String, ByteArray>,
|
||||
private val chain: Chain
|
||||
private val redis: RedisReactiveCommands<String, ByteArray>,
|
||||
private val chain: Chain
|
||||
) : Reader<TxId, TxContainer>,
|
||||
OnTxRedisCache<TxContainer>(redis, chain, CachesProto.ValueContainer.ValueType.TX) {
|
||||
OnTxRedisCache<TxContainer>(redis, chain, CachesProto.ValueContainer.ValueType.TX) {
|
||||
|
||||
companion object {
|
||||
private val log = LoggerFactory.getLogger(TxRedisCache::class.java)
|
||||
@@ -67,15 +62,14 @@ open class TxRedisCache(
|
||||
}
|
||||
val meta = value.txMeta
|
||||
return TxContainer(
|
||||
meta.height,
|
||||
TxId(meta.hash.toByteArray()),
|
||||
BlockId(meta.blockHash.toByteArray()),
|
||||
value.value.toByteArray()
|
||||
meta.height,
|
||||
TxId(meta.hash.toByteArray()),
|
||||
BlockId(meta.blockHash.toByteArray()),
|
||||
value.value.toByteArray()
|
||||
)
|
||||
}
|
||||
|
||||
open fun add(tx: TxContainer, block: BlockContainer): Mono<Void> {
|
||||
return super.add(tx.hash, tx, block, tx.height)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package io.emeraldpay.dshackle.config
|
||||
|
||||
class AccessLogConfig(
|
||||
val enabled: Boolean = false
|
||||
val enabled: Boolean = false
|
||||
) {
|
||||
|
||||
var filename: String = "./access_log.jsonl"
|
||||
@@ -14,9 +14,8 @@ class AccessLogConfig(
|
||||
|
||||
fun disabled(): AccessLogConfig {
|
||||
return AccessLogConfig(
|
||||
enabled = false
|
||||
enabled = false
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,5 +23,4 @@ class AccessLogReader : YamlConfigReader(), ConfigReader<AccessLogConfig> {
|
||||
}
|
||||
} ?: AccessLogConfig.default()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,14 +29,14 @@ class AuthConfig {
|
||||
}
|
||||
|
||||
class ClientBasicAuth(
|
||||
val username: String,
|
||||
val password: String
|
||||
val username: String,
|
||||
val password: String
|
||||
) : ClientAuth()
|
||||
|
||||
class ClientTlsAuth(
|
||||
var ca: String? = null,
|
||||
var certificate: String? = null,
|
||||
var key: String? = null
|
||||
var ca: String? = null,
|
||||
var certificate: String? = null,
|
||||
var key: String? = null
|
||||
) : ClientAuth()
|
||||
|
||||
/**
|
||||
@@ -58,4 +58,4 @@ class AuthConfig {
|
||||
var clientRequire: Boolean? = null
|
||||
var clientCa: String? = null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,5 +82,4 @@ class AuthConfigReader : YamlConfigReader() {
|
||||
auth
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,12 +17,12 @@ package io.emeraldpay.dshackle.config
|
||||
|
||||
class CacheConfig {
|
||||
|
||||
var redis: Redis? = null;
|
||||
var redis: Redis? = null
|
||||
|
||||
class Redis(
|
||||
var host: String = "127.0.0.1",
|
||||
var port: Int = 6379,
|
||||
var db: Int? = 0,
|
||||
var password: String? = null
|
||||
var host: String = "127.0.0.1",
|
||||
var port: Int = 6379,
|
||||
var db: Int? = 0,
|
||||
var password: String? = null
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,4 +58,4 @@ class CacheConfigReader : YamlConfigReader(), ConfigReader<CacheConfig> {
|
||||
config
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,5 +20,4 @@ import org.yaml.snakeyaml.nodes.MappingNode
|
||||
interface ConfigReader<T> {
|
||||
|
||||
fun read(input: MappingNode?): T?
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,4 +31,4 @@ class EnvVariables {
|
||||
} ?: ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,11 +19,11 @@ package io.emeraldpay.dshackle.config
|
||||
import org.yaml.snakeyaml.error.Mark
|
||||
|
||||
open class InvalidConfigException(
|
||||
message: String
|
||||
message: String
|
||||
) : Exception(message)
|
||||
|
||||
class InvalidConfigYamlException(
|
||||
filename: String,
|
||||
mark: Mark,
|
||||
message: String
|
||||
) : InvalidConfigException("Invalid YAML configuration ${message}, at ${filename}:${mark.line}")
|
||||
filename: String,
|
||||
mark: Mark,
|
||||
message: String
|
||||
) : InvalidConfigException("Invalid YAML configuration $message, at $filename:${mark.line}")
|
||||
|
||||
@@ -25,4 +25,4 @@ class MainConfig {
|
||||
var tokens: TokensConfig? = null
|
||||
var monitoring: MonitoringConfig = MonitoringConfig.default()
|
||||
var accessLogConfig: AccessLogConfig = AccessLogConfig.default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ import org.yaml.snakeyaml.nodes.MappingNode
|
||||
import java.io.InputStream
|
||||
|
||||
class MainConfigReader(
|
||||
fileResolver: FileResolver
|
||||
fileResolver: FileResolver
|
||||
) : YamlConfigReader(), ConfigReader<MainConfig> {
|
||||
|
||||
companion object {
|
||||
@@ -73,5 +73,4 @@ class MainConfigReader(
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,14 +16,15 @@
|
||||
package io.emeraldpay.dshackle.config
|
||||
|
||||
class MonitoringConfig(
|
||||
val enabled: Boolean,
|
||||
val prometheus: PrometheusConfig
|
||||
val enabled: Boolean,
|
||||
val prometheus: PrometheusConfig
|
||||
) {
|
||||
|
||||
companion object {
|
||||
fun default(): MonitoringConfig {
|
||||
return MonitoringConfig(true, PrometheusConfig.default())
|
||||
}
|
||||
|
||||
fun disabled(): MonitoringConfig {
|
||||
return MonitoringConfig(false, PrometheusConfig.disabled())
|
||||
}
|
||||
@@ -33,19 +34,19 @@ class MonitoringConfig(
|
||||
var enableExtended: Boolean = false
|
||||
|
||||
data class PrometheusConfig(
|
||||
val enabled: Boolean,
|
||||
val path: String,
|
||||
val host: String,
|
||||
val port: Int
|
||||
val enabled: Boolean,
|
||||
val path: String,
|
||||
val host: String,
|
||||
val port: Int
|
||||
) {
|
||||
companion object {
|
||||
fun default(): PrometheusConfig {
|
||||
return PrometheusConfig(true, "/metrics", "127.0.0.1", 8081)
|
||||
}
|
||||
|
||||
fun disabled(): PrometheusConfig {
|
||||
return PrometheusConfig(false, "/", "127.0.0.1", 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ import org.slf4j.LoggerFactory
|
||||
import org.yaml.snakeyaml.nodes.MappingNode
|
||||
import java.io.InputStream
|
||||
|
||||
class MonitoringConfigReader: YamlConfigReader(), ConfigReader<MonitoringConfig> {
|
||||
class MonitoringConfigReader : YamlConfigReader(), ConfigReader<MonitoringConfig> {
|
||||
|
||||
companion object {
|
||||
private val log = LoggerFactory.getLogger(MonitoringConfigReader::class.java)
|
||||
@@ -64,5 +64,4 @@ class MonitoringConfigReader: YamlConfigReader(), ConfigReader<MonitoringConfig>
|
||||
val port = getValueAsInt(input, "port") ?: default.port
|
||||
return MonitoringConfig.PrometheusConfig(enabled, path, host, port)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ import io.emeraldpay.grpc.Chain
|
||||
open class ProxyConfig {
|
||||
|
||||
companion object {
|
||||
public const val CONFIG_ID = "parsed.proxy"
|
||||
const val CONFIG_ID = "parsed.proxy"
|
||||
}
|
||||
|
||||
var enabled: Boolean = true
|
||||
@@ -50,13 +50,13 @@ open class ProxyConfig {
|
||||
var routes: List<Route> = ArrayList()
|
||||
|
||||
class Route(
|
||||
/**
|
||||
* URL binding for the route. http://$host:$port/$id
|
||||
*/
|
||||
val id: String,
|
||||
/**
|
||||
* Blockchain to dispatch requests
|
||||
*/
|
||||
val blockchain: Chain
|
||||
/**
|
||||
* URL binding for the route. http://$host:$port/$id
|
||||
*/
|
||||
val id: String,
|
||||
/**
|
||||
* Blockchain to dispatch requests
|
||||
*/
|
||||
val blockchain: Chain
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,10 +19,8 @@ package io.emeraldpay.dshackle.config
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.yaml.snakeyaml.Yaml
|
||||
import org.yaml.snakeyaml.nodes.MappingNode
|
||||
import java.io.InputStream
|
||||
import java.io.InputStreamReader
|
||||
|
||||
/**
|
||||
* Read YAML config, part related to Proxy configuration
|
||||
@@ -84,5 +82,4 @@ class ProxyConfigReader : YamlConfigReader(), ConfigReader<ProxyConfig> {
|
||||
config.tls = authConfigReader.readServerTls(input)
|
||||
return config
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,12 +15,12 @@
|
||||
*/
|
||||
package io.emeraldpay.dshackle.config
|
||||
|
||||
import io.emeraldpay.etherjar.domain.Address
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import io.emeraldpay.etherjar.domain.Address
|
||||
|
||||
class TokensConfig(
|
||||
val tokens: List<Token>
|
||||
val tokens: List<Token>
|
||||
) {
|
||||
|
||||
class Token {
|
||||
@@ -30,7 +30,7 @@ class TokensConfig(
|
||||
|
||||
// coin name
|
||||
var name: String? = null
|
||||
var type: Type? = null;
|
||||
var type: Type? = null
|
||||
var address: String? = null
|
||||
|
||||
fun validate(): String? {
|
||||
@@ -40,9 +40,9 @@ class TokensConfig(
|
||||
name.isNullOrBlank() -> "name"
|
||||
type == null -> type
|
||||
address.isNullOrBlank() -> "address"
|
||||
blockchain != null
|
||||
&& BlockchainType.from(blockchain!!) == BlockchainType.ETHEREUM
|
||||
&& !Address.isValidAddress(address) -> "address"
|
||||
blockchain != null &&
|
||||
BlockchainType.from(blockchain!!) == BlockchainType.ETHEREUM &&
|
||||
!Address.isValidAddress(address) -> "address"
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
@@ -51,5 +51,4 @@ class TokensConfig(
|
||||
enum class Type {
|
||||
ERC20
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ package io.emeraldpay.dshackle.config
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.yaml.snakeyaml.nodes.MappingNode
|
||||
import java.io.InputStream
|
||||
import java.util.*
|
||||
import java.util.Locale
|
||||
|
||||
class TokensConfigReader : YamlConfigReader(), ConfigReader<TokensConfig> {
|
||||
|
||||
@@ -60,5 +60,4 @@ class TokensConfigReader : YamlConfigReader(), ConfigReader<TokensConfig> {
|
||||
TokensConfig(it)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,11 +17,9 @@
|
||||
package io.emeraldpay.dshackle.config
|
||||
|
||||
import io.emeraldpay.dshackle.Defaults
|
||||
import java.lang.ClassCastException
|
||||
import java.net.URI
|
||||
import java.util.*
|
||||
import kotlin.collections.ArrayList
|
||||
import kotlin.collections.HashMap
|
||||
import java.util.Arrays
|
||||
import java.util.Locale
|
||||
|
||||
open class UpstreamsConfig {
|
||||
var defaultOptions: MutableList<DefaultOptions> = ArrayList<DefaultOptions>()
|
||||
@@ -46,8 +44,10 @@ open class UpstreamsConfig {
|
||||
}
|
||||
val copy = Options()
|
||||
copy.minPeers = if (this.minPeers != null) this.minPeers else additional.minPeers
|
||||
copy.disableValidation = if (this.disableValidation != null) this.disableValidation else additional.disableValidation
|
||||
copy.providesBalance = if (this.providesBalance != null) this.providesBalance else additional.providesBalance
|
||||
copy.disableValidation =
|
||||
if (this.disableValidation != null) this.disableValidation else additional.disableValidation
|
||||
copy.providesBalance =
|
||||
if (this.providesBalance != null) this.providesBalance else additional.providesBalance
|
||||
return copy
|
||||
}
|
||||
|
||||
@@ -60,7 +60,6 @@ open class UpstreamsConfig {
|
||||
return options
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
class DefaultOptions : Options() {
|
||||
@@ -125,14 +124,14 @@ open class UpstreamsConfig {
|
||||
var msgSize: Int? = null
|
||||
}
|
||||
|
||||
|
||||
//TODO make it unmodifiable after initial load
|
||||
class Labels: HashMap<String, String>() {
|
||||
// TODO make it unmodifiable after initial load
|
||||
class Labels : HashMap<String, String>() {
|
||||
|
||||
companion object {
|
||||
@JvmStatic fun fromMap(map: Map<String, String>): Labels {
|
||||
@JvmStatic
|
||||
fun fromMap(map: Map<String, String>): Labels {
|
||||
val labels = Labels()
|
||||
map.entries.forEach() { kv ->
|
||||
map.entries.forEach { kv ->
|
||||
labels.put(kv.key, kv.value)
|
||||
}
|
||||
return labels
|
||||
@@ -140,7 +139,7 @@ open class UpstreamsConfig {
|
||||
}
|
||||
}
|
||||
|
||||
enum class UpstreamType private constructor(vararg code: String) {
|
||||
enum class UpstreamType(vararg code: String) {
|
||||
ETHEREUM_JSON_RPC("ethereum"),
|
||||
BITCOIN_JSON_RPC("bitcoin"),
|
||||
DSHACKLE("dshackle", "grpc"),
|
||||
@@ -168,12 +167,12 @@ open class UpstreamsConfig {
|
||||
}
|
||||
|
||||
class Methods(
|
||||
val enabled: Set<Method>,
|
||||
val disabled: Set<Method>
|
||||
val enabled: Set<Method>,
|
||||
val disabled: Set<Method>
|
||||
)
|
||||
|
||||
class Method(
|
||||
val name: String,
|
||||
val quorum: String? = null
|
||||
val name: String,
|
||||
val quorum: String? = null
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,14 +23,12 @@ import org.yaml.snakeyaml.nodes.MappingNode
|
||||
import org.yaml.snakeyaml.nodes.ScalarNode
|
||||
import reactor.util.function.Tuples
|
||||
import java.io.InputStream
|
||||
import java.lang.IllegalArgumentException
|
||||
import java.net.URI
|
||||
import java.time.Duration
|
||||
import java.util.*
|
||||
import kotlin.collections.ArrayList
|
||||
import java.util.Locale
|
||||
|
||||
class UpstreamsConfigReader(
|
||||
private val fileResolver: FileResolver
|
||||
private val fileResolver: FileResolver
|
||||
) : YamlConfigReader(), ConfigReader<UpstreamsConfig> {
|
||||
|
||||
private val log = LoggerFactory.getLogger(UpstreamsConfigReader::class.java)
|
||||
@@ -198,7 +196,10 @@ class UpstreamsConfigReader(
|
||||
upstream.methods = tryReadMethods(upNode)
|
||||
}
|
||||
|
||||
internal fun readUpstreamGrpc(upNode: MappingNode, upstream: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>) {
|
||||
internal fun readUpstreamGrpc(
|
||||
upNode: MappingNode,
|
||||
upstream: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>
|
||||
) {
|
||||
if (hasAny(upNode, "labels")) {
|
||||
log.warn("Labels should be not applied to gRPC upstream")
|
||||
}
|
||||
@@ -221,13 +222,13 @@ class UpstreamsConfigReader(
|
||||
if (hasAny(upNode, "labels")) {
|
||||
getMapping(upNode, "labels")?.let { labels ->
|
||||
labels.value.stream()
|
||||
.filter { n -> n.keyNode is ScalarNode && n.valueNode is ScalarNode }
|
||||
.map { n -> Tuples.of((n.keyNode as ScalarNode).value, (n.valueNode as ScalarNode).value) }
|
||||
.map { kv -> Tuples.of(kv.t1.trim(), kv.t2.trim()) }
|
||||
.filter { kv -> StringUtils.isNotEmpty(kv.t1) && StringUtils.isNotEmpty(kv.t2) }
|
||||
.forEach { kv ->
|
||||
upstream.labels[kv.t1] = kv.t2
|
||||
}
|
||||
.filter { n -> n.keyNode is ScalarNode && n.valueNode is ScalarNode }
|
||||
.map { n -> Tuples.of((n.keyNode as ScalarNode).value, (n.valueNode as ScalarNode).value) }
|
||||
.map { kv -> Tuples.of(kv.t1.trim(), kv.t2.trim()) }
|
||||
.filter { kv -> StringUtils.isNotEmpty(kv.t1) && StringUtils.isNotEmpty(kv.t2) }
|
||||
.forEach { kv ->
|
||||
upstream.labels[kv.t1] = kv.t2
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -247,21 +248,21 @@ class UpstreamsConfigReader(
|
||||
val enabled = getList<MappingNode>(mnode, "enabled")?.value?.map { m ->
|
||||
getValueAsString(m, "name")?.let { name ->
|
||||
UpstreamsConfig.Method(
|
||||
name = name,
|
||||
quorum = getValueAsString(m, "quorum")
|
||||
name = name,
|
||||
quorum = getValueAsString(m, "quorum")
|
||||
)
|
||||
}
|
||||
}?.filterNotNull()?.toSet() ?: emptySet()
|
||||
val disabled = getList<MappingNode>(mnode, "disabled")?.value?.map { m ->
|
||||
getValueAsString(m, "name")?.let { name ->
|
||||
UpstreamsConfig.Method(
|
||||
name = name
|
||||
name = name
|
||||
)
|
||||
}
|
||||
}?.filterNotNull()?.toSet() ?: emptySet()
|
||||
|
||||
UpstreamsConfig.Methods(
|
||||
enabled, disabled
|
||||
enabled, disabled
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -282,5 +283,4 @@ class UpstreamsConfigReader(
|
||||
}
|
||||
return options
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ import org.yaml.snakeyaml.nodes.Node
|
||||
import org.yaml.snakeyaml.nodes.ScalarNode
|
||||
import java.io.InputStream
|
||||
import java.io.InputStreamReader
|
||||
import java.util.*
|
||||
import java.util.Locale
|
||||
|
||||
abstract class YamlConfigReader {
|
||||
private val envVariables = EnvVariables()
|
||||
@@ -43,12 +43,12 @@ abstract class YamlConfigReader {
|
||||
return false
|
||||
}
|
||||
return mappingNode.value
|
||||
.stream()
|
||||
.filter { n -> n.keyNode is ScalarNode }
|
||||
.filter { n ->
|
||||
val sn = n.keyNode as ScalarNode
|
||||
key == sn.value
|
||||
}.count() > 0
|
||||
.stream()
|
||||
.filter { n -> n.keyNode is ScalarNode }
|
||||
.filter { n ->
|
||||
val sn = n.keyNode as ScalarNode
|
||||
key == sn.value
|
||||
}.count() > 0
|
||||
}
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
@@ -57,20 +57,20 @@ abstract class YamlConfigReader {
|
||||
return null
|
||||
}
|
||||
return mappingNode.value
|
||||
.stream()
|
||||
.filter { n -> n.keyNode is ScalarNode && type.isAssignableFrom(n.valueNode.javaClass) }
|
||||
.filter { n ->
|
||||
val sn = n.keyNode as ScalarNode
|
||||
key == sn.value
|
||||
}
|
||||
.map { n -> n.valueNode as T }
|
||||
.findFirst().let {
|
||||
if (it.isPresent) {
|
||||
it.get()
|
||||
} else {
|
||||
null
|
||||
}
|
||||
.stream()
|
||||
.filter { n -> n.keyNode is ScalarNode && type.isAssignableFrom(n.valueNode.javaClass) }
|
||||
.filter { n ->
|
||||
val sn = n.keyNode as ScalarNode
|
||||
key == sn.value
|
||||
}
|
||||
.map { n -> n.valueNode as T }
|
||||
.findFirst().let {
|
||||
if (it.isPresent) {
|
||||
it.get()
|
||||
} else {
|
||||
null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected fun getMapping(mappingNode: MappingNode?, key: String): MappingNode? {
|
||||
@@ -89,8 +89,8 @@ abstract class YamlConfigReader {
|
||||
|
||||
protected fun getListOfString(mappingNode: MappingNode?, key: String): List<String>? {
|
||||
return getList<ScalarNode>(mappingNode, key)?.value
|
||||
?.map { it.value }
|
||||
?.map(envVariables::postProcess)
|
||||
?.map { it.value }
|
||||
?.map(envVariables::postProcess)
|
||||
}
|
||||
|
||||
protected fun getValueAsString(mappingNode: MappingNode?, key: String): String? {
|
||||
@@ -130,7 +130,7 @@ abstract class YamlConfigReader {
|
||||
fun getValueAsBytes(mappingNode: MappingNode?, key: String): Int? {
|
||||
return getValueAsString(mappingNode, key)?.let(envVariables::postProcess)?.let {
|
||||
val m = Regex("^(\\d+)(m|mb|k|kb|b)?$").find(it.lowercase().trim())
|
||||
?: throw IllegalArgumentException("Not a data size: ${it}. Example of correct values: '1024', '1kb', '5mb'")
|
||||
?: throw IllegalArgumentException("Not a data size: $it. Example of correct values: '1024', '1kb', '5mb'")
|
||||
val multiplier = m.groups[2]?.let {
|
||||
when (it.value) {
|
||||
"k", "kb" -> 1024
|
||||
@@ -147,9 +147,9 @@ abstract class YamlConfigReader {
|
||||
|
||||
fun getBlockchain(id: String): Chain {
|
||||
return Chain.values().find { chain ->
|
||||
chain.name == id.uppercase(Locale.getDefault())
|
||||
|| chain.chainCode.uppercase(Locale.getDefault()) == id.uppercase(Locale.getDefault())
|
||||
|| chain.id.toString() == id
|
||||
chain.name == id.uppercase(Locale.getDefault()) ||
|
||||
chain.chainCode.uppercase(Locale.getDefault()) == id.uppercase(Locale.getDefault()) ||
|
||||
chain.id.toString() == id
|
||||
} ?: Chain.UNSPECIFIED
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,14 +23,14 @@ import java.math.BigInteger
|
||||
import java.time.Instant
|
||||
|
||||
class BlockContainer(
|
||||
val height: Long,
|
||||
val hash: BlockId,
|
||||
val difficulty: BigInteger,
|
||||
val timestamp: Instant,
|
||||
val full: Boolean,
|
||||
json: ByteArray?,
|
||||
val parsed: Any?,
|
||||
val transactions: List<TxId> = emptyList()
|
||||
val height: Long,
|
||||
val hash: BlockId,
|
||||
val difficulty: BigInteger,
|
||||
val timestamp: Instant,
|
||||
val full: Boolean,
|
||||
json: ByteArray?,
|
||||
val parsed: Any?,
|
||||
val transactions: List<TxId> = emptyList()
|
||||
) : SourceContainer(json, parsed) {
|
||||
|
||||
companion object {
|
||||
@@ -38,14 +38,14 @@ class BlockContainer(
|
||||
fun from(block: BlockJson<*>, raw: ByteArray): BlockContainer {
|
||||
val hasTransactions = block.transactions?.filterIsInstance<TransactionJson>()?.count() ?: 0 > 0
|
||||
return BlockContainer(
|
||||
block.number,
|
||||
BlockId.from(block),
|
||||
block.totalDifficulty,
|
||||
block.timestamp,
|
||||
hasTransactions,
|
||||
raw,
|
||||
block,
|
||||
block.transactions?.map { TxId.from(it.hash) } ?: emptyList()
|
||||
block.number,
|
||||
BlockId.from(block),
|
||||
block.totalDifficulty,
|
||||
block.timestamp,
|
||||
hasTransactions,
|
||||
raw,
|
||||
block,
|
||||
block.transactions?.map { TxId.from(it.hash) } ?: emptyList()
|
||||
)
|
||||
}
|
||||
|
||||
@@ -88,6 +88,4 @@ class BlockContainer(
|
||||
result = 31 * result + hash.hashCode()
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ import io.emeraldpay.etherjar.rpc.json.BlockJson
|
||||
import org.bouncycastle.util.encoders.Hex
|
||||
|
||||
class BlockId(
|
||||
value: ByteArray
|
||||
value: ByteArray
|
||||
) : HashId(value) {
|
||||
|
||||
companion object {
|
||||
@@ -51,6 +51,4 @@ class BlockId(
|
||||
return BlockId(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,14 +18,14 @@ package io.emeraldpay.dshackle.data
|
||||
import org.slf4j.LoggerFactory
|
||||
|
||||
class DefaultContainer<T>(
|
||||
val txId: TxId?,
|
||||
val blockId: BlockId?,
|
||||
val height: Long?,
|
||||
json: ByteArray,
|
||||
parsed: T
|
||||
val txId: TxId?,
|
||||
val blockId: BlockId?,
|
||||
val height: Long?,
|
||||
json: ByteArray,
|
||||
parsed: T
|
||||
) : SourceContainer(json, parsed) {
|
||||
|
||||
companion object {
|
||||
private val log = LoggerFactory.getLogger(DefaultContainer::class.java)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
package io.emeraldpay.dshackle.data
|
||||
|
||||
open class HashId(
|
||||
val value: ByteArray
|
||||
val value: ByteArray
|
||||
) {
|
||||
|
||||
companion object {
|
||||
@@ -56,6 +56,4 @@ open class HashId(
|
||||
override fun hashCode(): Int {
|
||||
return value.contentHashCode()
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ class RawJsonBuilder {
|
||||
buf.write(START)
|
||||
buf.write(COMMA)
|
||||
buf.write(ID_START)
|
||||
buf.write(id.toString().toByteArray());
|
||||
buf.write(id.toString().toByteArray())
|
||||
buf.write(COMMA)
|
||||
buf.write(RESULT_START)
|
||||
buf.write(data)
|
||||
@@ -44,6 +44,4 @@ class RawJsonBuilder {
|
||||
|
||||
return buf.toByteArray()
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,11 +16,9 @@
|
||||
*/
|
||||
package io.emeraldpay.dshackle.data
|
||||
|
||||
import java.lang.ClassCastException
|
||||
|
||||
abstract class SourceContainer(
|
||||
val json: ByteArray?,
|
||||
private val parsed: Any?
|
||||
val json: ByteArray?,
|
||||
private val parsed: Any?
|
||||
) {
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
@@ -34,7 +32,6 @@ abstract class SourceContainer(
|
||||
throw ClassCastException("Cannot cast ${parsed.javaClass} to $clazz")
|
||||
}
|
||||
|
||||
|
||||
override fun equals(other: Any?): Boolean {
|
||||
if (this === other) return true
|
||||
if (other !is SourceContainer) return false
|
||||
@@ -50,4 +47,4 @@ abstract class SourceContainer(
|
||||
override fun hashCode(): Int {
|
||||
return json?.contentHashCode() ?: 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,11 +20,11 @@ import io.emeraldpay.dshackle.Global
|
||||
import io.emeraldpay.etherjar.rpc.json.TransactionJson
|
||||
|
||||
class TxContainer(
|
||||
val height: Long?,
|
||||
val hash: TxId,
|
||||
val blockId: BlockId?,
|
||||
json: ByteArray?,
|
||||
parsed: Any? = null
|
||||
val height: Long?,
|
||||
val hash: TxId,
|
||||
val blockId: BlockId?,
|
||||
json: ByteArray?,
|
||||
parsed: Any? = null
|
||||
) : SourceContainer(json, parsed) {
|
||||
|
||||
companion object {
|
||||
@@ -41,11 +41,11 @@ class TxContainer(
|
||||
|
||||
fun from(tx: TransactionJson, raw: ByteArray): TxContainer {
|
||||
return TxContainer(
|
||||
tx.blockNumber,
|
||||
TxId.from(tx.hash),
|
||||
tx.blockHash?.let { BlockId.from(it) },
|
||||
raw,
|
||||
tx
|
||||
tx.blockNumber,
|
||||
TxId.from(tx.hash),
|
||||
tx.blockHash?.let { BlockId.from(it) },
|
||||
raw,
|
||||
tx
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -70,6 +70,4 @@ class TxContainer(
|
||||
result = 31 * result + hash.hashCode()
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,10 +19,9 @@ package io.emeraldpay.dshackle.data
|
||||
import io.emeraldpay.etherjar.domain.TransactionId
|
||||
import io.emeraldpay.etherjar.rpc.json.TransactionJson
|
||||
import org.bouncycastle.util.encoders.Hex
|
||||
import java.math.BigInteger
|
||||
|
||||
class TxId(
|
||||
value: ByteArray
|
||||
value: ByteArray
|
||||
) : HashId(value) {
|
||||
|
||||
companion object {
|
||||
@@ -47,4 +46,4 @@ class TxId(
|
||||
return TxId(bytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,20 +26,18 @@ import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics
|
||||
import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics
|
||||
import io.micrometer.core.instrument.binder.system.ProcessorMetrics
|
||||
import io.micrometer.core.instrument.config.MeterFilter
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.stereotype.Service
|
||||
import io.micrometer.prometheus.PrometheusConfig
|
||||
|
||||
import io.micrometer.prometheus.PrometheusMeterRegistry
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
import java.io.IOException
|
||||
import java.net.InetSocketAddress
|
||||
import javax.annotation.PostConstruct
|
||||
|
||||
|
||||
@Service
|
||||
class MonitoringSetup(
|
||||
@Autowired private val monitoringConfig: MonitoringConfig
|
||||
@Autowired private val monitoringConfig: MonitoringConfig
|
||||
) {
|
||||
|
||||
companion object {
|
||||
@@ -50,7 +48,7 @@ class MonitoringSetup(
|
||||
fun setup() {
|
||||
val prometheusRegistry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT)
|
||||
Metrics.globalRegistry.add(prometheusRegistry)
|
||||
Metrics.globalRegistry.config().meterFilter(object: MeterFilter {
|
||||
Metrics.globalRegistry.config().meterFilter(object : MeterFilter {
|
||||
override fun map(id: Meter.Id): Meter.Id {
|
||||
if (id.name.startsWith("jvm") || id.name.startsWith("process") || id.name.startsWith("system")) {
|
||||
return id
|
||||
@@ -76,18 +74,24 @@ class MonitoringSetup(
|
||||
// prometheus is a single thread periodic call, no reason to setup anything complex
|
||||
try {
|
||||
log.info("Run Prometheus metrics on ${monitoringConfig.prometheus.host}:${monitoringConfig.prometheus.port}${monitoringConfig.prometheus.path}")
|
||||
val server = HttpServer.create(InetSocketAddress(monitoringConfig.prometheus.host, monitoringConfig.prometheus.port), 0);
|
||||
val server = HttpServer.create(
|
||||
InetSocketAddress(
|
||||
monitoringConfig.prometheus.host,
|
||||
monitoringConfig.prometheus.port
|
||||
),
|
||||
0
|
||||
)
|
||||
server.createContext(monitoringConfig.prometheus.path) { httpExchange ->
|
||||
val response = prometheusRegistry.scrape()
|
||||
httpExchange.sendResponseHeaders(200, response.toByteArray().size.toLong());
|
||||
httpExchange.sendResponseHeaders(200, response.toByteArray().size.toLong())
|
||||
httpExchange.responseBody.use { os ->
|
||||
os.write(response.toByteArray())
|
||||
}
|
||||
}
|
||||
Thread(server::start).start();
|
||||
Thread(server::start).start()
|
||||
} catch (e: IOException) {
|
||||
log.error("Failed to start Prometheus Server", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,14 +15,20 @@
|
||||
*/
|
||||
package io.emeraldpay.dshackle.monitoring.accesslog
|
||||
|
||||
import io.grpc.*
|
||||
import io.grpc.ForwardingServerCall
|
||||
import io.grpc.ForwardingServerCallListener
|
||||
import io.grpc.Metadata
|
||||
import io.grpc.MethodDescriptor
|
||||
import io.grpc.ServerCall
|
||||
import io.grpc.ServerCallHandler
|
||||
import io.grpc.ServerInterceptor
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
class AccessHandlerGrpc(
|
||||
@Autowired private val accessLogWriter: AccessLogWriter
|
||||
@Autowired private val accessLogWriter: AccessLogWriter
|
||||
) : ServerInterceptor {
|
||||
|
||||
companion object {
|
||||
@@ -30,9 +36,10 @@ class AccessHandlerGrpc(
|
||||
}
|
||||
|
||||
override fun <ReqT : Any, RespT : Any> interceptCall(
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>): ServerCall.Listener<ReqT> {
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>
|
||||
): ServerCall.Listener<ReqT> {
|
||||
|
||||
return when (val method = call.methodDescriptor.bareMethodName) {
|
||||
"SubscribeHead" -> processSubscribeHead(call, headers, next)
|
||||
@@ -51,103 +58,109 @@ class AccessHandlerGrpc(
|
||||
}
|
||||
|
||||
private fun <ReqT : Any, RespT : Any, E> process(
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>,
|
||||
builder: EventsBuilder.RequestReply<E, ReqT, RespT>
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>,
|
||||
builder: EventsBuilder.RequestReply<E, ReqT, RespT>
|
||||
): ServerCall.Listener<ReqT> {
|
||||
builder.start(headers, call.attributes)
|
||||
val callWrapper: ServerCall<ReqT, RespT> = StdCallResponse(
|
||||
call, builder, accessLogWriter
|
||||
call, builder, accessLogWriter
|
||||
)
|
||||
return StdCallListener(
|
||||
next.startCall(callWrapper, headers),
|
||||
builder
|
||||
next.startCall(callWrapper, headers),
|
||||
builder
|
||||
)
|
||||
}
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
private fun <ReqT : Any, RespT : Any> processSubscribeHead(
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>
|
||||
): ServerCall.Listener<ReqT> {
|
||||
return process(call, headers, next,
|
||||
EventsBuilder.SubscribeHead() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||
return process(
|
||||
call, headers, next,
|
||||
EventsBuilder.SubscribeHead() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||
)
|
||||
}
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
private fun <ReqT : Any, RespT : Any> processSubscribeBalance(
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>,
|
||||
subscribe: Boolean
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>,
|
||||
subscribe: Boolean
|
||||
): ServerCall.Listener<ReqT> {
|
||||
return process(call, headers, next,
|
||||
EventsBuilder.SubscribeBalance(subscribe) as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||
return process(
|
||||
call, headers, next,
|
||||
EventsBuilder.SubscribeBalance(subscribe) as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||
)
|
||||
}
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
private fun <ReqT : Any, RespT : Any> processSubscribeTxStatus(
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>
|
||||
): ServerCall.Listener<ReqT> {
|
||||
return process(call, headers, next,
|
||||
EventsBuilder.TxStatus() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||
return process(
|
||||
call, headers, next,
|
||||
EventsBuilder.TxStatus() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||
)
|
||||
}
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
private fun <ReqT : Any, RespT : Any> processNativeCall(
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>
|
||||
): ServerCall.Listener<ReqT> {
|
||||
return process(call, headers, next,
|
||||
EventsBuilder.NativeCall() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||
return process(
|
||||
call, headers, next,
|
||||
EventsBuilder.NativeCall() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||
)
|
||||
}
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
private fun <ReqT : Any, RespT : Any> processNativeSubscribe(
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>
|
||||
): ServerCall.Listener<ReqT> {
|
||||
return process(call, headers, next,
|
||||
EventsBuilder.NativeSubscribe() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||
return process(
|
||||
call, headers, next,
|
||||
EventsBuilder.NativeSubscribe() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
private fun <ReqT : Any, RespT : Any> processDescribe(
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>
|
||||
): ServerCall.Listener<ReqT> {
|
||||
return process(call, headers, next,
|
||||
EventsBuilder.Describe() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||
return process(
|
||||
call, headers, next,
|
||||
EventsBuilder.Describe() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||
)
|
||||
}
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
private fun <ReqT : Any, RespT : Any> processStatus(
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>
|
||||
call: ServerCall<ReqT, RespT>,
|
||||
headers: Metadata,
|
||||
next: ServerCallHandler<ReqT, RespT>
|
||||
): ServerCall.Listener<ReqT> {
|
||||
return process(call, headers, next,
|
||||
EventsBuilder.Status() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||
return process(
|
||||
call, headers, next,
|
||||
EventsBuilder.Status() as EventsBuilder.RequestReply<*, ReqT, RespT>
|
||||
)
|
||||
}
|
||||
|
||||
open class StdCallListener<Req, EB : EventsBuilder.RequestReply<*, Req, *>>(
|
||||
val next: ServerCall.Listener<Req>,
|
||||
val builder: EB
|
||||
val next: ServerCall.Listener<Req>,
|
||||
val builder: EB
|
||||
) : ForwardingServerCallListener<Req>() {
|
||||
|
||||
override fun onMessage(message: Req) {
|
||||
@@ -161,9 +174,9 @@ class AccessHandlerGrpc(
|
||||
}
|
||||
|
||||
open class StdCallResponse<ReqT : Any, RespT : Any, EB : EventsBuilder.RequestReply<*, ReqT, RespT>>(
|
||||
val next: ServerCall<ReqT, RespT>,
|
||||
val builder: EB,
|
||||
val accessLogWriter: AccessLogWriter
|
||||
val next: ServerCall<ReqT, RespT>,
|
||||
val builder: EB,
|
||||
val accessLogWriter: AccessLogWriter
|
||||
) : ForwardingServerCall<ReqT, RespT>() {
|
||||
|
||||
override fun getMethodDescriptor(): MethodDescriptor<ReqT, RespT> {
|
||||
@@ -177,9 +190,8 @@ class AccessHandlerGrpc(
|
||||
override fun sendMessage(message: RespT) {
|
||||
super.sendMessage(message)
|
||||
accessLogWriter.submit(
|
||||
builder.onReply(message)!!
|
||||
builder.onReply(message)!!
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,8 +19,8 @@ import kotlin.concurrent.withLock
|
||||
*/
|
||||
@Service
|
||||
class AccessHandlerHttp(
|
||||
@Autowired private val mainConfig: MainConfig,
|
||||
@Autowired accessLogWriter: AccessLogWriter
|
||||
@Autowired private val mainConfig: MainConfig,
|
||||
@Autowired accessLogWriter: AccessLogWriter
|
||||
) {
|
||||
|
||||
companion object {
|
||||
@@ -40,7 +40,7 @@ class AccessHandlerHttp(
|
||||
fun create(req: HttpServerRequest, blockchain: Chain): RequestHandler
|
||||
}
|
||||
|
||||
class NoOpFactory() : HandlerFactory {
|
||||
class NoOpFactory : HandlerFactory {
|
||||
override fun create(req: HttpServerRequest, blockchain: Chain): RequestHandler {
|
||||
return NoOpHandler()
|
||||
}
|
||||
@@ -70,9 +70,9 @@ class AccessHandlerHttp(
|
||||
}
|
||||
|
||||
class StandardHandler(
|
||||
private val accessLogWriter: AccessLogWriter,
|
||||
private val httpRequest: HttpServerRequest,
|
||||
private val blockchain: Chain
|
||||
private val accessLogWriter: AccessLogWriter,
|
||||
private val httpRequest: HttpServerRequest,
|
||||
private val blockchain: Chain
|
||||
) : RequestHandler {
|
||||
|
||||
private var request: BlockchainOuterClass.NativeCallRequest? = null
|
||||
@@ -89,13 +89,13 @@ class AccessHandlerHttp(
|
||||
builder.start(httpRequest)
|
||||
builder.onRequest(request!!)
|
||||
responses
|
||||
.map {
|
||||
builder.onReply(it, Events.Channel.JSONRPC).also { item ->
|
||||
//since for JSON RPC you get a single response then the timestamp of all items included in it must have the same timestamp
|
||||
item.ts = responseTime
|
||||
}
|
||||
.map {
|
||||
builder.onReply(it, Events.Channel.JSONRPC).also { item ->
|
||||
// since for JSON RPC you get a single response then the timestamp of all items included in it must have the same timestamp
|
||||
item.ts = responseTime
|
||||
}
|
||||
.let(accessLogWriter::submit)
|
||||
}
|
||||
.let(accessLogWriter::submit)
|
||||
}
|
||||
|
||||
override fun onRequest(request: BlockchainOuterClass.NativeCallRequest) {
|
||||
@@ -108,4 +108,4 @@ class AccessHandlerHttp(
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,9 @@ import io.emeraldpay.dshackle.config.MainConfig
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Repository
|
||||
import java.io.*
|
||||
import java.io.BufferedOutputStream
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
import java.util.concurrent.ConcurrentLinkedQueue
|
||||
@@ -30,7 +32,7 @@ import javax.annotation.PostConstruct
|
||||
|
||||
@Repository
|
||||
class AccessLogWriter(
|
||||
@Autowired mainConfig: MainConfig
|
||||
@Autowired mainConfig: MainConfig
|
||||
) {
|
||||
|
||||
companion object {
|
||||
@@ -120,5 +122,4 @@ class AccessLogWriter(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonInclude
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import org.slf4j.LoggerFactory
|
||||
import java.time.Instant
|
||||
import java.util.*
|
||||
import java.util.UUID
|
||||
|
||||
class Events {
|
||||
|
||||
@@ -32,140 +32,152 @@ class Events {
|
||||
}
|
||||
|
||||
abstract class Base(
|
||||
val id: UUID,
|
||||
val method: String,
|
||||
val channel: Channel
|
||||
val id: UUID,
|
||||
val method: String,
|
||||
val channel: Channel
|
||||
) {
|
||||
val version = "accesslog/v1beta"
|
||||
var ts = Instant.now()
|
||||
}
|
||||
|
||||
abstract class ChainBase(
|
||||
val blockchain: Chain, method: String, id: UUID, channel: Channel
|
||||
val blockchain: Chain,
|
||||
method: String,
|
||||
id: UUID,
|
||||
channel: Channel
|
||||
) : Base(id, method, channel)
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
class SubscribeHead(
|
||||
blockchain: Chain, id: UUID,
|
||||
// initial request details
|
||||
val request: StreamRequestDetails,
|
||||
// index of the current response
|
||||
val index: Int
|
||||
blockchain: Chain,
|
||||
id: UUID,
|
||||
// initial request details
|
||||
val request: StreamRequestDetails,
|
||||
// index of the current response
|
||||
val index: Int
|
||||
) : ChainBase(blockchain, "SubscribeHead", id, Channel.GRPC)
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
class SubscribeBalance(
|
||||
blockchain: Chain, id: UUID, subscribe: Boolean,
|
||||
// initial request details
|
||||
val request: StreamRequestDetails,
|
||||
val balanceRequest: BalanceRequest,
|
||||
val addressBalance: AddressBalance,
|
||||
// index of the current response
|
||||
val index: Int
|
||||
blockchain: Chain,
|
||||
id: UUID,
|
||||
subscribe: Boolean,
|
||||
// initial request details
|
||||
val request: StreamRequestDetails,
|
||||
val balanceRequest: BalanceRequest,
|
||||
val addressBalance: AddressBalance,
|
||||
// index of the current response
|
||||
val index: Int
|
||||
) : ChainBase(blockchain, if (subscribe) "SubscribeBalance" else "GetBalance", id, Channel.GRPC)
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
class TxStatus(
|
||||
blockchain: Chain, id: UUID,
|
||||
val request: StreamRequestDetails,
|
||||
val txStatusRequest: TxStatusRequest,
|
||||
val txStatus: TxStatusResponse,
|
||||
// index of the current response
|
||||
val index: Int
|
||||
blockchain: Chain,
|
||||
id: UUID,
|
||||
val request: StreamRequestDetails,
|
||||
val txStatusRequest: TxStatusRequest,
|
||||
val txStatus: TxStatusResponse,
|
||||
// index of the current response
|
||||
val index: Int
|
||||
) : ChainBase(blockchain, "SubscribeTxStatus", id, Channel.GRPC)
|
||||
|
||||
data class TxStatusRequest(
|
||||
val txId: String
|
||||
val txId: String
|
||||
)
|
||||
|
||||
data class TxStatusResponse(
|
||||
val confirmations: Int
|
||||
val confirmations: Int
|
||||
)
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
class NativeCall(
|
||||
blockchain: Chain, id: UUID, channel: Channel,
|
||||
blockchain: Chain,
|
||||
id: UUID,
|
||||
channel: Channel,
|
||||
|
||||
// info about the initial request, that may include several native calls
|
||||
val request: StreamRequestDetails,
|
||||
// total native calls passes within the initial request
|
||||
val total: Int,
|
||||
// index of the call specific for the current response
|
||||
val index: Int,
|
||||
val selector: String? = null,
|
||||
val quorum: Long? = null,
|
||||
val minAvailability: String? = null,
|
||||
// info about the initial request, that may include several native calls
|
||||
val request: StreamRequestDetails,
|
||||
// total native calls passes within the initial request
|
||||
val total: Int,
|
||||
// index of the call specific for the current response
|
||||
val index: Int,
|
||||
val selector: String? = null,
|
||||
val quorum: Long? = null,
|
||||
val minAvailability: String? = null,
|
||||
|
||||
val succeed: Boolean,
|
||||
val rpcError: Int? = null,
|
||||
val payloadSizeBytes: Long,
|
||||
val nativeCall: NativeCallItemDetails
|
||||
val succeed: Boolean,
|
||||
val rpcError: Int? = null,
|
||||
val payloadSizeBytes: Long,
|
||||
val nativeCall: NativeCallItemDetails
|
||||
) : ChainBase(blockchain, "NativeCall", id, channel)
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
class NativeSubscribe(
|
||||
blockchain: Chain, id: UUID, channel: Channel,
|
||||
blockchain: Chain,
|
||||
id: UUID,
|
||||
channel: Channel,
|
||||
|
||||
// info about the initial request, that may include several native calls
|
||||
val request: StreamRequestDetails,
|
||||
val payloadSizeBytes: Long,
|
||||
val nativeSubscribe: NativeSubscribeItemDetails
|
||||
// info about the initial request, that may include several native calls
|
||||
val request: StreamRequestDetails,
|
||||
val payloadSizeBytes: Long,
|
||||
val nativeSubscribe: NativeSubscribeItemDetails
|
||||
) : ChainBase(blockchain, "NativeSubscribe", id, channel)
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
class Describe(
|
||||
id: UUID,
|
||||
val request: StreamRequestDetails
|
||||
id: UUID,
|
||||
val request: StreamRequestDetails
|
||||
) : Base(id, "Describe", Channel.GRPC)
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
class Status(
|
||||
blockchain: Chain, id: UUID,
|
||||
val request: StreamRequestDetails
|
||||
blockchain: Chain,
|
||||
id: UUID,
|
||||
val request: StreamRequestDetails
|
||||
) : ChainBase(blockchain, "Status", id, Channel.GRPC)
|
||||
|
||||
data class StreamRequestDetails(
|
||||
val id: UUID,
|
||||
val start: Instant,
|
||||
val remote: Remote
|
||||
val id: UUID,
|
||||
val start: Instant,
|
||||
val remote: Remote
|
||||
)
|
||||
|
||||
data class Remote(
|
||||
val ips: List<String>,
|
||||
val ip: String,
|
||||
val userAgent: String
|
||||
val ips: List<String>,
|
||||
val ip: String,
|
||||
val userAgent: String
|
||||
)
|
||||
|
||||
data class NativeCallItemDetails(
|
||||
val method: String,
|
||||
val id: Int,
|
||||
val payloadSizeBytes: Long
|
||||
val method: String,
|
||||
val id: Int,
|
||||
val payloadSizeBytes: Long
|
||||
)
|
||||
|
||||
data class NativeCallReplyDetails(
|
||||
val id: Int,
|
||||
val succeed: Boolean,
|
||||
val replySizeBytes: Long,
|
||||
val ts: Instant = Instant.now()
|
||||
val id: Int,
|
||||
val succeed: Boolean,
|
||||
val replySizeBytes: Long,
|
||||
val ts: Instant = Instant.now()
|
||||
)
|
||||
|
||||
data class NativeSubscribeItemDetails(
|
||||
val method: String,
|
||||
val payloadSizeBytes: Long
|
||||
val method: String,
|
||||
val payloadSizeBytes: Long
|
||||
)
|
||||
|
||||
data class NativeSubscribeReplyDetails(
|
||||
val replySizeBytes: Long,
|
||||
val ts: Instant = Instant.now()
|
||||
val replySizeBytes: Long,
|
||||
val ts: Instant = Instant.now()
|
||||
)
|
||||
|
||||
data class BalanceRequest(
|
||||
val asset: String,
|
||||
val addressType: String
|
||||
val asset: String,
|
||||
val addressType: String
|
||||
)
|
||||
|
||||
data class AddressBalance(
|
||||
val asset: String,
|
||||
val address: String
|
||||
val asset: String,
|
||||
val address: String
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,8 @@ import reactor.netty.http.server.HttpServerRequest
|
||||
import java.net.InetAddress
|
||||
import java.net.InetSocketAddress
|
||||
import java.time.Instant
|
||||
import java.util.*
|
||||
import java.util.Locale
|
||||
import java.util.UUID
|
||||
|
||||
class EventsBuilder {
|
||||
|
||||
@@ -48,23 +49,23 @@ class EventsBuilder {
|
||||
fun onReply(msg: Resp): E
|
||||
}
|
||||
|
||||
abstract class Base<T>() : StartingHttp2Request, StartingHttp1Request {
|
||||
abstract class Base<T> : StartingHttp2Request, StartingHttp1Request {
|
||||
companion object {
|
||||
private val remoteIpHeaders = listOf(
|
||||
"x-real-ip",
|
||||
"x-forwarded-for"
|
||||
"x-real-ip",
|
||||
"x-forwarded-for"
|
||||
)
|
||||
private val remoteIpKeys = listOf(
|
||||
Metadata.Key.of("x-real-ip", Metadata.ASCII_STRING_MARSHALLER),
|
||||
Metadata.Key.of("x-forwarded-for", Metadata.ASCII_STRING_MARSHALLER)
|
||||
Metadata.Key.of("x-real-ip", Metadata.ASCII_STRING_MARSHALLER),
|
||||
Metadata.Key.of("x-forwarded-for", Metadata.ASCII_STRING_MARSHALLER)
|
||||
)
|
||||
private val invalidCharacters = Regex("[\n\t]+")
|
||||
}
|
||||
|
||||
var requestDetails = Events.StreamRequestDetails(
|
||||
UUID.randomUUID(),
|
||||
Instant.now(),
|
||||
Events.Remote(emptyList(), "", "")
|
||||
UUID.randomUUID(),
|
||||
Instant.now(),
|
||||
Events.Remote(emptyList(), "", "")
|
||||
)
|
||||
|
||||
var chainId: Int = Chain.UNSPECIFIED.id
|
||||
@@ -84,35 +85,37 @@ class EventsBuilder {
|
||||
|
||||
private fun findBestIp(ips: List<InetAddress>): InetAddress? {
|
||||
// check if a real remote address is provided, otherwise use any local address
|
||||
return ips.sortedWith(kotlin.Comparator { a, b ->
|
||||
val aLocal = a.isLoopbackAddress || a.isSiteLocalAddress
|
||||
val bLocal = b.isLoopbackAddress || b.isSiteLocalAddress
|
||||
when {
|
||||
aLocal && bLocal -> 0
|
||||
aLocal -> 1
|
||||
else -> -1
|
||||
return ips.sortedWith(
|
||||
kotlin.Comparator { a, b ->
|
||||
val aLocal = a.isLoopbackAddress || a.isSiteLocalAddress
|
||||
val bLocal = b.isLoopbackAddress || b.isSiteLocalAddress
|
||||
when {
|
||||
aLocal && bLocal -> 0
|
||||
aLocal -> 1
|
||||
else -> -1
|
||||
}
|
||||
}
|
||||
}).firstOrNull()
|
||||
).firstOrNull()
|
||||
}
|
||||
|
||||
private fun clean(s: String): String {
|
||||
return StringUtils.truncate(s, 128)
|
||||
.replace(invalidCharacters, " ")
|
||||
.trim()
|
||||
.replace(invalidCharacters, " ")
|
||||
.trim()
|
||||
}
|
||||
|
||||
protected abstract fun getT(): T
|
||||
|
||||
override fun start(metadata: Metadata, attributes: Attributes) {
|
||||
val userAgent = metadata.get(Metadata.Key.of("user-agent", Metadata.ASCII_STRING_MARSHALLER))
|
||||
?.let(this@Base::clean)
|
||||
?: ""
|
||||
?.let(this@Base::clean)
|
||||
?: ""
|
||||
val ips = ArrayList<InetAddress>()
|
||||
remoteIpKeys.forEach { key ->
|
||||
metadata.get(key)?.let {
|
||||
it.trim().ifEmpty { null }
|
||||
?.let(this@Base::toInetAddress)
|
||||
?.let(ips::add)
|
||||
?.let(this@Base::toInetAddress)
|
||||
?.let(ips::add)
|
||||
}
|
||||
}
|
||||
attributes.get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR)?.let { addr ->
|
||||
@@ -122,24 +125,26 @@ class EventsBuilder {
|
||||
}
|
||||
val ip = findBestIp(ips)?.hostAddress ?: ""
|
||||
this.requestDetails = this.requestDetails
|
||||
.copy(remote = Events.Remote(
|
||||
ips = ips.map { it.hostAddress },
|
||||
ip = ip,
|
||||
userAgent = userAgent
|
||||
))
|
||||
.copy(
|
||||
remote = Events.Remote(
|
||||
ips = ips.map { it.hostAddress },
|
||||
ip = ip,
|
||||
userAgent = userAgent
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
override fun start(request: HttpServerRequest) {
|
||||
val headers = request.requestHeaders()
|
||||
val userAgent = headers.get("user-agent")
|
||||
?.let(this@Base::clean)
|
||||
?: ""
|
||||
?.let(this@Base::clean)
|
||||
?: ""
|
||||
val ips = ArrayList<InetAddress>()
|
||||
remoteIpHeaders.forEach { key ->
|
||||
headers.get(key)?.let {
|
||||
it.trim().ifEmpty { null }
|
||||
?.let(this@Base::toInetAddress)
|
||||
?.let(ips::add)
|
||||
?.let(this@Base::toInetAddress)
|
||||
?.let(ips::add)
|
||||
}
|
||||
}
|
||||
request.remoteAddress()?.let { addr ->
|
||||
@@ -147,11 +152,13 @@ class EventsBuilder {
|
||||
}
|
||||
val ip = findBestIp(ips)?.hostAddress ?: ""
|
||||
this.requestDetails = this.requestDetails
|
||||
.copy(remote = Events.Remote(
|
||||
ips = ips.map { it.hostAddress },
|
||||
ip = ip,
|
||||
userAgent = userAgent
|
||||
))
|
||||
.copy(
|
||||
remote = Events.Remote(
|
||||
ips = ips.map { it.hostAddress },
|
||||
ip = ip,
|
||||
userAgent = userAgent
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
fun withChain(chain: Int): T {
|
||||
@@ -161,9 +168,9 @@ class EventsBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
class SubscribeHead() :
|
||||
Base<SubscribeHead>(),
|
||||
RequestReply<Events.SubscribeHead, Common.Chain, BlockchainOuterClass.ChainHead> {
|
||||
class SubscribeHead :
|
||||
Base<SubscribeHead>(),
|
||||
RequestReply<Events.SubscribeHead, Common.Chain, BlockchainOuterClass.ChainHead> {
|
||||
|
||||
private var index = 0
|
||||
|
||||
@@ -177,14 +184,14 @@ class EventsBuilder {
|
||||
|
||||
override fun onReply(msg: BlockchainOuterClass.ChainHead): Events.SubscribeHead {
|
||||
return Events.SubscribeHead(
|
||||
chain, UUID.randomUUID(), requestDetails, index++
|
||||
chain, UUID.randomUUID(), requestDetails, index++
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
class SubscribeBalance(val subscribe: Boolean) :
|
||||
Base<SubscribeBalance>(),
|
||||
RequestReply<Events.SubscribeBalance, BlockchainOuterClass.BalanceRequest, BlockchainOuterClass.AddressBalance> {
|
||||
Base<SubscribeBalance>(),
|
||||
RequestReply<Events.SubscribeBalance, BlockchainOuterClass.BalanceRequest, BlockchainOuterClass.AddressBalance> {
|
||||
|
||||
private var index = 0
|
||||
private var balanceRequest: Events.BalanceRequest? = null
|
||||
@@ -195,8 +202,8 @@ class EventsBuilder {
|
||||
|
||||
override fun onRequest(msg: BlockchainOuterClass.BalanceRequest) {
|
||||
balanceRequest = Events.BalanceRequest(
|
||||
msg.asset.code.uppercase(Locale.getDefault()),
|
||||
msg.address.addrTypeCase.name
|
||||
msg.asset.code.uppercase(Locale.getDefault()),
|
||||
msg.address.addrTypeCase.name
|
||||
)
|
||||
}
|
||||
|
||||
@@ -207,14 +214,14 @@ class EventsBuilder {
|
||||
val addressBalance = Events.AddressBalance(msg.asset.code, msg.address.address)
|
||||
val chain = Chain.byId(msg.asset.chain.number)
|
||||
return Events.SubscribeBalance(
|
||||
chain, UUID.randomUUID(), subscribe, requestDetails, balanceRequest!!, addressBalance, index++
|
||||
chain, UUID.randomUUID(), subscribe, requestDetails, balanceRequest!!, addressBalance, index++
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
class TxStatus() :
|
||||
Base<TxStatus>(),
|
||||
RequestReply<Events.TxStatus, BlockchainOuterClass.TxStatusRequest, BlockchainOuterClass.TxStatus> {
|
||||
class TxStatus :
|
||||
Base<TxStatus>(),
|
||||
RequestReply<Events.TxStatus, BlockchainOuterClass.TxStatusRequest, BlockchainOuterClass.TxStatus> {
|
||||
private var index = 0
|
||||
private var txStatusRequest: Events.TxStatusRequest? = null
|
||||
|
||||
@@ -225,21 +232,20 @@ class EventsBuilder {
|
||||
|
||||
override fun onReply(msg: BlockchainOuterClass.TxStatus): Events.TxStatus {
|
||||
return Events.TxStatus(
|
||||
chain, UUID.randomUUID(), requestDetails, txStatusRequest!!,
|
||||
Events.TxStatusResponse(msg.confirmations),
|
||||
index++
|
||||
chain, UUID.randomUUID(), requestDetails, txStatusRequest!!,
|
||||
Events.TxStatusResponse(msg.confirmations),
|
||||
index++
|
||||
)
|
||||
}
|
||||
|
||||
override fun getT(): TxStatus {
|
||||
return this
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
class NativeCall :
|
||||
Base<NativeCall>(),
|
||||
RequestReply<Events.NativeCall, BlockchainOuterClass.NativeCallRequest, BlockchainOuterClass.NativeCallReplyItem> {
|
||||
Base<NativeCall>(),
|
||||
RequestReply<Events.NativeCall, BlockchainOuterClass.NativeCallRequest, BlockchainOuterClass.NativeCallReplyItem> {
|
||||
val items = ArrayList<Events.NativeCallItemDetails>()
|
||||
val replies = HashMap<Int, Events.NativeCallReplyDetails>()
|
||||
private var index = 0
|
||||
@@ -252,11 +258,11 @@ class EventsBuilder {
|
||||
withChain(msg.chain.number)
|
||||
msg.itemsList.forEach { item ->
|
||||
this.items.add(
|
||||
Events.NativeCallItemDetails(
|
||||
item.method,
|
||||
item.id,
|
||||
item.payload.size().toLong()
|
||||
)
|
||||
Events.NativeCallItemDetails(
|
||||
item.method,
|
||||
item.id,
|
||||
item.payload.size().toLong()
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -264,38 +270,40 @@ class EventsBuilder {
|
||||
override fun onReply(msg: BlockchainOuterClass.NativeCallReplyItem): Events.NativeCall {
|
||||
val item = items.find { it.id == msg.id }!!
|
||||
return Events.NativeCall(
|
||||
request = requestDetails,
|
||||
total = items.size,
|
||||
index = index++,
|
||||
succeed = msg.succeed,
|
||||
blockchain = chain,
|
||||
nativeCall = item,
|
||||
payloadSizeBytes = item.payloadSizeBytes,
|
||||
id = UUID.randomUUID(),
|
||||
channel = Events.Channel.GRPC
|
||||
request = requestDetails,
|
||||
total = items.size,
|
||||
index = index++,
|
||||
succeed = msg.succeed,
|
||||
blockchain = chain,
|
||||
nativeCall = item,
|
||||
payloadSizeBytes = item.payloadSizeBytes,
|
||||
id = UUID.randomUUID(),
|
||||
channel = Events.Channel.GRPC
|
||||
)
|
||||
}
|
||||
|
||||
fun onReply(reply: io.emeraldpay.dshackle.rpc.NativeCall.CallResult,
|
||||
channel: Events.Channel): Events.NativeCall {
|
||||
fun onReply(
|
||||
reply: io.emeraldpay.dshackle.rpc.NativeCall.CallResult,
|
||||
channel: Events.Channel
|
||||
): Events.NativeCall {
|
||||
val item = items.find { it.id == reply.id }!!
|
||||
return Events.NativeCall(
|
||||
request = requestDetails,
|
||||
total = items.size,
|
||||
index = index++,
|
||||
succeed = !reply.isError(),
|
||||
blockchain = chain,
|
||||
nativeCall = item,
|
||||
payloadSizeBytes = item.payloadSizeBytes,
|
||||
id = UUID.randomUUID(),
|
||||
channel = channel
|
||||
request = requestDetails,
|
||||
total = items.size,
|
||||
index = index++,
|
||||
succeed = !reply.isError(),
|
||||
blockchain = chain,
|
||||
nativeCall = item,
|
||||
payloadSizeBytes = item.payloadSizeBytes,
|
||||
id = UUID.randomUUID(),
|
||||
channel = channel
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
class NativeSubscribe :
|
||||
Base<NativeSubscribe>(),
|
||||
RequestReply<Events.NativeSubscribe, BlockchainOuterClass.NativeSubscribeRequest, BlockchainOuterClass.NativeSubscribeReplyItem> {
|
||||
Base<NativeSubscribe>(),
|
||||
RequestReply<Events.NativeSubscribe, BlockchainOuterClass.NativeSubscribeRequest, BlockchainOuterClass.NativeSubscribeReplyItem> {
|
||||
var item: Events.NativeSubscribeItemDetails? = null
|
||||
val replies = HashMap<Int, Events.NativeSubscribeReplyDetails>()
|
||||
|
||||
@@ -306,26 +314,26 @@ class EventsBuilder {
|
||||
override fun onRequest(msg: BlockchainOuterClass.NativeSubscribeRequest) {
|
||||
withChain(msg.chain.number)
|
||||
this.item = Events.NativeSubscribeItemDetails(
|
||||
msg.method,
|
||||
msg.payload.size().toLong()
|
||||
msg.method,
|
||||
msg.payload.size().toLong()
|
||||
)
|
||||
}
|
||||
|
||||
override fun onReply(msg: BlockchainOuterClass.NativeSubscribeReplyItem): Events.NativeSubscribe {
|
||||
return Events.NativeSubscribe(
|
||||
request = requestDetails,
|
||||
blockchain = chain,
|
||||
nativeSubscribe = item!!,
|
||||
payloadSizeBytes = msg.payload?.size()?.toLong() ?: 0L,
|
||||
id = UUID.randomUUID(),
|
||||
channel = Events.Channel.GRPC
|
||||
request = requestDetails,
|
||||
blockchain = chain,
|
||||
nativeSubscribe = item!!,
|
||||
payloadSizeBytes = msg.payload?.size()?.toLong() ?: 0L,
|
||||
id = UUID.randomUUID(),
|
||||
channel = Events.Channel.GRPC
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
class Describe :
|
||||
Base<Describe>(),
|
||||
RequestReply<Events.Describe, BlockchainOuterClass.DescribeRequest, BlockchainOuterClass.DescribeResponse> {
|
||||
Base<Describe>(),
|
||||
RequestReply<Events.Describe, BlockchainOuterClass.DescribeRequest, BlockchainOuterClass.DescribeResponse> {
|
||||
|
||||
override fun getT(): Describe {
|
||||
return this
|
||||
@@ -336,15 +344,15 @@ class EventsBuilder {
|
||||
|
||||
override fun onReply(msg: BlockchainOuterClass.DescribeResponse): Events.Describe {
|
||||
return Events.Describe(
|
||||
id = UUID.randomUUID(),
|
||||
request = requestDetails
|
||||
id = UUID.randomUUID(),
|
||||
request = requestDetails
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
class Status :
|
||||
Base<Status>(),
|
||||
RequestReply<Events.Status, BlockchainOuterClass.StatusRequest, BlockchainOuterClass.ChainStatus> {
|
||||
Base<Status>(),
|
||||
RequestReply<Events.Status, BlockchainOuterClass.StatusRequest, BlockchainOuterClass.ChainStatus> {
|
||||
override fun getT(): Status {
|
||||
return this
|
||||
}
|
||||
@@ -355,11 +363,10 @@ class EventsBuilder {
|
||||
override fun onReply(msg: BlockchainOuterClass.ChainStatus): Events.Status {
|
||||
val chain = Chain.byId(msg.chainValue)
|
||||
return Events.Status(
|
||||
blockchain = chain,
|
||||
request = requestDetails,
|
||||
id = UUID.randomUUID()
|
||||
blockchain = chain,
|
||||
request = requestDetails,
|
||||
id = UUID.randomUUID()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,10 +23,10 @@ import org.slf4j.LoggerFactory
|
||||
* JSON RPC call to the proxy
|
||||
*/
|
||||
class ProxyCall(
|
||||
/**
|
||||
* Type of the request. The response format depends on it
|
||||
*/
|
||||
val type: RpcType
|
||||
/**
|
||||
* Type of the request. The response format depends on it
|
||||
*/
|
||||
val type: RpcType
|
||||
) {
|
||||
|
||||
companion object {
|
||||
@@ -55,4 +55,4 @@ class ProxyCall(
|
||||
*/
|
||||
BATCH
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,8 +24,8 @@ import io.emeraldpay.dshackle.config.ProxyConfig
|
||||
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp
|
||||
import io.emeraldpay.dshackle.rpc.NativeCall
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import io.emeraldpay.etherjar.rpc.RpcException
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import io.micrometer.core.instrument.Counter
|
||||
import io.micrometer.core.instrument.Metrics
|
||||
import io.micrometer.core.instrument.Timer
|
||||
@@ -38,16 +38,14 @@ import org.slf4j.LoggerFactory
|
||||
import org.springframework.http.HttpHeaders
|
||||
import reactor.core.publisher.Flux
|
||||
import reactor.core.publisher.Mono
|
||||
import reactor.netty.DisposableServer
|
||||
import reactor.netty.http.server.HttpServer
|
||||
import reactor.netty.http.server.HttpServerRequest
|
||||
import reactor.netty.http.server.HttpServerResponse
|
||||
import reactor.netty.http.server.HttpServerRoutes
|
||||
import java.util.*
|
||||
import java.util.EnumMap
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock
|
||||
import java.util.function.BiFunction
|
||||
import kotlin.collections.HashMap
|
||||
import kotlin.concurrent.read
|
||||
import kotlin.concurrent.write
|
||||
|
||||
@@ -55,12 +53,12 @@ import kotlin.concurrent.write
|
||||
* HTTP Proxy Server
|
||||
*/
|
||||
class ProxyServer(
|
||||
private var config: ProxyConfig,
|
||||
private val readRpcJson: ReadRpcJson,
|
||||
private val writeRpcJson: WriteRpcJson,
|
||||
private val nativeCall: NativeCall,
|
||||
private val tlsSetup: TlsSetup,
|
||||
private val accessHandler: AccessHandlerHttp.HandlerFactory
|
||||
private var config: ProxyConfig,
|
||||
private val readRpcJson: ReadRpcJson,
|
||||
private val writeRpcJson: WriteRpcJson,
|
||||
private val nativeCall: NativeCall,
|
||||
private val tlsSetup: TlsSetup,
|
||||
private val accessHandler: AccessHandlerHttp.HandlerFactory
|
||||
) {
|
||||
|
||||
companion object {
|
||||
@@ -101,19 +99,19 @@ class ProxyServer(
|
||||
}
|
||||
log.info("Listening Proxy on ${config.host}:${config.port}")
|
||||
var serverBuilder = HttpServer.create()
|
||||
.doOnChannelInit { _, channel, _ ->
|
||||
channel.pipeline().addFirst(errorHandler)
|
||||
}
|
||||
.host(config.host)
|
||||
.port(config.port)
|
||||
.doOnChannelInit { _, channel, _ ->
|
||||
channel.pipeline().addFirst(errorHandler)
|
||||
}
|
||||
.host(config.host)
|
||||
.port(config.port)
|
||||
|
||||
tlsSetup.setupServer("proxy", config.tls, false)?.let { sslContext ->
|
||||
serverBuilder = serverBuilder.secure { secure -> secure.sslContext(sslContext) }
|
||||
}
|
||||
|
||||
serverBuilder
|
||||
.route(this::setupRoutes)
|
||||
.bindNow()
|
||||
.route(this::setupRoutes)
|
||||
.bindNow()
|
||||
}
|
||||
|
||||
fun setupRoutes(routes: HttpServerRoutes) {
|
||||
@@ -139,26 +137,26 @@ class ProxyServer(
|
||||
}
|
||||
}
|
||||
val request = BlockchainOuterClass.NativeCallRequest.newBuilder()
|
||||
.setChain(Common.ChainRef.forNumber(chain.id))
|
||||
.addAllItems(call.items)
|
||||
.build()
|
||||
.setChain(Common.ChainRef.forNumber(chain.id))
|
||||
.addAllItems(call.items)
|
||||
.build()
|
||||
handler.onRequest(request)
|
||||
val jsons = nativeCall
|
||||
.nativeCallResult(Mono.just(request))
|
||||
.doOnNext {
|
||||
metricById(it.id)?.requestMetric?.increment()
|
||||
.nativeCallResult(Mono.just(request))
|
||||
.doOnNext {
|
||||
metricById(it.id)?.requestMetric?.increment()
|
||||
}
|
||||
.doOnNext {
|
||||
handler.onResponse(it)
|
||||
metricById(it.id)?.callMetric?.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS)
|
||||
}
|
||||
.doOnError {
|
||||
// when error happened the whole flux is stopped and no result is produced, so we should mark all the requests as failed
|
||||
call.items.forEach { item ->
|
||||
requestMetrics.get(chain, item.method).errorMetric.increment()
|
||||
}
|
||||
.doOnNext {
|
||||
handler.onResponse(it)
|
||||
metricById(it.id)?.callMetric?.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS)
|
||||
}
|
||||
.doOnError {
|
||||
//when error happened the whole flux is stopped and no result is produced, so we should mark all the requests as failed
|
||||
call.items.forEach { item ->
|
||||
requestMetrics.get(chain, item.method).errorMetric.increment()
|
||||
}
|
||||
}
|
||||
.transform(writeRpcJson.toJsons(call))
|
||||
}
|
||||
.transform(writeRpcJson.toJsons(call))
|
||||
return if (call.type == ProxyCall.RpcType.SINGLE) {
|
||||
jsons.next()
|
||||
} else {
|
||||
@@ -166,21 +164,25 @@ class ProxyServer(
|
||||
}
|
||||
}
|
||||
|
||||
fun processRequest(chain: Chain, request: Mono<ByteArray>, handler: AccessHandlerHttp.RequestHandler): Flux<ByteBuf> {
|
||||
fun processRequest(
|
||||
chain: Chain,
|
||||
request: Mono<ByteArray>,
|
||||
handler: AccessHandlerHttp.RequestHandler
|
||||
): Flux<ByteBuf> {
|
||||
return request
|
||||
.map(readRpcJson)
|
||||
.flatMapMany { call ->
|
||||
execute(chain, call, handler)
|
||||
}
|
||||
.onErrorResume(RpcException::class.java) { err ->
|
||||
val id = err.details?.let {
|
||||
if (it is JsonRpcResponse.Id) it else JsonRpcResponse.NumberId(-1)
|
||||
} ?: JsonRpcResponse.NumberId(-1)
|
||||
.map(readRpcJson)
|
||||
.flatMapMany { call ->
|
||||
execute(chain, call, handler)
|
||||
}
|
||||
.onErrorResume(RpcException::class.java) { err ->
|
||||
val id = err.details?.let {
|
||||
if (it is JsonRpcResponse.Id) it else JsonRpcResponse.NumberId(-1)
|
||||
} ?: JsonRpcResponse.NumberId(-1)
|
||||
|
||||
val json = JsonRpcResponse.error(err.code, err.rpcMessage, id)
|
||||
Mono.just(Global.objectMapper.writeValueAsString(json))
|
||||
}
|
||||
.map { Unpooled.wrappedBuffer(it.toByteArray()) }
|
||||
val json = JsonRpcResponse.error(err.code, err.rpcMessage, id)
|
||||
Mono.just(Global.objectMapper.writeValueAsString(json))
|
||||
}
|
||||
.map { Unpooled.wrappedBuffer(it.toByteArray()) }
|
||||
}
|
||||
|
||||
fun proxy(routeConfig: ProxyConfig.Route): BiFunction<HttpServerRequest, HttpServerResponse, Publisher<Void>> {
|
||||
@@ -188,13 +190,13 @@ class ProxyServer(
|
||||
// handle access events
|
||||
val eventHandler = accessHandler.create(req, routeConfig.blockchain)
|
||||
val request = req.receive()
|
||||
.aggregate()
|
||||
.asByteArray()
|
||||
.aggregate()
|
||||
.asByteArray()
|
||||
val results = processRequest(routeConfig.blockchain, request, eventHandler)
|
||||
// make sure that the access log handler is closed at the end, so it can render the logs
|
||||
.doFinally { eventHandler.close() }
|
||||
// make sure that the access log handler is closed at the end, so it can render the logs
|
||||
.doFinally { eventHandler.close() }
|
||||
resp.addHeader(HttpHeaders.CONTENT_TYPE, "application/json")
|
||||
.send(results)
|
||||
.send(results)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,25 +253,25 @@ class ProxyServer(
|
||||
|
||||
class RequestMetricsBasic(chain: Chain) : RequestMetrics {
|
||||
override val callMetric = Timer.builder("request.jsonrpc.call")
|
||||
.tags("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tags("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
override val errorMetric = Counter.builder("request.jsonrpc.err")
|
||||
.tags("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tags("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
override val requestMetric = Counter.builder("request.jsonrpc.request.total")
|
||||
.tags("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tags("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
}
|
||||
|
||||
class RequestMetricsWithMethod(chain: Chain, method: String) : RequestMetrics {
|
||||
override val callMetric = Timer.builder("request.jsonrpc.call")
|
||||
.tags("chain", chain.chainCode, "method", method)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tags("chain", chain.chainCode, "method", method)
|
||||
.register(Metrics.globalRegistry)
|
||||
override val errorMetric = Counter.builder("request.jsonrpc.err")
|
||||
.tags("chain", chain.chainCode, "method", method)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tags("chain", chain.chainCode, "method", method)
|
||||
.register(Metrics.globalRegistry)
|
||||
override val requestMetric = Counter.builder("request.jsonrpc.request.total")
|
||||
.tags("chain", chain.chainCode, "method", method)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tags("chain", chain.chainCode, "method", method)
|
||||
.register(Metrics.globalRegistry)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,19 +25,16 @@ import io.emeraldpay.etherjar.rpc.RpcException
|
||||
import io.emeraldpay.etherjar.rpc.RpcResponseError
|
||||
import io.emeraldpay.etherjar.rpc.json.RequestJson
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
import java.io.IOException
|
||||
import java.util.*
|
||||
import java.util.function.Function
|
||||
import java.util.stream.Collectors
|
||||
|
||||
|
||||
/**
|
||||
* Reader for JSON RPC request
|
||||
*/
|
||||
@Service
|
||||
open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
|
||||
open class ReadRpcJson : Function<ByteArray, ProxyCall> {
|
||||
|
||||
companion object {
|
||||
private val log = LoggerFactory.getLogger(ReadRpcJson::class.java)
|
||||
@@ -55,21 +52,37 @@ open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
|
||||
val id = json["id"]
|
||||
if ("2.0" != json["jsonrpc"]) {
|
||||
if (json["jsonrpc"] == null) {
|
||||
throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "jsonrpc version is not set", id?.let { JsonRpcResponse.Id.from(it) })
|
||||
throw RpcException(
|
||||
RpcResponseError.CODE_INVALID_REQUEST,
|
||||
"jsonrpc version is not set",
|
||||
id?.let { JsonRpcResponse.Id.from(it) }
|
||||
)
|
||||
}
|
||||
throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "Unsupported JSON RPC version: " + json["jsonrpc"].toString(), id?.let { JsonRpcResponse.Id.from(it) })
|
||||
throw RpcException(
|
||||
RpcResponseError.CODE_INVALID_REQUEST,
|
||||
"Unsupported JSON RPC version: " + json["jsonrpc"].toString(),
|
||||
id?.let { JsonRpcResponse.Id.from(it) }
|
||||
)
|
||||
}
|
||||
if (!(json["method"] != null && json["method"] is String)) {
|
||||
throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "Method is not set", id?.let { JsonRpcResponse.Id.from(it) })
|
||||
throw RpcException(
|
||||
RpcResponseError.CODE_INVALID_REQUEST,
|
||||
"Method is not set",
|
||||
id?.let { JsonRpcResponse.Id.from(it) }
|
||||
)
|
||||
}
|
||||
if (json.containsKey("params") && json["params"] !is List<*>) {
|
||||
throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "Params must be an array", id?.let { JsonRpcResponse.Id.from(it) })
|
||||
throw RpcException(
|
||||
RpcResponseError.CODE_INVALID_REQUEST,
|
||||
"Params must be an array",
|
||||
id?.let { JsonRpcResponse.Id.from(it) }
|
||||
)
|
||||
}
|
||||
RequestJson<Any>(
|
||||
json["method"].toString(),
|
||||
//params MAY be omitted
|
||||
(json["params"] ?: emptyList<Any>()) as List<*>,
|
||||
id
|
||||
json["method"].toString(),
|
||||
// params MAY be omitted
|
||||
(json["params"] ?: emptyList<Any>()) as List<*>,
|
||||
id
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -81,7 +94,7 @@ open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
|
||||
fun getStartOfJson(buf: ByteArray): Byte {
|
||||
val count = buf.size
|
||||
var i = 0
|
||||
//if cannot find anything in the first 255 bytes, just consider it as invalid
|
||||
// if cannot find anything in the first 255 bytes, just consider it as invalid
|
||||
while (i < 256 && i < count) {
|
||||
if (buf[i] != spaces[0] && buf[i] != spaces[1] && buf[i] != spaces[2]) {
|
||||
return buf[i]
|
||||
@@ -127,17 +140,17 @@ open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
|
||||
// our internal ids for calls
|
||||
var seq = 0
|
||||
val batch = list.stream()
|
||||
.map<RequestJson<Any>>(jsonExtractor)
|
||||
.map { json ->
|
||||
val id = seq++
|
||||
context.ids[id] = json.id
|
||||
BlockchainOuterClass.NativeCallItem.newBuilder()
|
||||
.setId(id)
|
||||
.setMethod(json.method)
|
||||
.setPayload(ByteString.copyFrom(objectMapper.writeValueAsBytes(json.params)))
|
||||
.build()
|
||||
}
|
||||
.collect(Collectors.toList())
|
||||
.map<RequestJson<Any>>(jsonExtractor)
|
||||
.map { json ->
|
||||
val id = seq++
|
||||
context.ids[id] = json.id
|
||||
BlockchainOuterClass.NativeCallItem.newBuilder()
|
||||
.setId(id)
|
||||
.setMethod(json.method)
|
||||
.setPayload(ByteString.copyFrom(objectMapper.writeValueAsBytes(json.params)))
|
||||
.build()
|
||||
}
|
||||
.collect(Collectors.toList())
|
||||
context.items.addAll(batch)
|
||||
return context
|
||||
} catch (e: RpcException) {
|
||||
@@ -147,5 +160,4 @@ open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
|
||||
throw RpcException(RpcResponseError.CODE_INVALID_JSON, e.message)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
package io.emeraldpay.dshackle.proxy
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||
import io.emeraldpay.dshackle.Global
|
||||
import io.emeraldpay.dshackle.rpc.NativeCall
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
||||
@@ -31,7 +30,7 @@ import java.util.function.Function
|
||||
* Writer for JSON RPC requests
|
||||
*/
|
||||
@Service
|
||||
open class WriteRpcJson() {
|
||||
open class WriteRpcJson {
|
||||
|
||||
companion object {
|
||||
private val log = LoggerFactory.getLogger(WriteRpcJson::class.java)
|
||||
@@ -45,35 +44,35 @@ open class WriteRpcJson() {
|
||||
open fun toJsons(call: ProxyCall): Function<Flux<NativeCall.CallResult>, Flux<String>> {
|
||||
return Function { flux ->
|
||||
flux
|
||||
.flatMap { response ->
|
||||
if (!call.ids.containsKey(response.id)) {
|
||||
log.warn("ID wasn't requested: ${response.id}")
|
||||
return@flatMap Flux.empty<String>()
|
||||
}
|
||||
val json = toJson(call, response)
|
||||
if (json == null) {
|
||||
Flux.empty<String>()
|
||||
} else {
|
||||
Flux.just(json)
|
||||
}
|
||||
.flatMap { response ->
|
||||
if (!call.ids.containsKey(response.id)) {
|
||||
log.warn("ID wasn't requested: ${response.id}")
|
||||
return@flatMap Flux.empty<String>()
|
||||
}
|
||||
.onErrorResume { t ->
|
||||
if (t is NativeCall.CallFailure) {
|
||||
Mono.just(toJson(call, t)!!)
|
||||
} else {
|
||||
Mono.empty()
|
||||
}
|
||||
val json = toJson(call, response)
|
||||
if (json == null) {
|
||||
Flux.empty<String>()
|
||||
} else {
|
||||
Flux.just(json)
|
||||
}
|
||||
.onErrorContinue { t, _ ->
|
||||
log.warn("Failed to convert to JSON", t)
|
||||
}
|
||||
.onErrorResume { t ->
|
||||
if (t is NativeCall.CallFailure) {
|
||||
Mono.just(toJson(call, t)!!)
|
||||
} else {
|
||||
Mono.empty()
|
||||
}
|
||||
}
|
||||
.onErrorContinue { t, _ ->
|
||||
log.warn("Failed to convert to JSON", t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
open fun toJson(call: ProxyCall, response: NativeCall.CallResult): String? {
|
||||
val id = call.ids[response.id]?.let {
|
||||
JsonRpcResponse.Id.from(it)
|
||||
} ?: return null;
|
||||
} ?: return null
|
||||
val json = if (response.isError()) {
|
||||
val error = response.error!!
|
||||
error.upstreamError?.let { upstreamError ->
|
||||
@@ -86,7 +85,7 @@ open class WriteRpcJson() {
|
||||
}
|
||||
|
||||
fun toJson(call: ProxyCall, error: NativeCall.CallFailure): String? {
|
||||
val id = call.ids[error.id] ?: return null;
|
||||
val id = call.ids[error.id] ?: return null
|
||||
val json = JsonRpcResponse.error(-32003, error.reason.message ?: "", JsonRpcResponse.Id.from(id))
|
||||
return objectMapper.writeValueAsString(json)
|
||||
}
|
||||
@@ -97,18 +96,18 @@ open class WriteRpcJson() {
|
||||
fun asArray(): Function<Flux<String>, Flux<String>> {
|
||||
return Function { flux ->
|
||||
val body = flux.zipWith(Flux.concat(Mono.just(false), Flux.just(true).repeat()))
|
||||
.map {
|
||||
if (it.t2) {
|
||||
"," + it.t1
|
||||
} else {
|
||||
it.t1
|
||||
}
|
||||
.map {
|
||||
if (it.t2) {
|
||||
"," + it.t1
|
||||
} else {
|
||||
it.t1
|
||||
}
|
||||
}
|
||||
Flux.concat(
|
||||
Mono.just("["),
|
||||
body,
|
||||
Mono.just("]")
|
||||
Mono.just("["),
|
||||
body,
|
||||
Mono.just("]")
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,9 +20,8 @@ import io.emeraldpay.dshackle.upstream.Head
|
||||
import io.emeraldpay.dshackle.upstream.Upstream
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
||||
import io.emeraldpay.etherjar.rpc.RpcException
|
||||
|
||||
open class AlwaysQuorum: CallQuorum {
|
||||
open class AlwaysQuorum : CallQuorum {
|
||||
|
||||
private var resolved = false
|
||||
private var result: ByteArray? = null
|
||||
@@ -60,4 +59,4 @@ open class AlwaysQuorum: CallQuorum {
|
||||
override fun toString(): String {
|
||||
return "Quorum: Accept Any"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ import io.emeraldpay.dshackle.upstream.Head
|
||||
import io.emeraldpay.dshackle.upstream.Upstream
|
||||
|
||||
open class BroadcastQuorum(
|
||||
val quorum: Int = 3
|
||||
val quorum: Int = 3
|
||||
) : CallQuorum, ValueAwareQuorum<String>(String::class.java) {
|
||||
|
||||
private var result: ByteArray? = null
|
||||
@@ -61,4 +61,4 @@ open class BroadcastQuorum(
|
||||
override fun toString(): String {
|
||||
return "Quorum: Broadcast to $quorum upstreams"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,10 +20,6 @@ import io.emeraldpay.dshackle.upstream.Head
|
||||
import io.emeraldpay.dshackle.upstream.Upstream
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
||||
import io.emeraldpay.etherjar.domain.TransactionId
|
||||
import io.emeraldpay.etherjar.rpc.RpcException
|
||||
import io.emeraldpay.etherjar.rpc.json.BlockJson
|
||||
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
|
||||
import reactor.util.function.Tuple2
|
||||
import java.util.function.BiFunction
|
||||
import java.util.function.Predicate
|
||||
@@ -54,4 +50,4 @@ interface CallQuorum {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,15 +16,11 @@
|
||||
*/
|
||||
package io.emeraldpay.dshackle.quorum
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import io.emeraldpay.dshackle.upstream.Head
|
||||
import io.emeraldpay.dshackle.upstream.Upstream
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
||||
import io.emeraldpay.etherjar.rpc.JacksonRpcConverter
|
||||
import io.emeraldpay.etherjar.rpc.RpcException
|
||||
|
||||
open class NonEmptyQuorum(
|
||||
val maxTries: Int = 3
|
||||
val maxTries: Int = 3
|
||||
) : CallQuorum, ValueAwareQuorum<Any>(Any::class.java) {
|
||||
|
||||
private var result: ByteArray? = null
|
||||
@@ -59,4 +55,4 @@ open class NonEmptyQuorum(
|
||||
override fun toString(): String {
|
||||
return "Quorum: Accept Non Error Result"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,17 +16,14 @@
|
||||
*/
|
||||
package io.emeraldpay.dshackle.quorum
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import io.emeraldpay.dshackle.upstream.Head
|
||||
import io.emeraldpay.dshackle.upstream.Upstream
|
||||
import io.emeraldpay.etherjar.hex.HexQuantity
|
||||
import io.emeraldpay.etherjar.rpc.JacksonRpcConverter
|
||||
import io.emeraldpay.etherjar.rpc.RpcException
|
||||
import java.util.concurrent.locks.ReentrantLock
|
||||
import kotlin.concurrent.withLock
|
||||
|
||||
open class NonceQuorum(
|
||||
val tries: Int = 3
|
||||
val tries: Int = 3
|
||||
) : CallQuorum, ValueAwareQuorum<String>(String::class.java) {
|
||||
|
||||
private val lock = ReentrantLock()
|
||||
@@ -74,4 +71,4 @@ open class NonceQuorum(
|
||||
override fun toString(): String {
|
||||
return "Quorum: Confirm with $tries upstreams"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ import io.emeraldpay.dshackle.upstream.Head
|
||||
import io.emeraldpay.dshackle.upstream.Upstream
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
||||
import io.emeraldpay.etherjar.rpc.RpcException
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
|
||||
/**
|
||||
@@ -28,7 +27,7 @@ import java.util.concurrent.atomic.AtomicReference
|
||||
*
|
||||
* NOTE: NativeCall checks the quorums and applies a HeightSelector if NotLaggingQuorum is enabled for a call
|
||||
*/
|
||||
class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum {
|
||||
class NotLaggingQuorum(val maxLag: Long = 0) : CallQuorum {
|
||||
|
||||
private val result: AtomicReference<ByteArray> = AtomicReference()
|
||||
private val failed = AtomicReference(false)
|
||||
@@ -73,4 +72,4 @@ class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum {
|
||||
override fun toString(): String {
|
||||
return "Quorum: late <= $maxLag blocks"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,4 +35,4 @@ interface QuorumReaderFactory {
|
||||
return QuorumRpcReader(apis, quorum)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,8 +30,8 @@ import reactor.util.function.Tuples
|
||||
* Makes request with applying Quorum
|
||||
*/
|
||||
class QuorumRpcReader(
|
||||
private val apis: ApiSource,
|
||||
private val quorum: CallQuorum
|
||||
private val apis: ApiSource,
|
||||
private val quorum: CallQuorum
|
||||
) : Reader<JsonRpcRequest, QuorumRpcReader.Result> {
|
||||
|
||||
companion object {
|
||||
@@ -58,8 +58,8 @@ class QuorumRpcReader(
|
||||
val defaultResult: Mono<Result> = Mono.just(quorum).flatMap { q ->
|
||||
if (q.isFailed()) {
|
||||
Mono.error<Result>(
|
||||
q.getError()?.asException(JsonRpcResponse.NumberId(1))
|
||||
?: RpcException(-32000, "Unknown Upstream error")
|
||||
q.getError()?.asException(JsonRpcResponse.NumberId(1))
|
||||
?: RpcException(-32000, "Unknown Upstream error")
|
||||
)
|
||||
} else {
|
||||
log.warn("Did not get any result from upstream. Method [${key.method}] using [$q]")
|
||||
@@ -68,72 +68,71 @@ class QuorumRpcReader(
|
||||
}
|
||||
|
||||
return Flux.from(apis)
|
||||
.takeUntil {
|
||||
quorum.isFailed() || quorum.isResolved()
|
||||
}
|
||||
.flatMap { api ->
|
||||
api.getApi()
|
||||
.read(key)
|
||||
.flatMap { response ->
|
||||
response.requireResult()
|
||||
.onErrorResume { err ->
|
||||
if (err is RpcException || err is JsonRpcException) {
|
||||
// on error notify quorum, it may use error message or other details
|
||||
val cleanErr: JsonRpcException = when (err) {
|
||||
is RpcException -> JsonRpcException.from(err)
|
||||
is JsonRpcException -> err
|
||||
else -> throw IllegalStateException("Cannot convert from exception", err)
|
||||
}
|
||||
quorum.record(cleanErr, api)
|
||||
// it it's failed after that, then we don't need more calls, stop api source
|
||||
if (quorum.isFailed()) {
|
||||
apis.resolve()
|
||||
} else {
|
||||
apis.request(1)
|
||||
}
|
||||
} else {
|
||||
log.warn("Result processing error", err)
|
||||
}
|
||||
Mono.empty()
|
||||
}
|
||||
.takeUntil {
|
||||
quorum.isFailed() || quorum.isResolved()
|
||||
}
|
||||
.flatMap { api ->
|
||||
api.getApi()
|
||||
.read(key)
|
||||
.flatMap { response ->
|
||||
response.requireResult()
|
||||
.onErrorResume { err ->
|
||||
if (err is RpcException || err is JsonRpcException) {
|
||||
// on error notify quorum, it may use error message or other details
|
||||
val cleanErr: JsonRpcException = when (err) {
|
||||
is RpcException -> JsonRpcException.from(err)
|
||||
is JsonRpcException -> err
|
||||
else -> throw IllegalStateException("Cannot convert from exception", err)
|
||||
}
|
||||
quorum.record(cleanErr, api)
|
||||
// it it's failed after that, then we don't need more calls, stop api source
|
||||
if (quorum.isFailed()) {
|
||||
apis.resolve()
|
||||
} else {
|
||||
apis.request(1)
|
||||
}
|
||||
} else {
|
||||
log.warn("Result processing error", err)
|
||||
}
|
||||
Mono.empty()
|
||||
}
|
||||
.map { Tuples.of(it, api) }
|
||||
}
|
||||
.retryWhen(retrySpec)
|
||||
// record all correct responses until quorum reached
|
||||
.reduce(quorum, { res, a ->
|
||||
if (res.record(a.t1, a.t2)) {
|
||||
apis.resolve()
|
||||
} else {
|
||||
apis.request(1)
|
||||
}
|
||||
res
|
||||
})
|
||||
// if last call resulted in error it's still possible that request was resolved correctly. i.e. for BroadcastQuorum
|
||||
.onErrorResume { err ->
|
||||
if (quorum.isResolved()) {
|
||||
Mono.just(quorum)
|
||||
} else {
|
||||
Mono.error(err)
|
||||
}
|
||||
.map { Tuples.of(it, api) }
|
||||
}
|
||||
.retryWhen(retrySpec)
|
||||
// record all correct responses until quorum reached
|
||||
.reduce(quorum, { res, a ->
|
||||
if (res.record(a.t1, a.t2)) {
|
||||
apis.resolve()
|
||||
} else {
|
||||
apis.request(1)
|
||||
}
|
||||
.doOnNext {
|
||||
if (!it.isResolved() && !it.isFailed()) {
|
||||
log.debug("No quorum for ${key.method} using [${quorum}]. Error: ${it.getError()?.message ?: ""}")
|
||||
}
|
||||
res
|
||||
})
|
||||
// if last call resulted in error it's still possible that request was resolved correctly. i.e. for BroadcastQuorum
|
||||
.onErrorResume { err ->
|
||||
if (quorum.isResolved()) {
|
||||
Mono.just(quorum)
|
||||
} else {
|
||||
Mono.error(err)
|
||||
}
|
||||
// return nothing if not resolved
|
||||
.filter { it.isResolved() }
|
||||
.map {
|
||||
// TODO find actual quorum number
|
||||
QuorumRpcReader.Result(it.getResult()!!, 1)
|
||||
}
|
||||
.doOnNext {
|
||||
if (!it.isResolved() && !it.isFailed()) {
|
||||
log.debug("No quorum for ${key.method} using [$quorum]. Error: ${it.getError()?.message ?: ""}")
|
||||
}
|
||||
.switchIfEmpty(defaultResult)
|
||||
}
|
||||
// return nothing if not resolved
|
||||
.filter { it.isResolved() }
|
||||
.map {
|
||||
// TODO find actual quorum number
|
||||
QuorumRpcReader.Result(it.getResult()!!, 1)
|
||||
}
|
||||
.switchIfEmpty(defaultResult)
|
||||
}
|
||||
|
||||
|
||||
class Result(
|
||||
val value: ByteArray,
|
||||
val quorum: Int
|
||||
val value: ByteArray,
|
||||
val quorum: Int
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,18 +16,16 @@
|
||||
*/
|
||||
package io.emeraldpay.dshackle.quorum
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import io.emeraldpay.dshackle.Global
|
||||
import io.emeraldpay.dshackle.upstream.Upstream
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
||||
import io.emeraldpay.etherjar.rpc.JacksonRpcConverter
|
||||
import io.emeraldpay.etherjar.rpc.RpcException
|
||||
import org.slf4j.LoggerFactory
|
||||
|
||||
abstract class ValueAwareQuorum<T>(
|
||||
val clazz: Class<T>
|
||||
): CallQuorum {
|
||||
val clazz: Class<T>
|
||||
) : CallQuorum {
|
||||
|
||||
private val log = LoggerFactory.getLogger(ValueAwareQuorum::class.java)
|
||||
private var rpcError: JsonRpcError? = null
|
||||
@@ -45,7 +43,7 @@ abstract class ValueAwareQuorum<T>(
|
||||
} catch (e: Exception) {
|
||||
recordError(response, e.message, upstream)
|
||||
}
|
||||
return isResolved();
|
||||
return isResolved()
|
||||
}
|
||||
|
||||
override fun record(error: JsonRpcException, upstream: Upstream) {
|
||||
@@ -60,4 +58,4 @@ abstract class ValueAwareQuorum<T>(
|
||||
override fun getError(): JsonRpcError? {
|
||||
return rpcError
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,15 +20,14 @@ import io.emeraldpay.dshackle.Defaults
|
||||
import org.slf4j.LoggerFactory
|
||||
import reactor.core.publisher.Flux
|
||||
import reactor.core.publisher.Mono
|
||||
import java.time.Duration
|
||||
|
||||
/**
|
||||
* Composition of multiple readers.
|
||||
* Reader returns first value returned by any of the source readers by checking one by one until one of them returns a non-empty result.
|
||||
*/
|
||||
class CompoundReader<K, D>(
|
||||
private vararg val readers: Reader<K, D>
|
||||
): Reader<K, D> {
|
||||
private vararg val readers: Reader<K, D>
|
||||
) : Reader<K, D> {
|
||||
|
||||
companion object {
|
||||
private val log = LoggerFactory.getLogger(CompoundReader::class.java)
|
||||
@@ -39,13 +38,12 @@ class CompoundReader<K, D>(
|
||||
return Mono.empty()
|
||||
}
|
||||
return Flux.fromIterable(readers.asIterable())
|
||||
.flatMap({ rdr ->
|
||||
rdr.read(key)
|
||||
.timeout(Defaults.timeoutInternal, Mono.empty())
|
||||
.doOnError { t -> log.warn("Failed to read from $rdr", t) }
|
||||
.onErrorResume { Mono.empty() }
|
||||
}, 1)
|
||||
.next()
|
||||
.flatMap({ rdr ->
|
||||
rdr.read(key)
|
||||
.timeout(Defaults.timeoutInternal, Mono.empty())
|
||||
.doOnError { t -> log.warn("Failed to read from $rdr", t) }
|
||||
.onErrorResume { Mono.empty() }
|
||||
}, 1)
|
||||
.next()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,9 +18,9 @@ package io.emeraldpay.dshackle.reader
|
||||
|
||||
import reactor.core.publisher.Mono
|
||||
|
||||
class EmptyReader<K, D>: Reader<K, D> {
|
||||
class EmptyReader<K, D> : Reader<K, D> {
|
||||
|
||||
override fun read(key: K): Mono<D> {
|
||||
return Mono.empty()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,5 +21,4 @@ import reactor.core.publisher.Mono
|
||||
interface Reader<in K, D> {
|
||||
|
||||
fun read(key: K): Mono<D>
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,22 +22,21 @@ import java.util.function.Function
|
||||
* Reader wrapper that maps the input key from ne value to another (ex. convert from Long to String)
|
||||
*/
|
||||
class RekeyingReader<K, K1, D>(
|
||||
/**
|
||||
* Mapping between original Key and Key supported by the reader
|
||||
*/
|
||||
private val rekey: Function<K, K1>,
|
||||
/**
|
||||
* Actual reader
|
||||
*/
|
||||
private val reader: Reader<K1, D>
|
||||
/**
|
||||
* Mapping between original Key and Key supported by the reader
|
||||
*/
|
||||
private val rekey: Function<K, K1>,
|
||||
/**
|
||||
* Actual reader
|
||||
*/
|
||||
private val reader: Reader<K1, D>
|
||||
) : Reader<K, D> {
|
||||
|
||||
override fun read(key: K): Mono<D> {
|
||||
return Mono.just(key)
|
||||
.map(rekey)
|
||||
.flatMap {
|
||||
reader.read(it)
|
||||
}
|
||||
.map(rekey)
|
||||
.flatMap {
|
||||
reader.read(it)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,8 +25,8 @@ import reactor.core.publisher.Mono
|
||||
* Reader that requests data through upstream RPC using provided JSON RPC request builder
|
||||
*/
|
||||
class RpcReader<T>(
|
||||
private val up: Multistream,
|
||||
private val paramsBuilder: (T) -> JsonRpcRequest
|
||||
private val up: Multistream,
|
||||
private val paramsBuilder: (T) -> JsonRpcRequest
|
||||
) : Reader<T, ByteArray> {
|
||||
|
||||
companion object {
|
||||
@@ -45,11 +45,10 @@ class RpcReader<T>(
|
||||
|
||||
override fun read(key: T): Mono<ByteArray> {
|
||||
return up.getDirectApi(Selector.empty)
|
||||
.flatMap { rdr ->
|
||||
rdr.read(paramsBuilder(key)).flatMap {
|
||||
it.requireResult()
|
||||
}
|
||||
.flatMap { rdr ->
|
||||
rdr.read(paramsBuilder(key)).flatMap {
|
||||
it.requireResult()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,18 +22,17 @@ import java.util.function.Function
|
||||
* Reader wrapper that transforms output of the reader to a different format
|
||||
*/
|
||||
class TransformingReader<K, D0, D>(
|
||||
/**
|
||||
* Actual reader
|
||||
*/
|
||||
private val reader: Reader<K, D0>,
|
||||
/**
|
||||
* Result transformation
|
||||
*/
|
||||
private val transformer: Function<in D0, out D>
|
||||
/**
|
||||
* Actual reader
|
||||
*/
|
||||
private val reader: Reader<K, D0>,
|
||||
/**
|
||||
* Result transformation
|
||||
*/
|
||||
private val transformer: Function<in D0, out D>
|
||||
) : Reader<K, D> {
|
||||
|
||||
override fun read(key: K): Mono<D> {
|
||||
return reader.read(key).map(transformer)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,44 +31,45 @@ import org.springframework.context.annotation.DependsOn
|
||||
import org.springframework.stereotype.Service
|
||||
import reactor.core.publisher.Flux
|
||||
import reactor.core.publisher.Mono
|
||||
import java.util.*
|
||||
import java.util.Locale
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
@Service @DependsOn("monitoringSetup")
|
||||
@Service
|
||||
@DependsOn("monitoringSetup")
|
||||
class BlockchainRpc(
|
||||
@Autowired private val nativeCall: NativeCall,
|
||||
@Autowired private val nativeSubscribe: NativeSubscribe,
|
||||
@Autowired private val streamHead: StreamHead,
|
||||
@Autowired private val trackTx: List<TrackTx>,
|
||||
@Autowired private val trackAddress: List<TrackAddress>,
|
||||
@Autowired private val describe: Describe,
|
||||
@Autowired private val subscribeStatus: SubscribeStatus
|
||||
): ReactorBlockchainGrpc.BlockchainImplBase() {
|
||||
@Autowired private val nativeCall: NativeCall,
|
||||
@Autowired private val nativeSubscribe: NativeSubscribe,
|
||||
@Autowired private val streamHead: StreamHead,
|
||||
@Autowired private val trackTx: List<TrackTx>,
|
||||
@Autowired private val trackAddress: List<TrackAddress>,
|
||||
@Autowired private val describe: Describe,
|
||||
@Autowired private val subscribeStatus: SubscribeStatus
|
||||
) : ReactorBlockchainGrpc.BlockchainImplBase() {
|
||||
|
||||
private val log = LoggerFactory.getLogger(BlockchainRpc::class.java)
|
||||
|
||||
private val describeMetric = Counter.builder("request.grpc.request")
|
||||
.tag("type", "describe")
|
||||
.tag("chain", "NA")
|
||||
.register(Metrics.globalRegistry)
|
||||
.tag("type", "describe")
|
||||
.tag("chain", "NA")
|
||||
.register(Metrics.globalRegistry)
|
||||
private val subscribeStatusMetric = Counter.builder("request.grpc.request")
|
||||
.tag("type", "subscribeStatus")
|
||||
.tag("chain", "NA")
|
||||
.register(Metrics.globalRegistry)
|
||||
.tag("type", "subscribeStatus")
|
||||
.tag("chain", "NA")
|
||||
.register(Metrics.globalRegistry)
|
||||
private val errorMetric = Counter.builder("request.grpc.err")
|
||||
.register(Metrics.globalRegistry)
|
||||
.register(Metrics.globalRegistry)
|
||||
private val chainMetrics = ChainValue { chain -> RequestMetrics(chain) }
|
||||
|
||||
override fun nativeCall(request: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<BlockchainOuterClass.NativeCallReplyItem> {
|
||||
var startTime = 0L
|
||||
var metrics: RequestMetrics? = null
|
||||
return nativeCall.nativeCall(
|
||||
request
|
||||
.doOnNext {
|
||||
metrics = chainMetrics.get(it.chain)
|
||||
metrics!!.nativeCallMetric.increment()
|
||||
startTime = System.currentTimeMillis()
|
||||
}
|
||||
request
|
||||
.doOnNext {
|
||||
metrics = chainMetrics.get(it.chain)
|
||||
metrics!!.nativeCallMetric.increment()
|
||||
startTime = System.currentTimeMillis()
|
||||
}
|
||||
).doOnNext {
|
||||
metrics?.nativeCallRespMetric?.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS)
|
||||
}.doOnError { errorMetric.increment() }
|
||||
@@ -77,11 +78,11 @@ class BlockchainRpc(
|
||||
override fun nativeSubscribe(request: Mono<BlockchainOuterClass.NativeSubscribeRequest>): Flux<BlockchainOuterClass.NativeSubscribeReplyItem> {
|
||||
var metrics: RequestMetrics? = null
|
||||
return nativeSubscribe.nativeSubscribe(
|
||||
request
|
||||
.doOnNext {
|
||||
metrics = chainMetrics.get(it.chain)
|
||||
metrics!!.nativeSubscribeMetric.increment()
|
||||
}
|
||||
request
|
||||
.doOnNext {
|
||||
metrics = chainMetrics.get(it.chain)
|
||||
metrics!!.nativeSubscribeMetric.increment()
|
||||
}
|
||||
).doOnNext {
|
||||
metrics?.nativeSubscribeRespMetric?.increment()
|
||||
}.doOnError { errorMetric.increment() }
|
||||
@@ -89,8 +90,8 @@ class BlockchainRpc(
|
||||
|
||||
override fun subscribeHead(request: Mono<Common.Chain>): Flux<BlockchainOuterClass.ChainHead> {
|
||||
return streamHead.add(
|
||||
request
|
||||
.doOnNext { chainMetrics.get(it.type).subscribeHeadMetric.increment() }
|
||||
request
|
||||
.doOnNext { chainMetrics.get(it.type).subscribeHeadMetric.increment() }
|
||||
).doOnError { errorMetric.increment() }
|
||||
}
|
||||
|
||||
@@ -102,8 +103,8 @@ class BlockchainRpc(
|
||||
try {
|
||||
trackTx.find { it.isSupported(chain) }?.let { track ->
|
||||
track.subscribe(request)
|
||||
.doOnNext { metrics.subscribeHeadRespMetric.increment() }
|
||||
.doOnError { errorMetric.increment() }
|
||||
.doOnNext { metrics.subscribeHeadRespMetric.increment() }
|
||||
.doOnError { errorMetric.increment() }
|
||||
} ?: Flux.error(SilentException.UnsupportedBlockchain(chain))
|
||||
} catch (t: Throwable) {
|
||||
log.error("Internal error during Tx Subscription", t)
|
||||
@@ -122,12 +123,12 @@ class BlockchainRpc(
|
||||
try {
|
||||
trackAddress.find { it.isSupported(chain, asset) }?.let { track ->
|
||||
track.subscribe(request)
|
||||
.doOnNext { metrics.subscribeBalanceRespMetric.increment() }
|
||||
.doOnError { errorMetric.increment() }
|
||||
.doOnNext { metrics.subscribeBalanceRespMetric.increment() }
|
||||
.doOnError { errorMetric.increment() }
|
||||
} ?: Flux.error<BlockchainOuterClass.AddressBalance>(SilentException.UnsupportedBlockchain(chain))
|
||||
.doOnSubscribe {
|
||||
log.error("Balance for $chain:$asset is not supported")
|
||||
}
|
||||
.doOnSubscribe {
|
||||
log.error("Balance for $chain:$asset is not supported")
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
log.error("Internal error during Balance Subscription", t)
|
||||
errorMetric.increment()
|
||||
@@ -146,13 +147,16 @@ class BlockchainRpc(
|
||||
try {
|
||||
trackAddress.find { it.isSupported(chain, asset) }?.let { track ->
|
||||
track.getBalance(request)
|
||||
.doOnNext {
|
||||
metrics.getBalanceRespMetric.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS)
|
||||
}
|
||||
.doOnNext {
|
||||
metrics.getBalanceRespMetric.record(
|
||||
System.currentTimeMillis() - startTime,
|
||||
TimeUnit.MILLISECONDS
|
||||
)
|
||||
}
|
||||
} ?: Flux.error<BlockchainOuterClass.AddressBalance>(SilentException.UnsupportedBlockchain(chain))
|
||||
.doOnSubscribe {
|
||||
log.error("Balance for $chain:$asset is not supported")
|
||||
}
|
||||
.doOnSubscribe {
|
||||
log.error("Balance for $chain:$asset is not supported")
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
log.error("Internal error during Balance Request", t)
|
||||
errorMetric.increment()
|
||||
@@ -164,61 +168,61 @@ class BlockchainRpc(
|
||||
override fun describe(request: Mono<BlockchainOuterClass.DescribeRequest>): Mono<BlockchainOuterClass.DescribeResponse> {
|
||||
describeMetric.increment()
|
||||
return describe.describe(request)
|
||||
.doOnError { errorMetric.increment() }
|
||||
.doOnError { errorMetric.increment() }
|
||||
}
|
||||
|
||||
override fun subscribeStatus(request: Mono<BlockchainOuterClass.StatusRequest>): Flux<BlockchainOuterClass.ChainStatus> {
|
||||
subscribeStatusMetric.increment()
|
||||
return subscribeStatus.subscribeStatus(request)
|
||||
.doOnError { errorMetric.increment() }
|
||||
.doOnError { errorMetric.increment() }
|
||||
}
|
||||
|
||||
class RequestMetrics(chain: Chain) {
|
||||
val nativeCallMetric = Counter.builder("request.grpc.request")
|
||||
.tag("type", "nativeCall")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tag("type", "nativeCall")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
val nativeCallRespMetric = Timer.builder("request.grpc.response")
|
||||
.tag("type", "nativeCall")
|
||||
.tag("chain", chain.chainCode)
|
||||
.publishPercentileHistogram()
|
||||
.register(Metrics.globalRegistry)
|
||||
.tag("type", "nativeCall")
|
||||
.tag("chain", chain.chainCode)
|
||||
.publishPercentileHistogram()
|
||||
.register(Metrics.globalRegistry)
|
||||
val nativeSubscribeMetric = Counter.builder("request.grpc.request")
|
||||
.tag("type", "nativeSubscribe")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tag("type", "nativeSubscribe")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
val nativeSubscribeRespMetric = Counter.builder("request.grpc.response")
|
||||
.tag("type", "nativeSubscribe")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tag("type", "nativeSubscribe")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
val subscribeHeadMetric = Counter.builder("request.grpc.request")
|
||||
.tag("type", "subscribeHead")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tag("type", "subscribeHead")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
val subscribeHeadRespMetric = Counter.builder("request.grpc.reply")
|
||||
.tag("type", "subscribeHead")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tag("type", "subscribeHead")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
val subscribeTxMetric = Counter.builder("request.grpc.request")
|
||||
.tag("type", "subscribeTx")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tag("type", "subscribeTx")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
val subscribeBalanceMetric = Counter.builder("request.grpc.request")
|
||||
.tag("type", "subscribeBalance")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tag("type", "subscribeBalance")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
val subscribeBalanceRespMetric = Counter.builder("request.grpc.reply")
|
||||
.tag("type", "subscribeBalance")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tag("type", "subscribeBalance")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
val getBalanceMetric = Counter.builder("request.grpc.request")
|
||||
.tag("type", "getBalance")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
.tag("type", "getBalance")
|
||||
.tag("chain", chain.chainCode)
|
||||
.register(Metrics.globalRegistry)
|
||||
val getBalanceRespMetric = Timer.builder("request.grpc.response")
|
||||
.tag("type", "getBalance")
|
||||
.tag("chain", chain.chainCode)
|
||||
.publishPercentileHistogram()
|
||||
.register(Metrics.globalRegistry)
|
||||
.tag("type", "getBalance")
|
||||
.tag("chain", chain.chainCode)
|
||||
.publishPercentileHistogram()
|
||||
.register(Metrics.globalRegistry)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,15 +19,17 @@ package io.emeraldpay.dshackle.rpc
|
||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||
import io.emeraldpay.api.proto.Common
|
||||
import io.emeraldpay.dshackle.startup.QuorumForLabels
|
||||
import io.emeraldpay.dshackle.upstream.*
|
||||
import io.emeraldpay.dshackle.upstream.Capability
|
||||
import io.emeraldpay.dshackle.upstream.DefaultUpstream
|
||||
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
import reactor.core.publisher.Mono
|
||||
|
||||
@Service
|
||||
class Describe(
|
||||
@Autowired private val multistreamHolder: MultistreamHolder,
|
||||
@Autowired private val subscribeStatus: SubscribeStatus
|
||||
@Autowired private val multistreamHolder: MultistreamHolder,
|
||||
@Autowired private val subscribeStatus: SubscribeStatus
|
||||
) {
|
||||
|
||||
fun describe(requestMono: Mono<BlockchainOuterClass.DescribeRequest>): Mono<BlockchainOuterClass.DescribeResponse> {
|
||||
@@ -39,9 +41,9 @@ class Describe(
|
||||
val targets = chainUpstreams.getMethods().getSupportedMethods()
|
||||
val capabilities: MutableSet<Capability> = mutableSetOf()
|
||||
val chainDescription = BlockchainOuterClass.DescribeChain.newBuilder()
|
||||
.setChain(Common.ChainRef.forNumber(chain.id))
|
||||
.addAllSupportedMethods(targets)
|
||||
.setStatus(status)
|
||||
.setChain(Common.ChainRef.forNumber(chain.id))
|
||||
.addAllSupportedMethods(targets)
|
||||
.setStatus(status)
|
||||
chainUpstreams.getAll().let { ups ->
|
||||
ups.forEach { up ->
|
||||
val nodes = QuorumForLabels()
|
||||
@@ -50,25 +52,27 @@ class Describe(
|
||||
}
|
||||
nodes.getAll().forEach { node ->
|
||||
val nodeDetails = BlockchainOuterClass.NodeDetails.newBuilder()
|
||||
.setQuorum(node.quorum)
|
||||
.addAllLabels(node.labels.entries.map { label ->
|
||||
.setQuorum(node.quorum)
|
||||
.addAllLabels(
|
||||
node.labels.entries.map { label ->
|
||||
BlockchainOuterClass.Label.newBuilder()
|
||||
.setName(label.key)
|
||||
.setValue(label.value)
|
||||
.build()
|
||||
})
|
||||
.setName(label.key)
|
||||
.setValue(label.value)
|
||||
.build()
|
||||
}
|
||||
)
|
||||
chainDescription.addNodes(nodeDetails)
|
||||
}
|
||||
capabilities.addAll(up.getCapabilities())
|
||||
}
|
||||
}
|
||||
chainDescription.addAllCapabilities(
|
||||
capabilities.map {
|
||||
when (it) {
|
||||
Capability.RPC -> BlockchainOuterClass.Capabilities.CAP_CALLS
|
||||
Capability.BALANCE -> BlockchainOuterClass.Capabilities.CAP_BALANCE
|
||||
}
|
||||
capabilities.map {
|
||||
when (it) {
|
||||
Capability.RPC -> BlockchainOuterClass.Capabilities.CAP_CALLS
|
||||
Capability.BALANCE -> BlockchainOuterClass.Capabilities.CAP_BALANCE
|
||||
}
|
||||
}
|
||||
)
|
||||
resp.addChains(chainDescription.build())
|
||||
}
|
||||
@@ -76,5 +80,4 @@ class Describe(
|
||||
resp.build()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,12 +17,11 @@ class EthereumAddresses {
|
||||
Flux.just(Address.from(addresses.addressSingle.address))
|
||||
Common.AnyAddress.AddrTypeCase.ADDRESS_MULTI ->
|
||||
Flux.fromIterable(addresses.addressMulti.addressesList)
|
||||
.map { Address.from(it.address) }
|
||||
.map { Address.from(it.address) }
|
||||
else -> {
|
||||
log.error("Unsupported address type: ${addresses.addrTypeCase}")
|
||||
Flux.empty()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,33 +21,35 @@ import com.google.protobuf.ByteString
|
||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||
import io.emeraldpay.dshackle.Global
|
||||
import io.emeraldpay.dshackle.SilentException
|
||||
import io.emeraldpay.dshackle.upstream.*
|
||||
import io.emeraldpay.dshackle.quorum.AlwaysQuorum
|
||||
import io.emeraldpay.dshackle.quorum.CallQuorum
|
||||
import io.emeraldpay.dshackle.quorum.NotLaggingQuorum
|
||||
import io.emeraldpay.dshackle.quorum.QuorumReaderFactory
|
||||
import io.emeraldpay.dshackle.upstream.ApiSource
|
||||
import io.emeraldpay.dshackle.upstream.Multistream
|
||||
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
||||
import io.emeraldpay.dshackle.upstream.Selector
|
||||
import io.emeraldpay.dshackle.upstream.calls.EthereumCallSelector
|
||||
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import io.emeraldpay.etherjar.rpc.RpcException
|
||||
import io.emeraldpay.etherjar.rpc.RpcResponseError
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
import reactor.core.publisher.*
|
||||
import reactor.core.publisher.Flux
|
||||
import reactor.core.publisher.Mono
|
||||
import reactor.kotlin.core.publisher.toMono
|
||||
import java.lang.Exception
|
||||
import java.util.*
|
||||
import java.util.EnumMap
|
||||
|
||||
@Service
|
||||
open class NativeCall(
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
) {
|
||||
|
||||
private val log = LoggerFactory.getLogger(NativeCall::class.java)
|
||||
@@ -69,19 +71,19 @@ open class NativeCall(
|
||||
|
||||
open fun nativeCall(requestMono: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<BlockchainOuterClass.NativeCallReplyItem> {
|
||||
return nativeCallResult(requestMono)
|
||||
.map(this::buildResponse)
|
||||
.onErrorResume(this::processException)
|
||||
.map(this::buildResponse)
|
||||
.onErrorResume(this::processException)
|
||||
}
|
||||
|
||||
open fun nativeCallResult(requestMono: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<CallResult> {
|
||||
return requestMono.flatMapMany(this::prepareCall)
|
||||
.map(this::parseParams)
|
||||
.parallel()
|
||||
.flatMap {
|
||||
this.fetch(it)
|
||||
.doOnError { e -> log.warn("Error during native call: ${e.message}") }
|
||||
}
|
||||
.sequential()
|
||||
.map(this::parseParams)
|
||||
.parallel()
|
||||
.flatMap {
|
||||
this.fetch(it)
|
||||
.doOnError { e -> log.warn("Error during native call: ${e.message}") }
|
||||
}
|
||||
.sequential()
|
||||
}
|
||||
|
||||
fun parseParams(it: CallContext<RawCallDetails>): CallContext<ParsedCallDetails> {
|
||||
@@ -91,14 +93,14 @@ open class NativeCall(
|
||||
|
||||
fun buildResponse(it: CallResult): BlockchainOuterClass.NativeCallReplyItem {
|
||||
val result = BlockchainOuterClass.NativeCallReplyItem.newBuilder()
|
||||
.setSucceed(!it.isError())
|
||||
.setId(it.id)
|
||||
.setSucceed(!it.isError())
|
||||
.setId(it.id)
|
||||
if (it.isError()) {
|
||||
it.error?.let { error ->
|
||||
result.setErrorMessage(error.message)
|
||||
}
|
||||
} else {
|
||||
result.setPayload(ByteString.copyFrom(it.result))
|
||||
result.payload = ByteString.copyFrom(it.result)
|
||||
}
|
||||
|
||||
return result.build()
|
||||
@@ -112,11 +114,11 @@ open class NativeCall(
|
||||
0
|
||||
}
|
||||
return BlockchainOuterClass.NativeCallReplyItem.newBuilder()
|
||||
.setSucceed(false)
|
||||
.setErrorMessage(it?.message ?: "Internal error")
|
||||
.setId(id)
|
||||
.build()
|
||||
.toMono()
|
||||
.setSucceed(false)
|
||||
.setErrorMessage(it?.message ?: "Internal error")
|
||||
.setId(id)
|
||||
.build()
|
||||
.toMono()
|
||||
}
|
||||
|
||||
fun prepareCall(request: BlockchainOuterClass.NativeCallRequest): Flux<CallContext<RawCallDetails>> {
|
||||
@@ -130,31 +132,35 @@ open class NativeCall(
|
||||
}
|
||||
|
||||
val upstream = multistreamHolder.getUpstream(chain)
|
||||
?: return Flux.error(CallFailure(0, SilentException.UnsupportedBlockchain(chain)))
|
||||
?: return Flux.error(CallFailure(0, SilentException.UnsupportedBlockchain(chain)))
|
||||
|
||||
return prepareCall(request, upstream)
|
||||
}
|
||||
|
||||
fun prepareCall(request: BlockchainOuterClass.NativeCallRequest, upstream: Multistream): Flux<CallContext<RawCallDetails>> {
|
||||
fun prepareCall(
|
||||
request: BlockchainOuterClass.NativeCallRequest,
|
||||
upstream: Multistream
|
||||
): Flux<CallContext<RawCallDetails>> {
|
||||
val chain = Chain.byId(request.chainValue)
|
||||
return Flux.fromIterable(request.itemsList).flatMap {
|
||||
val method = it.method
|
||||
val params = it.payload.toStringUtf8()
|
||||
|
||||
// for ethereum the actual block needed for the call may be specified in the call parameters
|
||||
val callSpecificMatcher: Mono<Selector.Matcher> = if (BlockchainType.from(upstream.chain) == BlockchainType.ETHEREUM) {
|
||||
ethereumCallSelectors[chain]?.getMatcher(method, params, upstream.getHead())
|
||||
} else {
|
||||
null
|
||||
} ?: Mono.empty()
|
||||
val callSpecificMatcher: Mono<Selector.Matcher> =
|
||||
if (BlockchainType.from(upstream.chain) == BlockchainType.ETHEREUM) {
|
||||
ethereumCallSelectors[chain]?.getMatcher(method, params, upstream.getHead())
|
||||
} else {
|
||||
null
|
||||
} ?: Mono.empty()
|
||||
|
||||
callSpecificMatcher.defaultIfEmpty(Selector.empty).map { csm ->
|
||||
val matcher = Selector.Builder()
|
||||
.withMatcher(csm)
|
||||
.forMethod(method)
|
||||
.forLabels(Selector.convertToMatcher(request.selector))
|
||||
.withMatcher(csm)
|
||||
.forMethod(method)
|
||||
.forLabels(Selector.convertToMatcher(request.selector))
|
||||
|
||||
val callQuorum = upstream.getMethods().getQuorumFor(method) ?: AlwaysQuorum() // can be null in tests
|
||||
val callQuorum = upstream.getMethods().getQuorumFor(method) // can be null in tests
|
||||
callQuorum.init(upstream.getHead())
|
||||
|
||||
// for NotLaggingQuorum it makes sense to select compatible upstreams before the call
|
||||
@@ -172,22 +178,22 @@ open class NativeCall(
|
||||
|
||||
fun fetch(ctx: CallContext<ParsedCallDetails>): Mono<CallResult> {
|
||||
return ctx.upstream.getRoutedApi(ctx.matcher)
|
||||
.flatMap { api ->
|
||||
api.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
|
||||
.flatMap(JsonRpcResponse::requireResult)
|
||||
.map {
|
||||
CallResult.ok(ctx.id, it)
|
||||
}
|
||||
}.switchIfEmpty(
|
||||
Mono.just(ctx).flatMap(this::executeOnRemote)
|
||||
)
|
||||
.onErrorResume {
|
||||
if (it is CallFailure) {
|
||||
Mono.just(CallResult.fail(it.id, it.reason))
|
||||
} else {
|
||||
Mono.just(CallResult.fail(ctx.id, it))
|
||||
.flatMap { api ->
|
||||
api.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
|
||||
.flatMap(JsonRpcResponse::requireResult)
|
||||
.map {
|
||||
CallResult.ok(ctx.id, it)
|
||||
}
|
||||
}.switchIfEmpty(
|
||||
Mono.just(ctx).flatMap(this::executeOnRemote)
|
||||
)
|
||||
.onErrorResume {
|
||||
if (it is CallFailure) {
|
||||
Mono.just(CallResult.fail(it.id, it.reason))
|
||||
} else {
|
||||
Mono.just(CallResult.fail(ctx.id, it))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun executeOnRemote(ctx: CallContext<ParsedCallDetails>): Mono<CallResult> {
|
||||
@@ -196,21 +202,21 @@ open class NativeCall(
|
||||
}
|
||||
val reader = quorumReaderFactory.create(ctx.getApis(), ctx.callQuorum)
|
||||
return reader
|
||||
.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
|
||||
.map {
|
||||
CallResult(ctx.id, it.value, null)
|
||||
.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
|
||||
.map {
|
||||
CallResult(ctx.id, it.value, null)
|
||||
}
|
||||
.onErrorResume { t ->
|
||||
val failure = if (t is CallFailure) {
|
||||
CallResult.fail(t.id, t.reason)
|
||||
} else {
|
||||
CallResult.fail(ctx.id, t)
|
||||
}
|
||||
.onErrorResume { t ->
|
||||
val failure = if (t is CallFailure) {
|
||||
CallResult.fail(t.id, t.reason)
|
||||
} else {
|
||||
CallResult.fail(ctx.id, t)
|
||||
}
|
||||
Mono.just(failure)
|
||||
}
|
||||
.switchIfEmpty(
|
||||
Mono.just(CallResult.fail(ctx.id, 1, "No response or no available upstream for ${ctx.payload.method}"))
|
||||
)
|
||||
Mono.just(failure)
|
||||
}
|
||||
.switchIfEmpty(
|
||||
Mono.just(CallResult.fail(ctx.id, 1, "No response or no available upstream for ${ctx.payload.method}"))
|
||||
)
|
||||
}
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
@@ -222,11 +228,13 @@ open class NativeCall(
|
||||
return req as List<Any>
|
||||
}
|
||||
|
||||
open class CallContext<T>(val id: Int,
|
||||
val upstream: Multistream,
|
||||
val matcher: Selector.Matcher,
|
||||
val callQuorum: CallQuorum,
|
||||
val payload: T) {
|
||||
open class CallContext<T>(
|
||||
val id: Int,
|
||||
val upstream: Multistream,
|
||||
val matcher: Selector.Matcher,
|
||||
val callQuorum: CallQuorum,
|
||||
val payload: T
|
||||
) {
|
||||
fun <X> withPayload(payload: X): CallContext<X> {
|
||||
return CallContext(id, upstream, matcher, callQuorum, payload)
|
||||
}
|
||||
@@ -273,4 +281,4 @@ open class NativeCall(
|
||||
|
||||
class RawCallDetails(val method: String, val params: String)
|
||||
class ParsedCallDetails(val method: String, val params: List<Any>)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ import reactor.core.publisher.Mono
|
||||
|
||||
@Service
|
||||
class NativeSubscribe(
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
) {
|
||||
|
||||
companion object {
|
||||
@@ -45,9 +45,9 @@ class NativeSubscribe(
|
||||
|
||||
fun nativeSubscribe(request: Mono<BlockchainOuterClass.NativeSubscribeRequest>): Flux<BlockchainOuterClass.NativeSubscribeReplyItem> {
|
||||
return request
|
||||
.flatMapMany(this@NativeSubscribe::start)
|
||||
.map(this@NativeSubscribe::convertToProto)
|
||||
.onErrorMap(this@NativeSubscribe::convertToStatus)
|
||||
.flatMapMany(this@NativeSubscribe::start)
|
||||
.map(this@NativeSubscribe::convertToProto)
|
||||
.onErrorMap(this@NativeSubscribe::convertToStatus)
|
||||
}
|
||||
|
||||
fun start(it: BlockchainOuterClass.NativeSubscribeRequest): Publisher<out Any> {
|
||||
@@ -68,15 +68,15 @@ class NativeSubscribe(
|
||||
|
||||
fun convertToStatus(t: Throwable) = when (t) {
|
||||
is SilentException.UnsupportedBlockchain -> StatusException(
|
||||
Status.UNAVAILABLE.withDescription("BLOCKCHAIN UNAVAILABLE: ${t.blockchainId}")
|
||||
Status.UNAVAILABLE.withDescription("BLOCKCHAIN UNAVAILABLE: ${t.blockchainId}")
|
||||
)
|
||||
is UnsupportedOperationException -> StatusException(
|
||||
Status.UNIMPLEMENTED.withDescription(t.message)
|
||||
Status.UNIMPLEMENTED.withDescription(t.message)
|
||||
)
|
||||
else -> {
|
||||
log.warn("Unhandled error", t)
|
||||
StatusException(
|
||||
Status.INTERNAL.withDescription(t.message)
|
||||
Status.INTERNAL.withDescription(t.message)
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -84,15 +84,14 @@ class NativeSubscribe(
|
||||
fun subscribe(chain: Chain, method: String, params: Any?): Flux<out Any> {
|
||||
val up = multistreamHolder.getUpstream(chain) ?: return Flux.error(SilentException.UnsupportedBlockchain(chain))
|
||||
return (up as EthereumMultistream)
|
||||
.getSubscribe()
|
||||
.subscribe(method, params)
|
||||
.getSubscribe()
|
||||
.subscribe(method, params)
|
||||
}
|
||||
|
||||
fun convertToProto(value: Any): BlockchainOuterClass.NativeSubscribeReplyItem {
|
||||
val result = objectMapper.writeValueAsBytes(value)
|
||||
return BlockchainOuterClass.NativeSubscribeReplyItem.newBuilder()
|
||||
.setPayload(ByteString.copyFrom(result))
|
||||
.build()
|
||||
.setPayload(ByteString.copyFrom(result))
|
||||
.build()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ import reactor.core.publisher.Mono
|
||||
|
||||
@Service
|
||||
class StreamHead(
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
) {
|
||||
|
||||
private val log = LoggerFactory.getLogger(StreamHead::class.java)
|
||||
@@ -40,24 +40,23 @@ class StreamHead(
|
||||
Chain.byId(request.type.number)
|
||||
}.flatMapMany { chain ->
|
||||
val up = multistreamHolder.getUpstream(chain)
|
||||
?: return@flatMapMany Flux.error<BlockchainOuterClass.ChainHead>(Exception("Unavailable chain: $chain"))
|
||||
?: return@flatMapMany Flux.error<BlockchainOuterClass.ChainHead>(Exception("Unavailable chain: $chain"))
|
||||
up.getHead()
|
||||
.getFlux()
|
||||
.map { asProto(chain, it!!) }
|
||||
.onErrorContinue { t, _ ->
|
||||
log.warn("Head subscription error: ${t.message}")
|
||||
}
|
||||
.getFlux()
|
||||
.map { asProto(chain, it!!) }
|
||||
.onErrorContinue { t, _ ->
|
||||
log.warn("Head subscription error: ${t.message}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun asProto(chain: Chain, block: BlockContainer): BlockchainOuterClass.ChainHead {
|
||||
return BlockchainOuterClass.ChainHead.newBuilder()
|
||||
.setChainValue(chain.id)
|
||||
.setHeight(block.height)
|
||||
.setTimestamp(block.timestamp.toEpochMilli())
|
||||
.setWeight(ByteString.copyFrom(block.difficulty.toByteArray()))
|
||||
.setBlockId(block.hash.toHex())
|
||||
.build()
|
||||
.setChainValue(chain.id)
|
||||
.setHeight(block.height)
|
||||
.setTimestamp(block.timestamp.toEpochMilli())
|
||||
.setWeight(ByteString.copyFrom(block.difficulty.toByteArray()))
|
||||
.setBlockId(block.hash.toHex())
|
||||
.build()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,9 @@ package io.emeraldpay.dshackle.rpc
|
||||
|
||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||
import io.emeraldpay.api.proto.Common
|
||||
import io.emeraldpay.dshackle.upstream.*
|
||||
import io.emeraldpay.dshackle.upstream.Multistream
|
||||
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
||||
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
@@ -27,7 +29,7 @@ import reactor.core.publisher.Mono
|
||||
|
||||
@Service
|
||||
class SubscribeStatus(
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
) {
|
||||
|
||||
fun subscribeStatus(requestMono: Mono<BlockchainOuterClass.StatusRequest>): Flux<BlockchainOuterClass.ChainStatus> {
|
||||
@@ -52,10 +54,10 @@ class SubscribeStatus(
|
||||
|
||||
fun chainUnavailable(chain: Chain): BlockchainOuterClass.ChainStatus {
|
||||
return BlockchainOuterClass.ChainStatus.newBuilder()
|
||||
.setAvailability(BlockchainOuterClass.AvailabilityEnum.AVAIL_UNAVAILABLE)
|
||||
.setChain(Common.ChainRef.forNumber(chain.id))
|
||||
.setQuorum(0)
|
||||
.build()
|
||||
.setAvailability(BlockchainOuterClass.AvailabilityEnum.AVAIL_UNAVAILABLE)
|
||||
.setChain(Common.ChainRef.forNumber(chain.id))
|
||||
.setQuorum(0)
|
||||
.build()
|
||||
}
|
||||
|
||||
fun chainStatus(chain: Chain, available: UpstreamAvailability, ups: Multistream): BlockchainOuterClass.ChainStatus {
|
||||
@@ -67,12 +69,11 @@ class SubscribeStatus(
|
||||
0
|
||||
}
|
||||
return BlockchainOuterClass.ChainStatus.newBuilder()
|
||||
.setAvailability(BlockchainOuterClass.AvailabilityEnum.forNumber(available.grpcId))
|
||||
.setChain(Common.ChainRef.forNumber(chain.id))
|
||||
.setQuorum(quorum)
|
||||
.build()
|
||||
.setAvailability(BlockchainOuterClass.AvailabilityEnum.forNumber(available.grpcId))
|
||||
.setChain(Common.ChainRef.forNumber(chain.id))
|
||||
.setQuorum(quorum)
|
||||
.build()
|
||||
}
|
||||
|
||||
class ChainSubscription(val chain: Chain, val up: Multistream, val avail: UpstreamAvailability)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,6 @@ package io.emeraldpay.dshackle.rpc
|
||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import reactor.core.publisher.Flux
|
||||
import reactor.core.publisher.Mono
|
||||
|
||||
/**
|
||||
* Base interface to tracking balance on a single blockchain
|
||||
@@ -28,5 +27,4 @@ interface TrackAddress {
|
||||
fun isSupported(chain: Chain, asset: String): Boolean
|
||||
fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance>
|
||||
fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance>
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,6 @@ package io.emeraldpay.dshackle.rpc
|
||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||
import io.emeraldpay.api.proto.Common
|
||||
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.dshackle.Defaults
|
||||
import io.emeraldpay.dshackle.SilentException
|
||||
import io.emeraldpay.dshackle.upstream.Capability
|
||||
@@ -27,6 +26,7 @@ import io.emeraldpay.dshackle.upstream.Selector
|
||||
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream
|
||||
import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent
|
||||
import io.emeraldpay.dshackle.upstream.grpc.BitcoinGrpcUpstream
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.bitcoinj.params.MainNetParams
|
||||
@@ -37,14 +37,12 @@ import org.springframework.stereotype.Service
|
||||
import reactor.core.publisher.Flux
|
||||
import reactor.core.publisher.Mono
|
||||
import java.math.BigInteger
|
||||
import java.time.Duration
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import javax.annotation.PostConstruct
|
||||
import kotlin.collections.HashMap
|
||||
|
||||
@Service
|
||||
class TrackBitcoinAddress(
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
) : TrackAddress {
|
||||
|
||||
companion object {
|
||||
@@ -52,8 +50,8 @@ class TrackBitcoinAddress(
|
||||
}
|
||||
|
||||
override fun isSupported(chain: Chain, asset: String): Boolean {
|
||||
return (asset == "bitcoin" || asset == "btc" || asset == "satoshi")
|
||||
&& BlockchainType.from(chain) == BlockchainType.BITCOIN && multistreamHolder.isAvailable(chain)
|
||||
return (asset == "bitcoin" || asset == "btc" || asset == "satoshi") &&
|
||||
BlockchainType.from(chain) == BlockchainType.BITCOIN && multistreamHolder.isAvailable(chain)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -65,8 +63,8 @@ class TrackBitcoinAddress(
|
||||
* Criteria for a remote grpc upstream that can provide a balance
|
||||
*/
|
||||
private val balanceUpstreamMatcher = Selector.LocalAndMatcher(
|
||||
Selector.GrpcMatcher(),
|
||||
Selector.CapabilityMatcher(Capability.BALANCE)
|
||||
Selector.GrpcMatcher(),
|
||||
Selector.CapabilityMatcher(Capability.BALANCE)
|
||||
)
|
||||
|
||||
@PostConstruct
|
||||
@@ -99,7 +97,7 @@ class TrackBitcoinAddress(
|
||||
return when {
|
||||
request.address.hasAddressXpub() -> {
|
||||
val xpubAddresses = api.getXpubAddresses()
|
||||
?: return Flux.error(IllegalStateException("Xpub verification is not available"))
|
||||
?: return Flux.error(IllegalStateException("Xpub verification is not available"))
|
||||
|
||||
val addressXpub = request.address.addressXpub
|
||||
if (StringUtils.isEmpty(addressXpub.xpub)) {
|
||||
@@ -109,45 +107,52 @@ class TrackBitcoinAddress(
|
||||
val start = Math.max(0, addressXpub.start).toInt()
|
||||
val limit = Math.min(100, Math.max(1, addressXpub.limit)).toInt()
|
||||
xpubAddresses.activeAddresses(xpub, start, limit)
|
||||
.map { it.toString() }
|
||||
.doOnError { t -> log.error("Failed to process xpub. ${t.javaClass}:${t.message}") }
|
||||
.map { it.toString() }
|
||||
.doOnError { t -> log.error("Failed to process xpub. ${t.javaClass}:${t.message}") }
|
||||
}
|
||||
request.address.hasAddressSingle() -> {
|
||||
Flux.just(request.address.addressSingle.address)
|
||||
}
|
||||
request.address.hasAddressMulti() -> {
|
||||
Flux.fromIterable(
|
||||
request.address.addressMulti.addressesList
|
||||
.map { addr -> addr.address }
|
||||
//TODO why sorted?
|
||||
.sorted()
|
||||
request.address.addressMulti.addressesList
|
||||
.map { addr -> addr.address }
|
||||
// TODO why sorted?
|
||||
.sorted()
|
||||
)
|
||||
}
|
||||
else -> Flux.error(IllegalArgumentException("Unsupported address type"))
|
||||
}
|
||||
}
|
||||
|
||||
fun requestBalances(chain: Chain, api: BitcoinMultistream, addresses: Flux<String>, includeUtxo: Boolean): Flux<AddressBalance> {
|
||||
fun requestBalances(
|
||||
chain: Chain,
|
||||
api: BitcoinMultistream,
|
||||
addresses: Flux<String>,
|
||||
includeUtxo: Boolean
|
||||
): Flux<AddressBalance> {
|
||||
return addresses
|
||||
.map { Address(chain, it) }
|
||||
.flatMap { address ->
|
||||
balanceForAddress(api, address, includeUtxo)
|
||||
}
|
||||
.map { Address(chain, it) }
|
||||
.flatMap { address ->
|
||||
balanceForAddress(api, address, includeUtxo)
|
||||
}
|
||||
}
|
||||
|
||||
fun balanceForAddress(api: BitcoinMultistream, address: Address, includeUtxo: Boolean): Mono<AddressBalance> {
|
||||
return api.getReader()
|
||||
.listUnspent(address.bitcoinAddress)
|
||||
.map { unspent ->
|
||||
totalUnspent(address, includeUtxo, unspent)
|
||||
}
|
||||
.switchIfEmpty(Mono.just(0).map {
|
||||
.listUnspent(address.bitcoinAddress)
|
||||
.map { unspent ->
|
||||
totalUnspent(address, includeUtxo, unspent)
|
||||
}
|
||||
.switchIfEmpty(
|
||||
Mono.just(0).map {
|
||||
AddressBalance(address, BigInteger.ZERO)
|
||||
})
|
||||
.onErrorResume { t ->
|
||||
log.error("Failed to get unspent", t)
|
||||
Mono.empty()
|
||||
}
|
||||
)
|
||||
.onErrorResume { t ->
|
||||
log.error("Failed to get unspent", t)
|
||||
Mono.empty()
|
||||
}
|
||||
}
|
||||
|
||||
fun totalUnspent(address: Address, includeUtxo: Boolean, unspent: List<SimpleUnspent>): AddressBalance {
|
||||
@@ -156,10 +161,10 @@ class TrackBitcoinAddress(
|
||||
} else {
|
||||
unspent.map {
|
||||
AddressBalance(
|
||||
address,
|
||||
BigInteger.valueOf(it.value),
|
||||
if (includeUtxo) listOf(BalanceUtxo(it.txid, it.vout, it.value))
|
||||
else emptyList()
|
||||
address,
|
||||
BigInteger.valueOf(it.value),
|
||||
if (includeUtxo) listOf(BalanceUtxo(it.txid, it.vout, it.value))
|
||||
else emptyList()
|
||||
)
|
||||
}.reduce { a, b -> a.plus(b) }
|
||||
}
|
||||
@@ -169,26 +174,32 @@ class TrackBitcoinAddress(
|
||||
val ups = api.getApiSource(balanceUpstreamMatcher)
|
||||
ups.request(1)
|
||||
return Mono.from(ups)
|
||||
.map { up ->
|
||||
up.cast(BitcoinGrpcUpstream::class.java).remote
|
||||
}
|
||||
.timeout(Defaults.timeoutInternal, Mono.empty())
|
||||
.switchIfEmpty(
|
||||
Mono.just(0)
|
||||
.doOnNext {
|
||||
log.warn("No upstream providing balance for ${api.chain}")
|
||||
}
|
||||
.then(Mono.error(SilentException.DataUnavailable("BALANCE")))
|
||||
)
|
||||
.map { up ->
|
||||
up.cast(BitcoinGrpcUpstream::class.java).remote
|
||||
}
|
||||
.timeout(Defaults.timeoutInternal, Mono.empty())
|
||||
.switchIfEmpty(
|
||||
Mono.just(0)
|
||||
.doOnNext {
|
||||
log.warn("No upstream providing balance for ${api.chain}")
|
||||
}
|
||||
.then(Mono.error(SilentException.DataUnavailable("BALANCE")))
|
||||
)
|
||||
}
|
||||
|
||||
fun getRemoteBalance(api: BitcoinMultistream, request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
||||
fun getRemoteBalance(
|
||||
api: BitcoinMultistream,
|
||||
request: BlockchainOuterClass.BalanceRequest
|
||||
): Flux<BlockchainOuterClass.AddressBalance> {
|
||||
return getBalanceGrpc(api).flatMapMany { remote ->
|
||||
remote.getBalance(request)
|
||||
}
|
||||
}
|
||||
|
||||
fun subscribeRemoteBalance(api: BitcoinMultistream, request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
||||
fun subscribeRemoteBalance(
|
||||
api: BitcoinMultistream,
|
||||
request: BlockchainOuterClass.BalanceRequest
|
||||
): Flux<BlockchainOuterClass.AddressBalance> {
|
||||
return getBalanceGrpc(api).flatMapMany { remote ->
|
||||
remote.subscribeBalance(request)
|
||||
}
|
||||
@@ -197,43 +208,43 @@ class TrackBitcoinAddress(
|
||||
override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
||||
val chain = Chain.byId(request.asset.chainValue)
|
||||
val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java)
|
||||
?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue))
|
||||
?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue))
|
||||
return if (isBalanceAvailable(chain)) {
|
||||
val addresses = allAddresses(upstream, request)
|
||||
requestBalances(chain, upstream, addresses, request.includeUtxo)
|
||||
.map(this@TrackBitcoinAddress::buildResponse)
|
||||
.doOnError { t ->
|
||||
log.error("Failed to get balance", t)
|
||||
}
|
||||
.map(this@TrackBitcoinAddress::buildResponse)
|
||||
.doOnError { t ->
|
||||
log.error("Failed to get balance", t)
|
||||
}
|
||||
} else {
|
||||
getRemoteBalance(upstream, request)
|
||||
.doOnError { t ->
|
||||
log.error("Failed to get balance from remote", t)
|
||||
}
|
||||
.doOnError { t ->
|
||||
log.error("Failed to get balance from remote", t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
||||
val chain = Chain.byId(request.asset.chainValue)
|
||||
val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java)
|
||||
?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue))
|
||||
?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue))
|
||||
if (isBalanceAvailable(chain)) {
|
||||
val addresses = allAddresses(upstream, request).cache()
|
||||
val following = upstream.getHead().getFlux()
|
||||
.flatMap {
|
||||
requestBalances(chain, upstream, Flux.from(addresses), request.includeUtxo)
|
||||
}
|
||||
.flatMap {
|
||||
requestBalances(chain, upstream, Flux.from(addresses), request.includeUtxo)
|
||||
}
|
||||
val last = HashMap<String, BigInteger>()
|
||||
val result = following
|
||||
.filter { curr ->
|
||||
val prev = last[curr.address.address]
|
||||
//TODO utxo can change without changing balance
|
||||
val changed = prev == null || curr.balance != prev
|
||||
if (changed) {
|
||||
last[curr.address.address] = curr.balance
|
||||
}
|
||||
changed
|
||||
.filter { curr ->
|
||||
val prev = last[curr.address.address]
|
||||
// TODO utxo can change without changing balance
|
||||
val changed = prev == null || curr.balance != prev
|
||||
if (changed) {
|
||||
last[curr.address.address] = curr.balance
|
||||
}
|
||||
changed
|
||||
}
|
||||
|
||||
return result.map(this@TrackBitcoinAddress::buildResponse)
|
||||
} else {
|
||||
@@ -243,24 +254,30 @@ class TrackBitcoinAddress(
|
||||
|
||||
private fun buildResponse(address: AddressBalance): BlockchainOuterClass.AddressBalance {
|
||||
return BlockchainOuterClass.AddressBalance.newBuilder()
|
||||
.setBalance(address.balance.toString(10))
|
||||
.setAsset(Common.Asset.newBuilder()
|
||||
.setChainValue(address.address.chain.id)
|
||||
.setCode("BTC"))
|
||||
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.address))
|
||||
.addAllUtxo(
|
||||
address.utxo.map { utxo ->
|
||||
BlockchainOuterClass.Utxo.newBuilder()
|
||||
.setBalance(utxo.value.toString())
|
||||
.setIndex(utxo.vout.toLong())
|
||||
.setTxId(utxo.txid)
|
||||
.build()
|
||||
}
|
||||
)
|
||||
.build()
|
||||
.setBalance(address.balance.toString(10))
|
||||
.setAsset(
|
||||
Common.Asset.newBuilder()
|
||||
.setChainValue(address.address.chain.id)
|
||||
.setCode("BTC")
|
||||
)
|
||||
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.address))
|
||||
.addAllUtxo(
|
||||
address.utxo.map { utxo ->
|
||||
BlockchainOuterClass.Utxo.newBuilder()
|
||||
.setBalance(utxo.value.toString())
|
||||
.setIndex(utxo.vout.toLong())
|
||||
.setTxId(utxo.txid)
|
||||
.build()
|
||||
}
|
||||
)
|
||||
.build()
|
||||
}
|
||||
|
||||
open class AddressBalance(val address: Address, var balance: BigInteger = BigInteger.ZERO, var utxo: List<BalanceUtxo> = emptyList()) {
|
||||
open class AddressBalance(
|
||||
val address: Address,
|
||||
var balance: BigInteger = BigInteger.ZERO,
|
||||
var utxo: List<BalanceUtxo> = emptyList()
|
||||
) {
|
||||
constructor(chain: Chain, address: String, balance: BigInteger) : this(Address(chain, address), balance)
|
||||
|
||||
fun plus(other: AddressBalance) = AddressBalance(address, balance + other.balance, utxo.plus(other.utxo))
|
||||
@@ -268,7 +285,7 @@ class TrackBitcoinAddress(
|
||||
|
||||
open class BalanceUtxo(val txid: String, val vout: Int, val value: Long)
|
||||
|
||||
//TODO use bitcoin class for address
|
||||
// TODO use bitcoin class for address
|
||||
class Address(val chain: Chain, val address: String) {
|
||||
val network = if (chain == Chain.BITCOIN) {
|
||||
MainNetParams()
|
||||
@@ -276,7 +293,7 @@ class TrackBitcoinAddress(
|
||||
TestNet3Params()
|
||||
}
|
||||
val bitcoinAddress = org.bitcoinj.core.Address.fromString(
|
||||
network, address
|
||||
network, address
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,11 +18,11 @@ package io.emeraldpay.dshackle.rpc
|
||||
import com.google.protobuf.ByteString
|
||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||
import io.emeraldpay.api.proto.Common
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.dshackle.SilentException
|
||||
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
||||
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream
|
||||
import io.emeraldpay.dshackle.upstream.bitcoin.ExtractBlock
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
@@ -37,7 +37,7 @@ import kotlin.math.min
|
||||
|
||||
@Service
|
||||
class TrackBitcoinTx(
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
) : TrackTx {
|
||||
|
||||
companion object {
|
||||
@@ -51,61 +51,69 @@ class TrackBitcoinTx(
|
||||
override fun subscribe(request: BlockchainOuterClass.TxStatusRequest): Flux<BlockchainOuterClass.TxStatus> {
|
||||
val chain = Chain.byId(request.chainValue)
|
||||
val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java)
|
||||
?: return Flux.error(SilentException.UnsupportedBlockchain(chain))
|
||||
?: return Flux.error(SilentException.UnsupportedBlockchain(chain))
|
||||
val txid = request.txId
|
||||
val confirmations = max(min(1, request.confirmationLimit), 12)
|
||||
return subscribe(chain, upstream, txid)
|
||||
.takeUntil { tx ->
|
||||
tx.confirmations >= confirmations
|
||||
}.map(this::asProto)
|
||||
.takeUntil { tx ->
|
||||
tx.confirmations >= confirmations
|
||||
}.map(this::asProto)
|
||||
}
|
||||
|
||||
fun subscribe(chain: Chain, upstream: BitcoinMultistream, txid: String): Flux<TxStatus> {
|
||||
return loadExisting(upstream, txid)
|
||||
.flatMapMany { status ->
|
||||
if (status.mined) {
|
||||
//Head almost always knows the current height, so it can continue with calculating confirmations
|
||||
//without publishing an empty TxStatus first
|
||||
continueWithMined(upstream, status)
|
||||
} else {
|
||||
loadMempool(upstream, txid)
|
||||
.flatMapMany { tx ->
|
||||
val next = if (tx.found) {
|
||||
untilMined(upstream, tx)
|
||||
} else {
|
||||
untilFound(chain, upstream, txid)
|
||||
}
|
||||
//fist provide the current status, then updates
|
||||
Flux.concat(Mono.just(tx), next)
|
||||
}
|
||||
}
|
||||
.flatMapMany { status ->
|
||||
if (status.mined) {
|
||||
// Head almost always knows the current height, so it can continue with calculating confirmations
|
||||
// without publishing an empty TxStatus first
|
||||
continueWithMined(upstream, status)
|
||||
} else {
|
||||
loadMempool(upstream, txid)
|
||||
.flatMapMany { tx ->
|
||||
val next = if (tx.found) {
|
||||
untilMined(upstream, tx)
|
||||
} else {
|
||||
untilFound(chain, upstream, txid)
|
||||
}
|
||||
// fist provide the current status, then updates
|
||||
Flux.concat(Mono.just(tx), next)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun continueWithMined(upstream: BitcoinMultistream, status: TxStatus): Flux<TxStatus> {
|
||||
return upstream.getReader().getBlock(status.blockHash!!)
|
||||
.map { block ->
|
||||
TxStatus(status.txid, true, ExtractBlock.getHeight(block), true, status.blockHash, ExtractBlock.getTime(block), ExtractBlock.getDifficulty(block))
|
||||
}.flatMapMany { tx ->
|
||||
withConfirmations(upstream, tx)
|
||||
}
|
||||
.map { block ->
|
||||
TxStatus(
|
||||
status.txid,
|
||||
true,
|
||||
ExtractBlock.getHeight(block),
|
||||
true,
|
||||
status.blockHash,
|
||||
ExtractBlock.getTime(block),
|
||||
ExtractBlock.getDifficulty(block)
|
||||
)
|
||||
}.flatMapMany { tx ->
|
||||
withConfirmations(upstream, tx)
|
||||
}
|
||||
}
|
||||
|
||||
fun untilFound(chain: Chain, upstream: BitcoinMultistream, txid: String): Flux<TxStatus> {
|
||||
return Flux.interval(Duration.ofSeconds(1))
|
||||
.take(Duration.ofMinutes(10))
|
||||
.flatMap { loadMempool(upstream, txid) }
|
||||
.skipUntil { it.found }
|
||||
.flatMap { subscribe(chain, upstream, txid) }
|
||||
.doOnError { t ->
|
||||
log.error("Failed to wait until found", t)
|
||||
}
|
||||
.take(Duration.ofMinutes(10))
|
||||
.flatMap { loadMempool(upstream, txid) }
|
||||
.skipUntil { it.found }
|
||||
.flatMap { subscribe(chain, upstream, txid) }
|
||||
.doOnError { t ->
|
||||
log.error("Failed to wait until found", t)
|
||||
}
|
||||
}
|
||||
|
||||
fun untilMined(upstream: BitcoinMultistream, tx: TxStatus): Mono<TxStatus> {
|
||||
return upstream.getHead().getFlux().flatMap {
|
||||
loadExisting(upstream, tx.txid)
|
||||
.filter { it.mined }
|
||||
.filter { it.mined }
|
||||
}.single()
|
||||
}
|
||||
|
||||
@@ -136,34 +144,36 @@ class TrackBitcoinTx(
|
||||
|
||||
private fun asProto(tx: TxStatus): BlockchainOuterClass.TxStatus {
|
||||
val data = BlockchainOuterClass.TxStatus.newBuilder()
|
||||
.setTxId(tx.txid)
|
||||
.setConfirmations(tx.confirmations.toInt())
|
||||
.setTxId(tx.txid)
|
||||
.setConfirmations(tx.confirmations.toInt())
|
||||
|
||||
data.broadcasted = tx.found
|
||||
val isMined = tx.mined
|
||||
data.mined = isMined
|
||||
if (isMined) {
|
||||
data.setBlock(
|
||||
Common.BlockInfo.newBuilder()
|
||||
.setBlockId(tx.blockHash!!.substring(2))
|
||||
.setTimestamp(tx.blockTime!!.toEpochMilli())
|
||||
.setWeight(ByteString.copyFrom(tx.blockTotalDifficulty!!.toByteArray()))
|
||||
.setHeight(tx.height!!)
|
||||
Common.BlockInfo.newBuilder()
|
||||
.setBlockId(tx.blockHash!!.substring(2))
|
||||
.setTimestamp(tx.blockTime!!.toEpochMilli())
|
||||
.setWeight(ByteString.copyFrom(tx.blockTotalDifficulty!!.toByteArray()))
|
||||
.setHeight(tx.height!!)
|
||||
)
|
||||
}
|
||||
return data.build()
|
||||
}
|
||||
|
||||
class TxStatus(
|
||||
val txid: String,
|
||||
val found: Boolean = false,
|
||||
val height: Long? = null,
|
||||
val mined: Boolean = false,
|
||||
val blockHash: String? = null,
|
||||
val blockTime: Instant? = null,
|
||||
val blockTotalDifficulty: BigInteger? = null,
|
||||
val confirmations: Long = 0) {
|
||||
val txid: String,
|
||||
val found: Boolean = false,
|
||||
val height: Long? = null,
|
||||
val mined: Boolean = false,
|
||||
val blockHash: String? = null,
|
||||
val blockTime: Instant? = null,
|
||||
val blockTotalDifficulty: BigInteger? = null,
|
||||
val confirmations: Long = 0
|
||||
) {
|
||||
|
||||
fun withHead(headHeight: Long) = TxStatus(txid, found, height, mined, blockHash, blockTime, blockTotalDifficulty, headHeight - height!! + 1)
|
||||
fun withHead(headHeight: Long) =
|
||||
TxStatus(txid, found, height, mined, blockHash, blockTime, blockTotalDifficulty, headHeight - height!! + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ package io.emeraldpay.dshackle.rpc
|
||||
|
||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||
import io.emeraldpay.api.proto.Common
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.dshackle.SilentException
|
||||
import io.emeraldpay.dshackle.config.TokensConfig
|
||||
import io.emeraldpay.dshackle.upstream.Head
|
||||
@@ -11,25 +10,25 @@ import io.emeraldpay.dshackle.upstream.Selector
|
||||
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import io.emeraldpay.etherjar.domain.Address
|
||||
import io.emeraldpay.etherjar.erc20.ERC20Token
|
||||
import io.emeraldpay.etherjar.hex.Hex32
|
||||
import io.emeraldpay.etherjar.hex.HexQuantity
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
import reactor.core.publisher.Flux
|
||||
import reactor.core.publisher.Mono
|
||||
import java.math.BigInteger
|
||||
import java.util.*
|
||||
import java.util.Locale
|
||||
import javax.annotation.PostConstruct
|
||||
import kotlin.collections.HashMap
|
||||
|
||||
@Service
|
||||
class TrackERC20Address(
|
||||
@Autowired private val multistreamHolder: MultistreamHolder,
|
||||
@Autowired private val tokensConfig: TokensConfig
|
||||
@Autowired private val multistreamHolder: MultistreamHolder,
|
||||
@Autowired private val tokensConfig: TokensConfig
|
||||
) : TrackAddress {
|
||||
|
||||
companion object {
|
||||
@@ -46,8 +45,8 @@ class TrackERC20Address(
|
||||
val asset = token.name!!.lowercase(Locale.getDefault())
|
||||
val id = TokenId(chain, asset)
|
||||
val definition = TokenDefinition(
|
||||
chain, asset,
|
||||
ERC20Token(Address.from(token.address))
|
||||
chain, asset,
|
||||
ERC20Token(Address.from(token.address))
|
||||
)
|
||||
tokens[id] = definition
|
||||
log.info("Enable ERC20 balance for $chain:$asset")
|
||||
@@ -56,7 +55,7 @@ class TrackERC20Address(
|
||||
|
||||
override fun isSupported(chain: Chain, asset: String): Boolean {
|
||||
return tokens.containsKey(TokenId(chain, asset.lowercase(Locale.getDefault()))) &&
|
||||
BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain)
|
||||
BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain)
|
||||
}
|
||||
|
||||
override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
||||
@@ -64,9 +63,9 @@ class TrackERC20Address(
|
||||
val asset = request.asset.code.lowercase(Locale.getDefault())
|
||||
val tokenDefinition = tokens[TokenId(chain, asset)] ?: return Flux.empty()
|
||||
return ethereumAddresses.extract(request.address)
|
||||
.map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) }
|
||||
.flatMap { addr -> getBalance(addr).map(addr::withBalance) }
|
||||
.map { buildResponse(it) }
|
||||
.map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) }
|
||||
.flatMap { addr -> getBalance(addr).map(addr::withBalance) }
|
||||
.map { buildResponse(it) }
|
||||
}
|
||||
|
||||
override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
||||
@@ -76,63 +75,65 @@ class TrackERC20Address(
|
||||
val head = multistreamHolder.getUpstream(chain)?.getHead()?.getFlux() ?: Flux.empty()
|
||||
|
||||
return ethereumAddresses.extract(request.address)
|
||||
.map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) }
|
||||
.flatMap { addr ->
|
||||
val current = getBalance(addr)
|
||||
val updates = head.flatMap { getBalance(addr) }
|
||||
Flux.concat(current, updates)
|
||||
.distinctUntilChanged()
|
||||
.map { addr.withBalance(it) }
|
||||
}
|
||||
.map { buildResponse(it) }
|
||||
.map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) }
|
||||
.flatMap { addr ->
|
||||
val current = getBalance(addr)
|
||||
val updates = head.flatMap { getBalance(addr) }
|
||||
Flux.concat(current, updates)
|
||||
.distinctUntilChanged()
|
||||
.map { addr.withBalance(it) }
|
||||
}
|
||||
.map { buildResponse(it) }
|
||||
}
|
||||
|
||||
fun getBalance(addr: TrackedAddress): Mono<BigInteger> {
|
||||
val upstream = getUpstream(addr.chain)
|
||||
return upstream
|
||||
.getDirectApi(Selector.empty)
|
||||
.flatMap { api ->
|
||||
api.read(prepareEthCall(addr.token, addr.address, upstream.getHead()))
|
||||
.flatMap(JsonRpcResponse::requireStringResult)
|
||||
.map {
|
||||
Hex32.from(it).asQuantity().value
|
||||
}
|
||||
}
|
||||
.getDirectApi(Selector.empty)
|
||||
.flatMap { api ->
|
||||
api.read(prepareEthCall(addr.token, addr.address, upstream.getHead()))
|
||||
.flatMap(JsonRpcResponse::requireStringResult)
|
||||
.map {
|
||||
Hex32.from(it).asQuantity().value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun prepareEthCall(token: ERC20Token, target: Address, head: Head): JsonRpcRequest {
|
||||
val call = token
|
||||
.readBalanceOf(target)
|
||||
.toJson()
|
||||
.readBalanceOf(target)
|
||||
.toJson()
|
||||
val height = head.getCurrentHeight()?.let { HexQuantity.from(it).toHex() } ?: "latest"
|
||||
return JsonRpcRequest("eth_call", listOf(call, height))
|
||||
}
|
||||
|
||||
fun getUpstream(chain: Chain): EthereumMultistream {
|
||||
return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java)
|
||||
?: throw SilentException.UnsupportedBlockchain(chain)
|
||||
?: throw SilentException.UnsupportedBlockchain(chain)
|
||||
}
|
||||
|
||||
private fun buildResponse(address: TrackedAddress): BlockchainOuterClass.AddressBalance {
|
||||
return BlockchainOuterClass.AddressBalance.newBuilder()
|
||||
.setBalance(address.balance!!.toString(10))
|
||||
.setAsset(Common.Asset.newBuilder()
|
||||
.setChainValue(address.chain.id)
|
||||
.setCode(address.tokenName.uppercase(Locale.getDefault())))
|
||||
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex()))
|
||||
.build()
|
||||
.setBalance(address.balance!!.toString(10))
|
||||
.setAsset(
|
||||
Common.Asset.newBuilder()
|
||||
.setChainValue(address.chain.id)
|
||||
.setCode(address.tokenName.uppercase(Locale.getDefault()))
|
||||
)
|
||||
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex()))
|
||||
.build()
|
||||
}
|
||||
|
||||
class TrackedAddress(val chain: Chain,
|
||||
val address: Address,
|
||||
val token: ERC20Token,
|
||||
val tokenName: String,
|
||||
val balance: BigInteger? = null
|
||||
class TrackedAddress(
|
||||
val chain: Chain,
|
||||
val address: Address,
|
||||
val token: ERC20Token,
|
||||
val tokenName: String,
|
||||
val balance: BigInteger? = null
|
||||
) {
|
||||
fun withBalance(balance: BigInteger) = TrackedAddress(chain, address, token, tokenName, balance)
|
||||
}
|
||||
|
||||
data class TokenId(val chain: Chain, val name: String)
|
||||
data class TokenDefinition(val chain: Chain, val name: String, val token: ERC20Token)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,24 +18,24 @@ package io.emeraldpay.dshackle.rpc
|
||||
|
||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||
import io.emeraldpay.api.proto.Common
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.dshackle.Defaults
|
||||
import io.emeraldpay.dshackle.SilentException
|
||||
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
||||
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import io.emeraldpay.etherjar.domain.Address
|
||||
import io.emeraldpay.etherjar.domain.Wei
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
import reactor.core.publisher.Flux
|
||||
import reactor.core.publisher.Mono
|
||||
import java.util.*
|
||||
import java.util.Locale
|
||||
|
||||
@Service
|
||||
class TrackEthereumAddress(
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
) : TrackAddress {
|
||||
|
||||
private val log = LoggerFactory.getLogger(TrackEthereumAddress::class.java)
|
||||
@@ -43,46 +43,46 @@ class TrackEthereumAddress(
|
||||
|
||||
override fun isSupported(chain: Chain, asset: String): Boolean {
|
||||
return asset == "ether" &&
|
||||
BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain)
|
||||
BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain)
|
||||
}
|
||||
|
||||
override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
||||
return initAddress(request)
|
||||
.flatMap { a -> getBalance(a).map { a.withBalance(it) } }
|
||||
.map { buildResponse(it) }
|
||||
.flatMap { a -> getBalance(a).map { a.withBalance(it) } }
|
||||
.map { buildResponse(it) }
|
||||
}
|
||||
|
||||
override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
|
||||
val chain = Chain.byId(request.asset.chainValue)
|
||||
val head = multistreamHolder.getUpstream(chain)?.getHead()?.getFlux() ?: Flux.empty()
|
||||
val balances = initAddress(request)
|
||||
.flatMap { tracked ->
|
||||
val current = getBalance(tracked)
|
||||
.map {
|
||||
tracked.withBalance(it)
|
||||
}
|
||||
val updates = head
|
||||
.flatMap {
|
||||
getBalance(tracked)
|
||||
}.map {
|
||||
tracked.withBalance(it)
|
||||
}
|
||||
|
||||
Flux.concat(current, updates)
|
||||
.distinctUntilChanged {
|
||||
it.balance ?: Wei.ZERO
|
||||
}
|
||||
}
|
||||
.doOnError { t ->
|
||||
if (t is SilentException) {
|
||||
if (t is SilentException.UnsupportedBlockchain) {
|
||||
log.warn("Unsupported blockchain: ${t.blockchainId}")
|
||||
}
|
||||
log.debug("Failed to process subscription", t)
|
||||
} else {
|
||||
log.warn("Failed to process subscription", t)
|
||||
.flatMap { tracked ->
|
||||
val current = getBalance(tracked)
|
||||
.map {
|
||||
tracked.withBalance(it)
|
||||
}
|
||||
val updates = head
|
||||
.flatMap {
|
||||
getBalance(tracked)
|
||||
}.map {
|
||||
tracked.withBalance(it)
|
||||
}
|
||||
|
||||
Flux.concat(current, updates)
|
||||
.distinctUntilChanged {
|
||||
it.balance ?: Wei.ZERO
|
||||
}
|
||||
}
|
||||
.doOnError { t ->
|
||||
if (t is SilentException) {
|
||||
if (t is SilentException.UnsupportedBlockchain) {
|
||||
log.warn("Unsupported blockchain: ${t.blockchainId}")
|
||||
}
|
||||
log.debug("Failed to process subscription", t)
|
||||
} else {
|
||||
log.warn("Failed to process subscription", t)
|
||||
}
|
||||
}
|
||||
|
||||
return balances.map {
|
||||
buildResponse(it)
|
||||
@@ -91,7 +91,7 @@ class TrackEthereumAddress(
|
||||
|
||||
fun getUpstream(chain: Chain): EthereumMultistream {
|
||||
return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java)
|
||||
?: throw SilentException.UnsupportedBlockchain(chain)
|
||||
?: throw SilentException.UnsupportedBlockchain(chain)
|
||||
}
|
||||
|
||||
private fun initAddress(request: BlockchainOuterClass.BalanceRequest): Flux<TrackedAddress> {
|
||||
@@ -110,33 +110,36 @@ class TrackEthereumAddress(
|
||||
private fun createAddress(address: Common.SingleAddress, chain: Chain): TrackedAddress {
|
||||
val addressParsed = Address.from(address.address)
|
||||
return TrackedAddress(
|
||||
chain,
|
||||
addressParsed
|
||||
chain,
|
||||
addressParsed
|
||||
)
|
||||
}
|
||||
|
||||
fun getBalance(addr: TrackedAddress): Mono<Wei> {
|
||||
return getUpstream(addr.chain)
|
||||
.getReader()
|
||||
.balance()
|
||||
.read(addr.address)
|
||||
.timeout(Defaults.timeout)
|
||||
.getReader()
|
||||
.balance()
|
||||
.read(addr.address)
|
||||
.timeout(Defaults.timeout)
|
||||
}
|
||||
|
||||
private fun buildResponse(address: TrackedAddress): BlockchainOuterClass.AddressBalance {
|
||||
return BlockchainOuterClass.AddressBalance.newBuilder()
|
||||
.setBalance(address.balance!!.amount!!.toString(10))
|
||||
.setAsset(Common.Asset.newBuilder()
|
||||
.setChainValue(address.chain.id)
|
||||
.setCode("ETHER"))
|
||||
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex()))
|
||||
.build()
|
||||
.setBalance(address.balance!!.amount!!.toString(10))
|
||||
.setAsset(
|
||||
Common.Asset.newBuilder()
|
||||
.setChainValue(address.chain.id)
|
||||
.setCode("ETHER")
|
||||
)
|
||||
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex()))
|
||||
.build()
|
||||
}
|
||||
|
||||
class TrackedAddress(val chain: Chain,
|
||||
val address: Address,
|
||||
val balance: Wei? = null
|
||||
class TrackedAddress(
|
||||
val chain: Chain,
|
||||
val address: Address,
|
||||
val balance: Wei? = null
|
||||
) {
|
||||
fun withBalance(balance: Wei) = TrackedAddress(chain, address, balance)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,19 +19,19 @@ package io.emeraldpay.dshackle.rpc
|
||||
import com.google.protobuf.ByteString
|
||||
import io.emeraldpay.api.proto.BlockchainOuterClass
|
||||
import io.emeraldpay.api.proto.Common
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.dshackle.SilentException
|
||||
import io.emeraldpay.dshackle.data.BlockContainer
|
||||
import io.emeraldpay.dshackle.data.TxId
|
||||
import io.emeraldpay.dshackle.upstream.MultistreamHolder
|
||||
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import io.emeraldpay.etherjar.domain.BlockHash
|
||||
import io.emeraldpay.etherjar.domain.TransactionId
|
||||
import io.emeraldpay.etherjar.rpc.RpcException
|
||||
import io.emeraldpay.etherjar.rpc.json.BlockJson
|
||||
import io.emeraldpay.etherjar.rpc.json.TransactionJson
|
||||
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
@@ -48,7 +48,7 @@ import kotlin.math.min
|
||||
|
||||
@Service
|
||||
class TrackEthereumTx(
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
@Autowired private val multistreamHolder: MultistreamHolder
|
||||
) : TrackTx {
|
||||
|
||||
companion object {
|
||||
@@ -70,63 +70,62 @@ class TrackEthereumTx(
|
||||
val base = prepareTracking(request)
|
||||
val up = getUpstream(base.chain)
|
||||
return update(base)
|
||||
.defaultIfEmpty(base)
|
||||
.flatMapMany {
|
||||
Flux.concat(Mono.just(it), subscribe(it, up))
|
||||
.distinctUntilChanged(TxDetails::status)
|
||||
.map(this@TrackEthereumTx::asProto)
|
||||
.subscribeOn(scheduler)
|
||||
}
|
||||
.doOnError { t ->
|
||||
log.error("Subscription error", t)
|
||||
}
|
||||
.defaultIfEmpty(base)
|
||||
.flatMapMany {
|
||||
Flux.concat(Mono.just(it), subscribe(it, up))
|
||||
.distinctUntilChanged(TxDetails::status)
|
||||
.map(this@TrackEthereumTx::asProto)
|
||||
.subscribeOn(scheduler)
|
||||
}
|
||||
.doOnError { t ->
|
||||
log.error("Subscription error", t)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fun getUpstream(chain: Chain): EthereumMultistream {
|
||||
return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java)
|
||||
?: throw SilentException.UnsupportedBlockchain(chain)
|
||||
?: throw SilentException.UnsupportedBlockchain(chain)
|
||||
}
|
||||
|
||||
fun subscribe(base: TxDetails, up: EthereumMultistream): Flux<TxDetails> {
|
||||
var latestTx = base
|
||||
|
||||
val untilFound = Mono.just(latestTx)
|
||||
.subscribeOn(scheduler)
|
||||
.map {
|
||||
//replace with the latest value, it may be already found
|
||||
latestTx
|
||||
.subscribeOn(scheduler)
|
||||
.map {
|
||||
// replace with the latest value, it may be already found
|
||||
latestTx
|
||||
}
|
||||
.flatMap { latest ->
|
||||
if (!latest.status.found) {
|
||||
update(latest).defaultIfEmpty(latestTx)
|
||||
} else {
|
||||
Mono.just(latest)
|
||||
}
|
||||
.flatMap { latest ->
|
||||
if (!latest.status.found) {
|
||||
update(latest).defaultIfEmpty(latestTx)
|
||||
} else {
|
||||
Mono.just(latest)
|
||||
}
|
||||
}
|
||||
.flatMap { received ->
|
||||
if (!received.status.found) {
|
||||
Mono.error(SilentException("Retry not found"))
|
||||
} else {
|
||||
Mono.just(received)
|
||||
}
|
||||
.flatMap { received ->
|
||||
if (!received.status.found) {
|
||||
Mono.error(SilentException("Retry not found"))
|
||||
} else {
|
||||
Mono.just(received)
|
||||
}
|
||||
}
|
||||
.retryWhen(
|
||||
Retry.fixedDelay(10, Duration.ofSeconds(2))
|
||||
)
|
||||
.onErrorResume { Mono.empty() }
|
||||
}
|
||||
.retryWhen(
|
||||
Retry.fixedDelay(10, Duration.ofSeconds(2))
|
||||
)
|
||||
.onErrorResume { Mono.empty() }
|
||||
|
||||
val inBlocks = up.getHead().getFlux()
|
||||
.subscribeOn(scheduler)
|
||||
.flatMap { block ->
|
||||
onNewBlock(latestTx, block)
|
||||
}
|
||||
.subscribeOn(scheduler)
|
||||
.flatMap { block ->
|
||||
onNewBlock(latestTx, block)
|
||||
}
|
||||
|
||||
return Flux.merge(untilFound, inBlocks)
|
||||
.takeUntil(TxDetails::shouldClose)
|
||||
.doOnNext { newTx ->
|
||||
latestTx = newTx
|
||||
}
|
||||
.takeUntil(TxDetails::shouldClose)
|
||||
.doOnNext { newTx ->
|
||||
latestTx = newTx
|
||||
}
|
||||
}
|
||||
|
||||
fun onNewBlock(tx: TxDetails, block: BlockContainer): Mono<TxDetails> {
|
||||
@@ -134,7 +133,8 @@ class TrackEthereumTx(
|
||||
if (!tx.status.mined) {
|
||||
val justMined = block.transactions.contains(txid)
|
||||
return if (justMined) {
|
||||
Mono.just(tx.withStatus(
|
||||
Mono.just(
|
||||
tx.withStatus(
|
||||
mined = true,
|
||||
found = true,
|
||||
confirmations = 1,
|
||||
@@ -142,13 +142,14 @@ class TrackEthereumTx(
|
||||
blockTime = block.timestamp,
|
||||
blockTotalDifficulty = block.difficulty,
|
||||
blockHash = BlockHash(block.hash.value)
|
||||
))
|
||||
)
|
||||
)
|
||||
} else {
|
||||
update(tx)
|
||||
}
|
||||
} else {
|
||||
//verify if it's still on chain
|
||||
//TODO head is supposed to erase block when it was replaced, so can safely recalc here
|
||||
// verify if it's still on chain
|
||||
// TODO head is supposed to erase block when it was replaced, so can safely recalc here
|
||||
return update(tx)
|
||||
}
|
||||
}
|
||||
@@ -157,19 +158,19 @@ class TrackEthereumTx(
|
||||
val initialStatus = tx.status
|
||||
val upstream = getUpstream(tx.chain)
|
||||
return upstream.getReader()
|
||||
.txByHash().read(tx.txid)
|
||||
.onErrorResume(RpcException::class.java) { t ->
|
||||
log.warn("Upstream error, ignoring. {}", t.rpcMessage)
|
||||
Mono.empty<TransactionJson>()
|
||||
}
|
||||
.flatMap { updateFromBlock(upstream, tx, it) }
|
||||
.doOnError { t ->
|
||||
log.error("Failed to load tx block", t)
|
||||
}
|
||||
.switchIfEmpty(Mono.just(tx.withStatus(found = false)))
|
||||
.filter { current ->
|
||||
initialStatus != current.status || current.shouldClose()
|
||||
}
|
||||
.txByHash().read(tx.txid)
|
||||
.onErrorResume(RpcException::class.java) { t ->
|
||||
log.warn("Upstream error, ignoring. {}", t.rpcMessage)
|
||||
Mono.empty<TransactionJson>()
|
||||
}
|
||||
.flatMap { updateFromBlock(upstream, tx, it) }
|
||||
.doOnError { t ->
|
||||
log.error("Failed to load tx block", t)
|
||||
}
|
||||
.switchIfEmpty(Mono.just(tx.withStatus(found = false)))
|
||||
.filter { current ->
|
||||
initialStatus != current.status || current.shouldClose()
|
||||
}
|
||||
}
|
||||
|
||||
fun prepareTracking(request: BlockchainOuterClass.TxStatusRequest): TxDetails {
|
||||
@@ -178,10 +179,10 @@ class TrackEthereumTx(
|
||||
throw SilentException.UnsupportedBlockchain(request.chainValue)
|
||||
}
|
||||
val details = TxDetails(
|
||||
chain,
|
||||
Instant.now(),
|
||||
TransactionId.from(request.txId),
|
||||
min(max(1, request.confirmationLimit), 100)
|
||||
chain,
|
||||
Instant.now(),
|
||||
TransactionId.from(request.txId),
|
||||
min(max(1, request.confirmationLimit), 100)
|
||||
)
|
||||
return details
|
||||
}
|
||||
@@ -189,12 +190,12 @@ class TrackEthereumTx(
|
||||
fun setBlockDetails(tx: TxDetails, block: BlockJson<TransactionRefJson>): TxDetails {
|
||||
return if (block.number != null && block.totalDifficulty != null) {
|
||||
tx.withStatus(
|
||||
blockTotalDifficulty = block.totalDifficulty,
|
||||
blockTime = block.timestamp
|
||||
blockTotalDifficulty = block.totalDifficulty,
|
||||
blockTime = block.timestamp
|
||||
)
|
||||
} else {
|
||||
tx.withStatus(
|
||||
mined = false
|
||||
mined = false
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -205,22 +206,22 @@ class TrackEthereumTx(
|
||||
return Mono.empty()
|
||||
}
|
||||
return upstream.getReader()
|
||||
.blocksByHashParsed().read(tx.status.blockHash)
|
||||
.map { block ->
|
||||
setBlockDetails(tx, block)
|
||||
}.doOnError { t ->
|
||||
log.warn("Failed to update weight", t)
|
||||
}
|
||||
.blocksByHashParsed().read(tx.status.blockHash)
|
||||
.map { block ->
|
||||
setBlockDetails(tx, block)
|
||||
}.doOnError { t ->
|
||||
log.warn("Failed to update weight", t)
|
||||
}
|
||||
}
|
||||
|
||||
fun updateFromBlock(upstream: EthereumMultistream, tx: TxDetails, blockTx: TransactionJson): Mono<TxDetails> {
|
||||
return if (blockTx.blockNumber != null && blockTx.blockHash != null && blockTx.blockHash != ZERO_BLOCK) {
|
||||
val updated = tx.withStatus(
|
||||
blockHash = blockTx.blockHash,
|
||||
height = blockTx.blockNumber,
|
||||
found = true,
|
||||
mined = true,
|
||||
confirmations = 1
|
||||
blockHash = blockTx.blockHash,
|
||||
height = blockTx.blockNumber,
|
||||
found = true,
|
||||
mined = true,
|
||||
confirmations = 1
|
||||
)
|
||||
upstream.getHead().getFlux().next().map { head ->
|
||||
val height = updated.status.height
|
||||
@@ -228,72 +229,89 @@ class TrackEthereumTx(
|
||||
updated
|
||||
} else {
|
||||
updated.withStatus(
|
||||
confirmations = head.height - height + 1
|
||||
confirmations = head.height - height + 1
|
||||
)
|
||||
}
|
||||
}.doOnError { t ->
|
||||
log.error("Unable to load head details", t)
|
||||
}.flatMap(this::loadWeight)
|
||||
} else {
|
||||
Mono.just(tx.withStatus(
|
||||
Mono.just(
|
||||
tx.withStatus(
|
||||
found = true,
|
||||
mined = false
|
||||
))
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun asProto(tx: TxDetails): BlockchainOuterClass.TxStatus {
|
||||
val data = BlockchainOuterClass.TxStatus.newBuilder()
|
||||
.setTxId(tx.txid.toHex())
|
||||
.setConfirmations(tx.status.confirmations.toInt())
|
||||
.setTxId(tx.txid.toHex())
|
||||
.setConfirmations(tx.status.confirmations.toInt())
|
||||
|
||||
data.broadcasted = tx.status.found
|
||||
val isMined = tx.status.mined
|
||||
data.mined = isMined
|
||||
if (isMined) {
|
||||
data.setBlock(
|
||||
Common.BlockInfo.newBuilder()
|
||||
.setBlockId(tx.status.blockHash!!.toHex().substring(2))
|
||||
.setTimestamp(tx.status.blockTime!!.toEpochMilli())
|
||||
.setWeight(ByteString.copyFrom(tx.status.blockTotalDifficulty!!.toByteArray()))
|
||||
.setHeight(tx.status.height!!)
|
||||
Common.BlockInfo.newBuilder()
|
||||
.setBlockId(tx.status.blockHash!!.toHex().substring(2))
|
||||
.setTimestamp(tx.status.blockTime!!.toEpochMilli())
|
||||
.setWeight(ByteString.copyFrom(tx.status.blockTotalDifficulty!!.toByteArray()))
|
||||
.setHeight(tx.status.height!!)
|
||||
)
|
||||
}
|
||||
return data.build()
|
||||
}
|
||||
|
||||
class TxDetails(val chain: Chain,
|
||||
val since: Instant,
|
||||
val txid: TransactionId,
|
||||
val maxConfirmations: Int,
|
||||
val status: TxStatus
|
||||
class TxDetails(
|
||||
val chain: Chain,
|
||||
val since: Instant,
|
||||
val txid: TransactionId,
|
||||
val maxConfirmations: Int,
|
||||
val status: TxStatus
|
||||
) {
|
||||
|
||||
constructor(chain: Chain,
|
||||
since: Instant,
|
||||
txid: TransactionId,
|
||||
maxConfirmations: Int) : this(chain, since, txid, maxConfirmations, TxStatus())
|
||||
constructor(
|
||||
chain: Chain,
|
||||
since: Instant,
|
||||
txid: TransactionId,
|
||||
maxConfirmations: Int
|
||||
) : this(chain, since, txid, maxConfirmations, TxStatus())
|
||||
|
||||
fun copy(
|
||||
since: Instant = this.since,
|
||||
status: TxStatus = this.status
|
||||
since: Instant = this.since,
|
||||
status: TxStatus = this.status
|
||||
) = TxDetails(chain, since, txid, maxConfirmations, status)
|
||||
|
||||
fun withStatus(found: Boolean = this.status.found,
|
||||
height: Long? = this.status.height,
|
||||
mined: Boolean = this.status.mined,
|
||||
blockHash: BlockHash? = this.status.blockHash,
|
||||
blockTime: Instant? = this.status.blockTime,
|
||||
blockTotalDifficulty: BigInteger? = this.status.blockTotalDifficulty,
|
||||
confirmations: Long = this.status.confirmations): TxDetails {
|
||||
return copy(status = this.status.copy(found, height, mined, blockHash, blockTime, blockTotalDifficulty, confirmations))
|
||||
fun withStatus(
|
||||
found: Boolean = this.status.found,
|
||||
height: Long? = this.status.height,
|
||||
mined: Boolean = this.status.mined,
|
||||
blockHash: BlockHash? = this.status.blockHash,
|
||||
blockTime: Instant? = this.status.blockTime,
|
||||
blockTotalDifficulty: BigInteger? = this.status.blockTotalDifficulty,
|
||||
confirmations: Long = this.status.confirmations
|
||||
): TxDetails {
|
||||
return copy(
|
||||
status = this.status.copy(
|
||||
found,
|
||||
height,
|
||||
mined,
|
||||
blockHash,
|
||||
blockTime,
|
||||
blockTotalDifficulty,
|
||||
confirmations
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
fun shouldClose(): Boolean {
|
||||
return maxConfirmations <= this.status.confirmations
|
||||
|| since.isBefore(Instant.now().minus(TRACK_TTL))
|
||||
|| (!status.found && since.isBefore(Instant.now().minus(NOT_FOUND_TRACK_TTL)))
|
||||
|| (!status.mined && since.isBefore(Instant.now().minus(NOT_MINED_TRACK_TTL)))
|
||||
return maxConfirmations <= this.status.confirmations ||
|
||||
since.isBefore(Instant.now().minus(TRACK_TTL)) ||
|
||||
(!status.found && since.isBefore(Instant.now().minus(NOT_FOUND_TRACK_TTL))) ||
|
||||
(!status.mined && since.isBefore(Instant.now().minus(NOT_MINED_TRACK_TTL)))
|
||||
}
|
||||
|
||||
override fun toString(): String {
|
||||
@@ -320,25 +338,27 @@ class TrackEthereumTx(
|
||||
result = 31 * result + status.hashCode()
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
class TxStatus(val found: Boolean = false,
|
||||
val height: Long? = null,
|
||||
val mined: Boolean = false,
|
||||
val blockHash: BlockHash? = null,
|
||||
val blockTime: Instant? = null,
|
||||
val blockTotalDifficulty: BigInteger? = null,
|
||||
val confirmations: Long = 0) {
|
||||
class TxStatus(
|
||||
val found: Boolean = false,
|
||||
val height: Long? = null,
|
||||
val mined: Boolean = false,
|
||||
val blockHash: BlockHash? = null,
|
||||
val blockTime: Instant? = null,
|
||||
val blockTotalDifficulty: BigInteger? = null,
|
||||
val confirmations: Long = 0
|
||||
) {
|
||||
|
||||
fun copy(found: Boolean = this.found,
|
||||
height: Long? = this.height,
|
||||
mined: Boolean = this.mined,
|
||||
blockHash: BlockHash? = this.blockHash,
|
||||
blockTime: Instant? = this.blockTime,
|
||||
blockTotalDifficulty: BigInteger? = this.blockTotalDifficulty,
|
||||
confirmation: Long = this.confirmations)
|
||||
= TxStatus(found, height, mined, blockHash, blockTime, blockTotalDifficulty, confirmation)
|
||||
fun copy(
|
||||
found: Boolean = this.found,
|
||||
height: Long? = this.height,
|
||||
mined: Boolean = this.mined,
|
||||
blockHash: BlockHash? = this.blockHash,
|
||||
blockTime: Instant? = this.blockTime,
|
||||
blockTotalDifficulty: BigInteger? = this.blockTotalDifficulty,
|
||||
confirmation: Long = this.confirmations
|
||||
) = TxStatus(found, height, mined, blockHash, blockTime, blockTotalDifficulty, confirmation)
|
||||
|
||||
fun clean() = TxStatus(false, null, false, null, null, null, 0)
|
||||
|
||||
@@ -369,7 +389,5 @@ class TrackEthereumTx(
|
||||
override fun toString(): String {
|
||||
return "TxStatus(found=$found, height=$height, mined=$mined, blockHash=$blockHash, blockTime=$blockTime, blockTotalDifficulty=$blockTotalDifficulty, confirmations=$confirmations)"
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,4 +22,4 @@ import reactor.core.publisher.Flux
|
||||
interface TrackTx {
|
||||
fun isSupported(chain: Chain): Boolean
|
||||
fun subscribe(request: BlockchainOuterClass.TxStatusRequest): Flux<BlockchainOuterClass.TxStatus>
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
*/
|
||||
package io.emeraldpay.dshackle.startup
|
||||
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.dshackle.FileResolver
|
||||
import io.emeraldpay.dshackle.cache.CachesFactory
|
||||
import io.emeraldpay.dshackle.config.UpstreamsConfig
|
||||
@@ -34,49 +33,48 @@ import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcHttpClient
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
|
||||
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
|
||||
import io.emeraldpay.grpc.BlockchainType
|
||||
import io.emeraldpay.grpc.Chain
|
||||
import io.micrometer.core.instrument.Counter
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Repository
|
||||
import io.micrometer.core.instrument.Metrics
|
||||
import io.micrometer.core.instrument.Tag
|
||||
import io.micrometer.core.instrument.Timer
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Repository
|
||||
import java.net.URI
|
||||
import java.util.*
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import javax.annotation.PostConstruct
|
||||
import kotlin.collections.HashMap
|
||||
|
||||
@Repository
|
||||
open class ConfiguredUpstreams(
|
||||
@Autowired private val currentUpstreams: CurrentMultistreamHolder,
|
||||
@Autowired private val fileResolver: FileResolver,
|
||||
@Autowired private val config: UpstreamsConfig,
|
||||
@Autowired private val cachesFactory: CachesFactory
|
||||
@Autowired private val currentUpstreams: CurrentMultistreamHolder,
|
||||
@Autowired private val fileResolver: FileResolver,
|
||||
@Autowired private val config: UpstreamsConfig,
|
||||
@Autowired private val cachesFactory: CachesFactory
|
||||
) {
|
||||
|
||||
private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java)
|
||||
private var seq = AtomicInteger(0)
|
||||
|
||||
private val chainNames = mapOf(
|
||||
"ethereum" to Chain.ETHEREUM,
|
||||
"ethereum-classic" to Chain.ETHEREUM_CLASSIC,
|
||||
"eth" to Chain.ETHEREUM,
|
||||
"polygon" to Chain.MATIC,
|
||||
"matic" to Chain.MATIC,
|
||||
"etc" to Chain.ETHEREUM_CLASSIC,
|
||||
"morden" to Chain.TESTNET_MORDEN,
|
||||
"kovan" to Chain.TESTNET_KOVAN,
|
||||
"kovan-testnet" to Chain.TESTNET_KOVAN,
|
||||
"goerli" to Chain.TESTNET_GOERLI,
|
||||
"goerli-testnet" to Chain.TESTNET_GOERLI,
|
||||
"rinkeby" to Chain.TESTNET_RINKEBY,
|
||||
"rinkeby-testnet" to Chain.TESTNET_RINKEBY,
|
||||
"ropsten" to Chain.TESTNET_ROPSTEN,
|
||||
"ropsten-testnet" to Chain.TESTNET_ROPSTEN,
|
||||
"bitcoin" to Chain.BITCOIN,
|
||||
"bitcoin-testnet" to Chain.TESTNET_BITCOIN
|
||||
"ethereum" to Chain.ETHEREUM,
|
||||
"ethereum-classic" to Chain.ETHEREUM_CLASSIC,
|
||||
"eth" to Chain.ETHEREUM,
|
||||
"polygon" to Chain.MATIC,
|
||||
"matic" to Chain.MATIC,
|
||||
"etc" to Chain.ETHEREUM_CLASSIC,
|
||||
"morden" to Chain.TESTNET_MORDEN,
|
||||
"kovan" to Chain.TESTNET_KOVAN,
|
||||
"kovan-testnet" to Chain.TESTNET_KOVAN,
|
||||
"goerli" to Chain.TESTNET_GOERLI,
|
||||
"goerli-testnet" to Chain.TESTNET_GOERLI,
|
||||
"rinkeby" to Chain.TESTNET_RINKEBY,
|
||||
"rinkeby-testnet" to Chain.TESTNET_RINKEBY,
|
||||
"ropsten" to Chain.TESTNET_ROPSTEN,
|
||||
"ropsten-testnet" to Chain.TESTNET_ROPSTEN,
|
||||
"bitcoin" to Chain.BITCOIN,
|
||||
"bitcoin-testnet" to Chain.TESTNET_BITCOIN
|
||||
)
|
||||
|
||||
@PostConstruct
|
||||
@@ -95,7 +93,7 @@ open class ConfiguredUpstreams(
|
||||
return@forEach
|
||||
}
|
||||
val options = (up.options ?: UpstreamsConfig.Options())
|
||||
.merge(defaultOptions[chain] ?: UpstreamsConfig.Options.getDefaults())
|
||||
.merge(defaultOptions[chain] ?: UpstreamsConfig.Options.getDefaults())
|
||||
when (BlockchainType.from(chain)) {
|
||||
BlockchainType.ETHEREUM -> {
|
||||
buildEthereumUpstream(up.cast(UpstreamsConfig.EthereumConnection::class.java), chain, options)
|
||||
@@ -135,9 +133,10 @@ open class ConfiguredUpstreams(
|
||||
|
||||
fun buildMethods(config: UpstreamsConfig.Upstream<*>, chain: Chain): CallMethods {
|
||||
return if (config.methods != null) {
|
||||
ManagedCallMethods(currentUpstreams.getDefaultMethods(chain),
|
||||
config.methods!!.enabled.map { it.name }.toSet(),
|
||||
config.methods!!.disabled.map { it.name }.toSet()
|
||||
ManagedCallMethods(
|
||||
currentUpstreams.getDefaultMethods(chain),
|
||||
config.methods!!.enabled.map { it.name }.toSet(),
|
||||
config.methods!!.disabled.map { it.name }.toSet()
|
||||
).also {
|
||||
config.methods!!.enabled.forEach { m ->
|
||||
if (m.quorum != null) {
|
||||
@@ -150,9 +149,11 @@ open class ConfiguredUpstreams(
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildBitcoinUpstream(config: UpstreamsConfig.Upstream<UpstreamsConfig.BitcoinConnection>,
|
||||
chain: Chain,
|
||||
options: UpstreamsConfig.Options) {
|
||||
private fun buildBitcoinUpstream(
|
||||
config: UpstreamsConfig.Upstream<UpstreamsConfig.BitcoinConnection>,
|
||||
chain: Chain,
|
||||
options: UpstreamsConfig.Options
|
||||
) {
|
||||
|
||||
val conn = config.connection!!
|
||||
val directApi: Reader<JsonRpcRequest, JsonRpcResponse>? = buildHttpClient(config)
|
||||
@@ -171,19 +172,24 @@ open class ConfiguredUpstreams(
|
||||
}
|
||||
|
||||
val methods = buildMethods(config, chain)
|
||||
val upstream = BitcoinRpcUpstream(config.id
|
||||
?: "bitcoin-${seq.getAndIncrement()}", chain, directApi,
|
||||
options, config.role,
|
||||
QuorumForLabels.QuorumItem(1, config.labels),
|
||||
methods, esplora)
|
||||
val upstream = BitcoinRpcUpstream(
|
||||
config.id
|
||||
?: "bitcoin-${seq.getAndIncrement()}",
|
||||
chain, directApi,
|
||||
options, config.role,
|
||||
QuorumForLabels.QuorumItem(1, config.labels),
|
||||
methods, esplora
|
||||
)
|
||||
|
||||
upstream.start()
|
||||
currentUpstreams.update(UpstreamChange(chain, upstream, UpstreamChange.ChangeType.ADDED))
|
||||
}
|
||||
|
||||
private fun buildEthereumUpstream(config: UpstreamsConfig.Upstream<UpstreamsConfig.EthereumConnection>,
|
||||
chain: Chain,
|
||||
options: UpstreamsConfig.Options) {
|
||||
private fun buildEthereumUpstream(
|
||||
config: UpstreamsConfig.Upstream<UpstreamsConfig.EthereumConnection>,
|
||||
chain: Chain,
|
||||
options: UpstreamsConfig.Options
|
||||
) {
|
||||
val conn = config.connection!!
|
||||
|
||||
val urls = ArrayList<URI>()
|
||||
@@ -194,8 +200,8 @@ open class ConfiguredUpstreams(
|
||||
|
||||
val wsFactoryApi: EthereumWsFactory? = conn.ws?.let { endpoint ->
|
||||
val wsApi = EthereumWsFactory(
|
||||
endpoint.url,
|
||||
endpoint.origin ?: URI("http://localhost"),
|
||||
endpoint.url,
|
||||
endpoint.origin ?: URI("http://localhost"),
|
||||
)
|
||||
wsApi.config = endpoint
|
||||
endpoint.basicAuth?.let { auth ->
|
||||
@@ -208,11 +214,11 @@ open class ConfiguredUpstreams(
|
||||
log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}")
|
||||
val ethereumUpstream = if (wsFactoryApi != null && !conn.preferHttp) {
|
||||
EthereumWsUpstream(
|
||||
config.id!!,
|
||||
chain, wsFactoryApi,
|
||||
options, config.role,
|
||||
QuorumForLabels.QuorumItem(1, config.labels),
|
||||
methods
|
||||
config.id!!,
|
||||
chain, wsFactoryApi,
|
||||
options, config.role,
|
||||
QuorumForLabels.QuorumItem(1, config.labels),
|
||||
methods
|
||||
)
|
||||
} else {
|
||||
val directApi: Reader<JsonRpcRequest, JsonRpcResponse>? = buildHttpClient(config)
|
||||
@@ -221,11 +227,11 @@ open class ConfiguredUpstreams(
|
||||
return
|
||||
}
|
||||
EthereumRpcUpstream(
|
||||
config.id!!,
|
||||
chain, directApi, wsFactoryApi,
|
||||
options, config.role,
|
||||
QuorumForLabels.QuorumItem(1, config.labels),
|
||||
methods
|
||||
config.id!!,
|
||||
chain, directApi, wsFactoryApi,
|
||||
options, config.role,
|
||||
QuorumForLabels.QuorumItem(1, config.labels),
|
||||
methods
|
||||
)
|
||||
}
|
||||
|
||||
@@ -233,23 +239,26 @@ open class ConfiguredUpstreams(
|
||||
currentUpstreams.update(UpstreamChange(chain, ethereumUpstream, UpstreamChange.ChangeType.ADDED))
|
||||
}
|
||||
|
||||
private fun buildGrpcUpstream(config: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>, options: UpstreamsConfig.Options) {
|
||||
private fun buildGrpcUpstream(
|
||||
config: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>,
|
||||
options: UpstreamsConfig.Options
|
||||
) {
|
||||
val endpoint = config.connection!!
|
||||
val ds = GrpcUpstreams(
|
||||
config.id!!,
|
||||
endpoint.host!!,
|
||||
endpoint.port ?: 2449,
|
||||
endpoint.auth,
|
||||
fileResolver
|
||||
config.id!!,
|
||||
endpoint.host!!,
|
||||
endpoint.port,
|
||||
endpoint.auth,
|
||||
fileResolver
|
||||
).apply {
|
||||
timeout = options.timeout
|
||||
}
|
||||
log.info("Using ALL CHAINS (gRPC) upstream, at ${endpoint.host}:${endpoint.port}")
|
||||
ds.start()
|
||||
.doOnNext {
|
||||
log.info("Chain ${it.chain} has ${it.type} through gRPC at ${endpoint.host}:${endpoint.port}")
|
||||
}
|
||||
.subscribe(currentUpstreams::update)
|
||||
.doOnNext {
|
||||
log.info("Chain ${it.chain} has ${it.type} through gRPC at ${endpoint.host}:${endpoint.port}")
|
||||
}
|
||||
.subscribe(currentUpstreams::update)
|
||||
}
|
||||
|
||||
private fun buildHttpClient(config: UpstreamsConfig.Upstream<out UpstreamsConfig.RpcConnection>): JsonRpcHttpClient? {
|
||||
@@ -262,29 +271,29 @@ open class ConfiguredUpstreams(
|
||||
}
|
||||
}
|
||||
val metricsTags = listOf(
|
||||
// "unknown" is not supposed to happen
|
||||
Tag.of("upstream", config.id ?: "unknown"),
|
||||
// UNSPECIFIED shouldn't happen too
|
||||
Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED ).chainCode)
|
||||
// "unknown" is not supposed to happen
|
||||
Tag.of("upstream", config.id ?: "unknown"),
|
||||
// UNSPECIFIED shouldn't happen too
|
||||
Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED).chainCode)
|
||||
)
|
||||
val metrics = RpcMetrics(
|
||||
Timer.builder("upstream.rpc.conn")
|
||||
.description("Request time through a HTTP JSON RPC connection")
|
||||
.tags(metricsTags)
|
||||
.publishPercentileHistogram()
|
||||
.register(Metrics.globalRegistry),
|
||||
Counter.builder("upstream.rpc.err")
|
||||
.description("Errors received on request through HTTP JSON RPC connection")
|
||||
.tags(metricsTags)
|
||||
.register(Metrics.globalRegistry)
|
||||
Timer.builder("upstream.rpc.conn")
|
||||
.description("Request time through a HTTP JSON RPC connection")
|
||||
.tags(metricsTags)
|
||||
.publishPercentileHistogram()
|
||||
.register(Metrics.globalRegistry),
|
||||
Counter.builder("upstream.rpc.err")
|
||||
.description("Errors received on request through HTTP JSON RPC connection")
|
||||
.tags(metricsTags)
|
||||
.register(Metrics.globalRegistry)
|
||||
)
|
||||
urls.add(endpoint.url)
|
||||
JsonRpcHttpClient(
|
||||
endpoint.url.toString(),
|
||||
metrics,
|
||||
conn.rpc?.basicAuth,
|
||||
tls
|
||||
endpoint.url.toString(),
|
||||
metrics,
|
||||
conn.rpc?.basicAuth,
|
||||
tls
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,9 +17,8 @@
|
||||
package io.emeraldpay.dshackle.startup
|
||||
|
||||
import io.emeraldpay.dshackle.config.UpstreamsConfig
|
||||
import java.util.*
|
||||
import java.util.Collections
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock
|
||||
import kotlin.collections.ArrayList
|
||||
import kotlin.concurrent.read
|
||||
import kotlin.concurrent.write
|
||||
|
||||
@@ -71,7 +70,6 @@ class QuorumForLabels() {
|
||||
return nodes.hashCode()
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Details for a single element (upstream, node or aggregation)
|
||||
*/
|
||||
@@ -97,8 +95,5 @@ class QuorumForLabels() {
|
||||
result = 31 * result + labels.hashCode()
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user