solution: kotlin code conventions

This commit is contained in:
Aldo Borrero
2021-10-21 02:00:38 +02:00
committed by GitHub
parent 04ee74514c
commit 5167b8b125
188 changed files with 3261 additions and 3111 deletions

8
.gitignore vendored
View File

@@ -1,7 +1,11 @@
.gradle/
build/
out/
*.iml
./dshackle.yaml
./upstream.yaml
testsetup/
testsetup/
.idea/
.idea_modules/
*.iml
*.ipr
*.iws

View File

@@ -27,6 +27,7 @@ plugins {
id 'io.spring.dependency-management' version '1.0.6.RELEASE'
id 'com.palantir.git-version' version '0.12.2'
id "com.google.protobuf" version "0.8.12"
id "org.jlleitschuh.gradle.ktlint" version "10.2.0"
}

View File

@@ -1,3 +1,5 @@
kotlin.code.style=official
# Languages
groovyVersion=2.5.14
kotlinVersion=1.5.30

View File

@@ -17,7 +17,7 @@ package io.emeraldpay.dshackle
import io.emeraldpay.api.proto.Common
import io.emeraldpay.grpc.Chain
import java.util.*
import java.util.EnumMap
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
@@ -25,7 +25,7 @@ import kotlin.concurrent.withLock
* Keeps a lazily created value associated with a Chain
*/
class ChainValue<V>(
private val factory: (chain: Chain) -> V
private val factory: (chain: Chain) -> V
) {
private val values = EnumMap<Chain, V>(Chain::class.java)
@@ -52,4 +52,4 @@ class ChainValue<V>(
}
}
}
}
}

View File

@@ -16,26 +16,26 @@
*/
package io.emeraldpay.dshackle
import com.fasterxml.jackson.core.Version
import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.module.SimpleModule
import io.emeraldpay.dshackle.config.*
import io.emeraldpay.dshackle.config.CacheConfig
import io.emeraldpay.dshackle.config.MainConfig
import io.emeraldpay.dshackle.config.MainConfigReader
import io.emeraldpay.dshackle.config.MonitoringConfig
import io.emeraldpay.dshackle.config.TokensConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.boot.ExitCodeGenerator
import org.springframework.boot.SpringApplication
import org.springframework.context.ApplicationContext
import org.springframework.context.annotation.*
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.core.env.Environment
import org.springframework.scheduling.annotation.EnableAsync
import org.springframework.scheduling.annotation.EnableScheduling
import reactor.core.scheduler.Scheduler
import reactor.core.scheduler.Schedulers
import java.io.File
import java.text.SimpleDateFormat
import java.util.*
import java.util.concurrent.Executors
import kotlin.system.exitProcess
@@ -43,8 +43,8 @@ import kotlin.system.exitProcess
@EnableScheduling
@EnableAsync
open class Config(
@Autowired private val env: Environment,
@Autowired private val ctx: ApplicationContext
@Autowired private val env: Environment,
@Autowired private val ctx: ApplicationContext
) {
companion object {
@@ -68,14 +68,15 @@ open class Config(
if (!FileResolver.isAccessible(target)) {
target = File(LOCAL_CONFIG)
if (!FileResolver.isAccessible(target)) {
throw IllegalStateException("Configuration is not found neither at ${DEFAULT_CONFIG} nor ${LOCAL_CONFIG}")
throw IllegalStateException("Configuration is not found neither at $DEFAULT_CONFIG nor $LOCAL_CONFIG")
}
}
target = target.normalize()
return target
}
@Bean @Qualifier("upstreamScheduler")
@Bean
@Qualifier("upstreamScheduler")
open fun upstreamScheduler(): Scheduler {
return Schedulers.fromExecutorService(Executors.newFixedThreadPool(16))
}
@@ -91,7 +92,7 @@ open class Config(
}
val reader = MainConfigReader(fileResolver)
return reader.read(f.inputStream())
?: throw IllegalStateException("Config is not available at ${f.absolutePath}")
?: throw IllegalStateException("Config is not available at ${f.absolutePath}")
}
@Bean
@@ -119,5 +120,4 @@ open class Config(
open fun monitoringConfig(@Autowired mainConfig: MainConfig): MonitoringConfig {
return mainConfig.monitoring
}
}
}

View File

@@ -25,4 +25,4 @@ class Defaults {
val timeoutInternal: Duration = timeout.dividedBy(4)
val retryConnection: Duration = Duration.ofSeconds(10)
}
}
}

View File

@@ -19,7 +19,7 @@ package io.emeraldpay.dshackle
import java.io.File
open class FileResolver(
private val baseDir: File
private val baseDir: File
) {
companion object {
@@ -35,5 +35,4 @@ open class FileResolver(
}
return File(baseDir, path)
}
}
}

View File

@@ -28,7 +28,7 @@ import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspentDeserializer
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import java.text.SimpleDateFormat
import java.util.*
import java.util.TimeZone
import java.util.concurrent.Executors
import java.util.concurrent.ScheduledExecutorService
@@ -57,12 +57,10 @@ class Global {
objectMapper.registerModule(JavaTimeModule())
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
objectMapper
.setDateFormat(SimpleDateFormat("yyyy-MM-dd\'T\'HH:mm:ss.SSS"))
.setTimeZone(TimeZone.getTimeZone("UTC"))
.setDateFormat(SimpleDateFormat("yyyy-MM-dd\'T\'HH:mm:ss.SSS"))
.setTimeZone(TimeZone.getTimeZone("UTC"))
return objectMapper
}
}
}
}

View File

@@ -18,7 +18,7 @@ package io.emeraldpay.dshackle
import io.emeraldpay.dshackle.config.MainConfig
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerGrpc
import io.grpc.*
import io.grpc.Server
import io.grpc.netty.NettyServerBuilder
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
@@ -29,15 +29,15 @@ import javax.annotation.PreDestroy
@Service
open class GrpcServer(
@Autowired val rpcs: List<io.grpc.BindableService>,
@Autowired val mainConfig: MainConfig,
@Autowired val tlsSetup: TlsSetup,
@Autowired val accessHandler: AccessHandlerGrpc
@Autowired val rpcs: List<io.grpc.BindableService>,
@Autowired val mainConfig: MainConfig,
@Autowired val tlsSetup: TlsSetup,
@Autowired val accessHandler: AccessHandlerGrpc
) {
private val log = LoggerFactory.getLogger(GrpcServer::class.java)
private var server: Server? = null;
private var server: Server? = null
@PostConstruct
fun start() {
@@ -45,14 +45,14 @@ open class GrpcServer(
log.debug("Running with DEBUG LOGGING")
log.info("Listening Native gRPC on ${mainConfig.host}:${mainConfig.port}")
val serverBuilder = NettyServerBuilder
.forAddress(InetSocketAddress(mainConfig.host, mainConfig.port))
.let {
if (mainConfig.accessLogConfig.enabled) {
it.intercept(accessHandler)
} else {
it
}
.forAddress(InetSocketAddress(mainConfig.host, mainConfig.port))
.let {
if (mainConfig.accessLogConfig.enabled) {
it.intercept(accessHandler)
} else {
it
}
}
tlsSetup.setupServer("Native gRPC", mainConfig.tls, true)?.let {
serverBuilder.sslContext(it)
@@ -75,5 +75,4 @@ open class GrpcServer(
server?.shutdownNow()
log.info("GRPC Server shot down")
}
}
}

View File

@@ -17,7 +17,6 @@
package io.emeraldpay.dshackle
import io.emeraldpay.dshackle.config.MainConfig
import io.emeraldpay.dshackle.config.ProxyConfig
import io.emeraldpay.dshackle.monitoring.MonitoringSetup
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp
import io.emeraldpay.dshackle.proxy.ProxyServer
@@ -26,8 +25,6 @@ import io.emeraldpay.dshackle.proxy.WriteRpcJson
import io.emeraldpay.dshackle.rpc.NativeCall
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.context.annotation.DependsOn
import org.springframework.core.env.Environment
import org.springframework.stereotype.Service
import javax.annotation.PostConstruct
@@ -36,14 +33,14 @@ import javax.annotation.PostConstruct
*/
@Service
class ProxyStarter(
@Autowired private val mainConfig: MainConfig,
@Autowired private val readRpcJson: ReadRpcJson,
@Autowired private val writeRpcJson: WriteRpcJson,
@Autowired private val nativeCall: NativeCall,
@Autowired private val tlsSetup: TlsSetup,
@Autowired private val accessHandlerHttp: AccessHandlerHttp,
// depend on Monitoring, declared here just to ensure it's properly initialized before the Proxy
@Autowired private val monitoringSetup: MonitoringSetup
@Autowired private val mainConfig: MainConfig,
@Autowired private val readRpcJson: ReadRpcJson,
@Autowired private val writeRpcJson: WriteRpcJson,
@Autowired private val nativeCall: NativeCall,
@Autowired private val tlsSetup: TlsSetup,
@Autowired private val accessHandlerHttp: AccessHandlerHttp,
// depend on Monitoring, declared here just to ensure it's properly initialized before the Proxy
@Autowired private val monitoringSetup: MonitoringSetup
) {
companion object {
@@ -60,5 +57,4 @@ class ProxyStarter(
val server = ProxyServer(config, readRpcJson, writeRpcJson, nativeCall, tlsSetup, accessHandlerHttp.factory)
server.start()
}
}
}

View File

@@ -22,7 +22,6 @@ import io.emeraldpay.grpc.Chain
*/
open class SilentException(message: String) : Exception(message) {
/**
* Blockchain is not available or not supported by current instance of the Dshackle
*/
@@ -31,4 +30,4 @@ open class SilentException(message: String) : Exception(message) {
}
class DataUnavailable(val code: String) : SilentException("Data is unavailable: $code")
}
}

View File

@@ -24,7 +24,7 @@ import org.springframework.context.annotation.Import
import org.springframework.core.io.ClassPathResource
import org.springframework.core.io.support.ResourcePropertySource
@SpringBootApplication(scanBasePackages = [ "io.emeraldpay.dshackle" ])
@SpringBootApplication(scanBasePackages = ["io.emeraldpay.dshackle"])
@Import(Config::class)
open class Starter
@@ -35,4 +35,4 @@ fun main(args: Array<String>) {
app.setDefaultProperties(ResourcePropertySource("version.properties").source)
app.setBanner(ResourceBanner(ClassPathResource("banner.txt")))
app.run(*args)
}
}

View File

@@ -29,7 +29,7 @@ import org.springframework.stereotype.Service
@Service
open class TlsSetup(
@Autowired val fileResolver: FileResolver
@Autowired val fileResolver: FileResolver
) {
companion object {
@@ -68,19 +68,19 @@ open class TlsSetup(
val sslContextBuilder = if (grpc) {
GrpcSslContexts.forServer(
fileResolver.resolve(config.certificate!!),
fileResolver.resolve(config.key!!)
fileResolver.resolve(config.certificate!!),
fileResolver.resolve(config.key!!)
)
} else {
SslContextBuilder.forServer(
fileResolver.resolve(config.certificate!!),
fileResolver.resolve(config.key!!)
fileResolver.resolve(config.certificate!!),
fileResolver.resolve(config.key!!)
)
}
if (StringUtils.isNotEmpty(config.clientCa)) {
log.info("Using TLS for client authentication for $category")
sslContextBuilder.trustManager(
fileResolver.resolve(config.clientCa!!)
fileResolver.resolve(config.clientCa!!)
)
if (config.clientRequire != null && config.clientRequire!!) {
sslContextBuilder.clientAuth(ClientAuth.REQUIRE)
@@ -96,5 +96,4 @@ open class TlsSetup(
}
return null
}
}
}

View File

@@ -25,8 +25,8 @@ import reactor.core.publisher.Mono
* Connects two caches to read through them. First is cache height->hash, second is hash->block.
*/
open class BlockByHeight(
private val heights: Reader<Long, BlockId>,
private val blocks: Reader<BlockId, BlockContainer>
private val heights: Reader<Long, BlockId>,
private val blocks: Reader<BlockId, BlockContainer>
) : Reader<Long, BlockContainer> {
companion object {
@@ -35,7 +35,6 @@ open class BlockByHeight(
override fun read(key: Long): Mono<BlockContainer> {
return heights.read(key)
.flatMap { blocks.read(it) }
.flatMap { blocks.read(it) }
}
}
}

View File

@@ -23,12 +23,12 @@ import io.emeraldpay.dshackle.reader.Reader
import reactor.core.publisher.Mono
open class BlocksMemCache(
maxSize: Int = 64
maxSize: Int = 64
) : Reader<BlockId, BlockContainer> {
private val mapping = Caffeine.newBuilder()
.maximumSize(maxSize.toLong())
.build<BlockId, BlockContainer>()
.maximumSize(maxSize.toLong())
.build<BlockId, BlockContainer>()
override fun read(key: BlockId): Mono<BlockContainer> {
return Mono.justOrEmpty(get(key))
@@ -45,4 +45,4 @@ open class BlocksMemCache(
open fun purge() {
mapping.cleanUp()
}
}
}

View File

@@ -32,10 +32,10 @@ import java.time.Instant
* Cache blocks in Redis database
*/
class BlocksRedisCache(
redis: RedisReactiveCommands<String, ByteArray>,
chain: Chain
redis: RedisReactiveCommands<String, ByteArray>,
chain: Chain
) : Reader<BlockId, BlockContainer>,
OnBlockRedisCache<BlockContainer>(redis, chain, CachesProto.ValueContainer.ValueType.BLOCK) {
OnBlockRedisCache<BlockContainer>(redis, chain, CachesProto.ValueContainer.ValueType.BLOCK) {
companion object {
private val log = LoggerFactory.getLogger(BlocksRedisCache::class.java)
@@ -59,21 +59,21 @@ class BlocksRedisCache(
}
val meta = value.blockMeta
return BlockContainer(
meta.height,
BlockId(meta.hash.toByteArray()),
BigInteger(meta.difficulty.toByteArray()),
Instant.ofEpochMilli(meta.timestamp),
false,
value.value.toByteArray(),
null,
meta.txHashesList.map {
TxId(it.toByteArray())
}
meta.height,
BlockId(meta.hash.toByteArray()),
BigInteger(meta.difficulty.toByteArray()),
Instant.ofEpochMilli(meta.timestamp),
false,
value.value.toByteArray(),
null,
meta.txHashesList.map {
TxId(it.toByteArray())
}
)
}
fun add(block: BlockContainer): Mono<Void> {
if (block.timestamp == null || block.hash == null) { //null in unit tests
if (block.timestamp == null || block.hash == null) { // null in unit tests
return Mono.empty()
}
if (block.full) {
@@ -81,5 +81,4 @@ class BlocksRedisCache(
}
return super.add(block, block)
}
}
}

View File

@@ -16,9 +16,12 @@
package io.emeraldpay.dshackle.cache
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.data.*
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.DefaultContainer
import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.reader.CompoundReader
import io.emeraldpay.dshackle.reader.EmptyReader
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.ethereum.EthereumFullBlocksReader
@@ -30,14 +33,14 @@ import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
open class Caches(
private val memBlocksByHash: BlocksMemCache,
private val blocksByHeight: HeightCache,
private val memTxsByHash: TxMemCache,
private val memReceipts: ReceiptMemCache,
private val redisBlocksByHash: BlocksRedisCache?,
private val redisTxsByHash: TxRedisCache?,
private val redisReceipts: ReceiptRedisCache?,
private val redisHeightByHashCache: HeightByHashRedisCache?
private val memBlocksByHash: BlocksMemCache,
private val blocksByHeight: HeightCache,
private val memTxsByHash: TxMemCache,
private val memReceipts: ReceiptMemCache,
private val redisBlocksByHash: BlocksRedisCache?,
private val redisTxsByHash: TxRedisCache?,
private val redisReceipts: ReceiptRedisCache?,
private val redisHeightByHashCache: HeightByHashRedisCache?
) {
companion object {
@@ -91,17 +94,17 @@ open class Caches(
if (currentHeight != null && data.height != null && memReceipts.acceptsRecentBlocks(currentHeight - data.height)) {
memReceipts.add(data)
}
//TODO move subscription to the caller
// TODO move subscription to the caller
redisReceipts?.add(data)?.subscribe()
}
fun cache(tag: Tag, tx: TxContainer) {
//do not cache transactions that are not in a block yet
// do not cache transactions that are not in a block yet
if (tx.blockId == null) {
return
}
memTxsByHash.add(tx)
//TODO move subscription to the caller
// TODO move subscription to the caller
getBlocksByHash().read(tx.blockId).flatMap { block ->
redisTxsByHash?.add(tx, block) ?: Mono.empty()
}.subscribe()
@@ -113,14 +116,14 @@ open class Caches(
redisHeightByHashCache?.add(block)?.let(job::add)
if (tag == Tag.LATEST) {
//for LATEST data cache it in memory, it may be short living so better to avoid Redis
// for LATEST data cache it in memory, it may be short living so better to avoid Redis
memoizeBlock(block)
} else if (tag == Tag.REQUESTED) {
val blockOnlyContainer: BlockContainer?
var jsonValue: BlockJson<*>? = null
if (block.full) {
jsonValue = Global.objectMapper.readValue<BlockJson<*>>(block.json, BlockJson::class.java)
//shouldn't cache block json with transactions, separate txes and blocks with refs
// shouldn't cache block json with transactions, separate txes and blocks with refs
val blockOnly = jsonValue.withoutTransactionDetails()
blockOnlyContainer = BlockContainer.from(blockOnly)
} else {
@@ -138,15 +141,17 @@ open class Caches(
TxContainer.from(tx)
}
if (redisTxsByHash != null) {
job.add(Flux.fromIterable(transactions)
job.add(
Flux.fromIterable(transactions)
.doOnNext { memTxsByHash.add(it) }
.flatMap { redisTxsByHash.add(it, block) }
.then())
.then()
)
}
}
}
}
Flux.fromIterable(job).flatMap { it }.subscribe() //TODO move out to a caller
Flux.fromIterable(job).flatMap { it }.subscribe() // TODO move out to a caller
}
/**
@@ -156,7 +161,7 @@ open class Caches(
memBlocksByHash.add(block)
memHeightByHash.add(block)
val replaced = blocksByHeight.add(block)
//evict cached transactions if an existing block was updated
// evict cached transactions if an existing block was updated
replaced?.let { evict(it) }
}
@@ -222,7 +227,7 @@ open class Caches(
REQUESTED
}
class Builder() {
class Builder {
private var blocksByHash: BlocksMemCache? = null
private var blocksByHeight: HeightCache? = null
private var txsByHash: TxMemCache? = null
@@ -285,8 +290,10 @@ open class Caches(
if (receipts == null) {
receipts = ReceiptMemCache()
}
return Caches(blocksByHash!!, blocksByHeight!!, txsByHash!!, receipts!!,
redisBlocksByHash, redisTxsByHash, redisReceiptCache, redisHeightByHashCache)
return Caches(
blocksByHash!!, blocksByHeight!!, txsByHash!!, receipts!!,
redisBlocksByHash, redisTxsByHash, redisReceiptCache, redisHeightByHashCache
)
}
}
}
}

View File

@@ -21,5 +21,4 @@ package io.emeraldpay.dshackle.cache
interface CachesEnabled {
fun setCaches(caches: Caches)
}
}

View File

@@ -27,14 +27,13 @@ import io.lettuce.core.codec.StringCodec
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Repository
import java.util.*
import java.util.EnumMap
import javax.annotation.PostConstruct
import kotlin.system.exitProcess
@Repository
open class CachesFactory(
@Autowired private val cacheConfig: CacheConfig
@Autowired private val cacheConfig: CacheConfig
) {
companion object {
@@ -50,14 +49,14 @@ open class CachesFactory(
val redisConfig = cacheConfig.redis ?: return
var uri = RedisURI.builder()
.withHost(redisConfig.host)
.withPort(redisConfig.port)
.withHost(redisConfig.host)
.withPort(redisConfig.port)
redisConfig.db?.let { value ->
uri = uri.withDatabase(value)
}
//log URI _before_ adding a password, to avoid leaking it to the log
// log URI _before_ adding a password, to avoid leaking it to the log
log.info("Use Redis cache at: ${uri.build().toURI()}")
redisConfig.password?.let { value ->
@@ -113,4 +112,4 @@ open class CachesFactory(
}
return existing
}
}
}

View File

@@ -20,5 +20,4 @@ class CurrentBlockCache<K, D> : Reader<K, D> {
fun evict() {
cache.set(ConcurrentHashMap())
}
}
}

View File

@@ -29,9 +29,9 @@ import reactor.core.publisher.Mono
*
*/
class HeightByHashAdding(
private val mem: Reader<BlockId, Long>,
private val redis: HeightByHashCache?,
private val upstreamReader: Reader<BlockId, BlockContainer>
private val mem: Reader<BlockId, Long>,
private val redis: HeightByHashCache?,
private val upstreamReader: Reader<BlockId, BlockContainer>
) : Reader<BlockId, Long> {
companion object {
@@ -39,7 +39,7 @@ class HeightByHashAdding(
}
constructor(caches: Caches, upstreamReader: Reader<BlockId, BlockContainer>) :
this(caches.getLastHeightByHash(), caches.getRedisHeightByHash(), upstreamReader)
this(caches.getLastHeightByHash(), caches.getRedisHeightByHash(), upstreamReader)
private val delegate: Reader<BlockId, Long>
@@ -48,26 +48,26 @@ class HeightByHashAdding(
delegate = object : Reader<BlockId, Long> {
override fun read(key: BlockId): Mono<Long> {
return mem.read(key)
.switchIfEmpty(
Mono.just(key)
.flatMap { redis.read(it) }
)
.switchIfEmpty(
Mono.just(key)
.flatMap { upstreamReader.read(it) }
.flatMap { redis.add(it).then(Mono.just(it.height)) }
)
.switchIfEmpty(
Mono.just(key)
.flatMap { redis.read(it) }
)
.switchIfEmpty(
Mono.just(key)
.flatMap { upstreamReader.read(it) }
.flatMap { redis.add(it).then(Mono.just(it.height)) }
)
}
}
} else {
delegate = object : Reader<BlockId, Long> {
override fun read(key: BlockId): Mono<Long> {
return mem.read(key)
.switchIfEmpty(
Mono.just(key)
.flatMap { upstreamReader.read(it) }
.map { it.height }
)
.switchIfEmpty(
Mono.just(key)
.flatMap { upstreamReader.read(it) }
.map { it.height }
)
}
}
}
@@ -76,5 +76,4 @@ class HeightByHashAdding(
override fun read(key: BlockId): Mono<Long> {
return delegate.read(key)
}
}
}

View File

@@ -23,4 +23,4 @@ import reactor.core.publisher.Mono
interface HeightByHashCache : Reader<BlockId, Long> {
fun add(block: BlockContainer): Mono<Void>
}
}

View File

@@ -23,7 +23,7 @@ import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono
class HeightByHashMemCache(
maxSize: Int = 256
maxSize: Int = 256
) : Reader<BlockId, Long> {
companion object {
@@ -31,8 +31,8 @@ class HeightByHashMemCache(
}
private val heights = Caffeine.newBuilder()
.maximumSize(maxSize.toLong())
.build<BlockId, Long>()
.maximumSize(maxSize.toLong())
.build<BlockId, Long>()
override fun read(key: BlockId): Mono<Long> {
return Mono.justOrEmpty(heights.getIfPresent(key))
@@ -41,4 +41,4 @@ class HeightByHashMemCache(
fun add(block: BlockContainer) {
heights.put(block.hash, block.height)
}
}
}

View File

@@ -32,8 +32,8 @@ import java.util.concurrent.TimeUnit
* reader would use full block's cache to find out height).
*/
class HeightByHashRedisCache(
private val redis: RedisReactiveCommands<String, ByteArray>,
private val chain: Chain
private val redis: RedisReactiveCommands<String, ByteArray>,
private val chain: Chain
) : Reader<BlockId, Long>, HeightByHashCache {
companion object {
@@ -45,41 +45,41 @@ class HeightByHashRedisCache(
@Suppress("UNCHECKED_CAST")
override fun read(key: BlockId): Mono<Long> {
return redis.get(key(key))
.flatMap { data ->
Mono.justOrEmpty(fromBytes(data)) as Mono<Long>
}.onErrorResume {
log.warn("Failed to read Block Height. ${it.javaClass}:${it.message}")
Mono.empty()
}
.flatMap { data ->
Mono.justOrEmpty(fromBytes(data)) as Mono<Long>
}.onErrorResume {
log.warn("Failed to read Block Height. ${it.javaClass}:${it.message}")
Mono.empty()
}
}
override fun add(block: BlockContainer): Mono<Void> {
return Mono.just(block)
.flatMap { blockData ->
// even if block replaced, the mapping hash-long is still valid, so can be cached for long time
// even for fresh blocks
val ttl = TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES)
.flatMap { blockData ->
// even if block replaced, the mapping hash-long is still valid, so can be cached for long time
// even for fresh blocks
val ttl = TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES)
val key = key(blockData.hash)
val value = asBytes(blockData.height)
redis.setex(key, ttl, value)
}
.doOnError {
log.warn("Failed to save Block Height. ${it.javaClass}:${it.message}")
}
//if failed to cache, just continue without it
.onErrorResume {
Mono.empty()
}
.then()
val key = key(blockData.hash)
val value = asBytes(blockData.height)
redis.setex(key, ttl, value)
}
.doOnError {
log.warn("Failed to save Block Height. ${it.javaClass}:${it.message}")
}
// if failed to cache, just continue without it
.onErrorResume {
Mono.empty()
}
.then()
}
fun asBytes(value: Long): ByteArray {
val result = ByteArray(8)
val bb = ByteBuffer.allocate(8)
.order(ByteOrder.BIG_ENDIAN)
.order(ByteOrder.BIG_ENDIAN)
bb.asLongBuffer()
.put(value)
.put(value)
bb.get(result)
return result
}
@@ -89,9 +89,9 @@ class HeightByHashRedisCache(
return null
}
return ByteBuffer.wrap(value)
.order(ByteOrder.BIG_ENDIAN)
.asLongBuffer()
.get()
.order(ByteOrder.BIG_ENDIAN)
.asLongBuffer()
.get()
}
/**
@@ -100,4 +100,4 @@ class HeightByHashRedisCache(
fun key(hash: BlockId): String {
return "height:${chain.id}:${hash.toHex()}"
}
}
}

View File

@@ -25,12 +25,12 @@ import reactor.core.publisher.Mono
* Memory cache for blocks heights, keeps mapping height->hash.
*/
open class HeightCache(
maxSize: Int = 512
maxSize: Int = 512
) : Reader<Long, BlockId> {
private val heights = Caffeine.newBuilder()
.maximumSize(maxSize.toLong())
.build<Long, BlockId>()
.maximumSize(maxSize.toLong())
.build<Long, BlockId>()
override fun read(key: Long): Mono<BlockId> {
return Mono.justOrEmpty(heights.getIfPresent(key))
@@ -45,5 +45,4 @@ open class HeightCache(
fun purge() {
heights.cleanUp()
}
}
}

View File

@@ -30,9 +30,9 @@ import java.util.concurrent.TimeUnit
import kotlin.math.min
abstract class OnBlockRedisCache<T>(
private val redis: RedisReactiveCommands<String, ByteArray>,
private val chain: Chain,
private val valueType: ValueContainer.ValueType
private val redis: RedisReactiveCommands<String, ByteArray>,
private val chain: Chain,
private val valueType: ValueContainer.ValueType
) : Reader<BlockId, T> {
companion object {
@@ -51,18 +51,18 @@ abstract class OnBlockRedisCache<T>(
fun toProto(block: BlockContainer, value: T): ValueContainer {
return ValueContainer.newBuilder()
.setType(valueType)
.setValue(ByteString.copyFrom(serializeValue(value)))
.setBlockMeta(buildMeta(block))
.build()
.setType(valueType)
.setValue(ByteString.copyFrom(serializeValue(value)))
.setBlockMeta(buildMeta(block))
.build()
}
open fun buildMeta(block: BlockContainer): CachesProto.BlockMeta.Builder {
return CachesProto.BlockMeta.newBuilder()
.setHash(ByteString.copyFrom(block.hash.value))
.setHeight(block.height)
.setDifficulty(ByteString.copyFrom(block.difficulty.toByteArray()))
.setTimestamp(block.timestamp.toEpochMilli())
.setHash(ByteString.copyFrom(block.hash.value))
.setHeight(block.height)
.setDifficulty(ByteString.copyFrom(block.difficulty.toByteArray()))
.setTimestamp(block.timestamp.toEpochMilli())
}
abstract fun serializeValue(value: T): ByteArray
@@ -83,7 +83,7 @@ abstract class OnBlockRedisCache<T>(
* Key in Redis
*/
fun key(hash: BlockId): String {
return "${prefix}:${chain.id}:${hash.toHex()}"
return "$prefix:${chain.id}:${hash.toHex()}"
}
/**
@@ -92,51 +92,51 @@ abstract class OnBlockRedisCache<T>(
*/
open fun add(container: BlockContainer, value: T): Mono<Void> {
return Mono.just(container)
.flatMap { block ->
val ttl = cachingTime(block.timestamp)
if (ttl > MIN_CACHE_TIME_SECONDS) {
val key = key(block.hash)
val proto = toProto(block, value)
redis.setex(key, ttl, proto.toByteArray())
} else {
Mono.empty()
}
}
.doOnError {
log.warn("Failed to save Block to Redis: ${it.message}")
}
//if failed to cache, just continue without it
.onErrorResume {
.flatMap { block ->
val ttl = cachingTime(block.timestamp)
if (ttl > MIN_CACHE_TIME_SECONDS) {
val key = key(block.hash)
val proto = toProto(block, value)
redis.setex(key, ttl, proto.toByteArray())
} else {
Mono.empty()
}
.then()
}
.doOnError {
log.warn("Failed to save Block to Redis: ${it.message}")
}
// if failed to cache, just continue without it
.onErrorResume {
Mono.empty()
}
.then()
}
/**
* Calculate time to cache the value
*/
fun cachingTime(blockTime: Instant): Long {
//default caching time is age of the block, i.e. block create hour ago
//keep for hour, but block created 10 seconds ago cache only for 10 seconds, because it
//still can be replaced in the blockchain
// default caching time is age of the block, i.e. block create hour ago
// keep for hour, but block created 10 seconds ago cache only for 10 seconds, because it
// still can be replaced in the blockchain
val age = Instant.now().epochSecond - blockTime.epochSecond
return min(age, TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES))
}
fun evict(id: BlockId): Mono<Void> {
return Mono.just(id)
.flatMap {
redis.del(key(it))
}
.then()
.flatMap {
redis.del(key(it))
}
.then()
}
override fun read(key: BlockId): Mono<T> {
return redis.get(key(key))
.map { data ->
fromProto(data)
}.onErrorResume {
Mono.empty()
}
.map { data ->
fromProto(data)
}.onErrorResume {
Mono.empty()
}
}
}
}

View File

@@ -30,9 +30,9 @@ import java.util.concurrent.TimeUnit
import kotlin.math.min
abstract class OnTxRedisCache<T>(
private val redis: RedisReactiveCommands<String, ByteArray>,
private val chain: Chain,
private val valueType: CachesProto.ValueContainer.ValueType
private val redis: RedisReactiveCommands<String, ByteArray>,
private val chain: Chain,
private val valueType: CachesProto.ValueContainer.ValueType
) : Reader<TxId, T> {
companion object {
@@ -56,42 +56,42 @@ abstract class OnTxRedisCache<T>(
* Key in Redis
*/
fun key(hash: TxId): String {
return "${prefix}:${chain.id}:${hash.toHex()}"
return "$prefix:${chain.id}:${hash.toHex()}"
}
fun evict(container: BlockContainer): Mono<Void> {
return Mono.just(container)
.map { block ->
block.transactions.map {
key(it)
}.toTypedArray()
}.flatMap { keys ->
redis.del(*keys)
}.then()
.map { block ->
block.transactions.map {
key(it)
}.toTypedArray()
}.flatMap { keys ->
redis.del(*keys)
}.then()
}
fun evict(id: TxId): Mono<Void> {
return Mono.just(id)
.flatMap {
redis.del(key(it))
}
.then()
.flatMap {
redis.del(key(it))
}
.then()
}
fun toProto(id: TxId, value: T): ByteArray {
val meta = buildMeta(id, value)
return CachesProto.ValueContainer.newBuilder()
.setType(valueType)
.setValue(ByteString.copyFrom(serializeValue(value)))
.setTxMeta(meta)
.build()
.toByteArray()
.setType(valueType)
.setValue(ByteString.copyFrom(serializeValue(value)))
.setTxMeta(meta)
.build()
.toByteArray()
}
open fun buildMeta(id: TxId, value: T): CachesProto.TxMeta.Builder {
return CachesProto.TxMeta.newBuilder()
.setHash(ByteString.copyFrom(id.value))
.setHash(ByteString.copyFrom(id.value))
}
abstract fun serializeValue(value: T): ByteArray
@@ -110,43 +110,43 @@ abstract class OnTxRedisCache<T>(
override fun read(key: TxId): Mono<T> {
return redis.get(key(key))
.map { data ->
fromProto(data)
}.onErrorResume {
Mono.empty()
}
.map { data ->
fromProto(data)
}.onErrorResume {
Mono.empty()
}
}
open fun add(id: TxId, value: T, block: BlockContainer?, blockHeight: Long?): Mono<Void> {
return Mono.just(id)
.flatMap {
val key = key(it)
val encodedValue = toProto(it, value)
val ttl = if (block?.timestamp != null) {
cachingTime(block.timestamp)
} else {
cachingTime(blockHeight)
}
//store
redis.setex(key, ttl, encodedValue)
.flatMap {
val key = key(it)
val encodedValue = toProto(it, value)
val ttl = if (block?.timestamp != null) {
cachingTime(block.timestamp)
} else {
cachingTime(blockHeight)
}
.doOnError {
log.warn("Failed to save TX to Redis: ${it.message}", it)
}
//if failed to cache, just continue without it
.onErrorResume {
Mono.empty()
}
.then()
// store
redis.setex(key, ttl, encodedValue)
}
.doOnError {
log.warn("Failed to save TX to Redis: ${it.message}", it)
}
// if failed to cache, just continue without it
.onErrorResume {
Mono.empty()
}
.then()
}
/**
* Calculate time to cache the value, based on block time
*/
fun cachingTime(blockTime: Instant): Long {
//default caching time is age of the block, i.e. block create hour ago
//keep for hour, but block create 10 seconds ago cache for 10 seconds, as it
//still can be replaced in the blockchain
// default caching time is age of the block, i.e. block create hour ago
// keep for hour, but block create 10 seconds ago cache for 10 seconds, as it
// still can be replaced in the blockchain
val age = Instant.now().epochSecond - blockTime.epochSecond
return min(age, TimeUnit.HOURS.toSeconds(MAX_CACHE_TIME_HOURS))
}
@@ -165,4 +165,4 @@ abstract class OnTxRedisCache<T>(
}
return min(confirmations * BLOCK_TIME_SECONDS, TimeUnit.HOURS.toSeconds(MAX_CACHE_TIME_HOURS))
}
}
}

View File

@@ -28,8 +28,8 @@ import reactor.core.publisher.Mono
* Keeps receipts for recent blocks in memory
*/
open class ReceiptMemCache(
// how many blocks to keeps in memory
val blocks: Int = 6
// how many blocks to keeps in memory
val blocks: Int = 6
) : Reader<TxId, ByteArray> {
companion object {
@@ -37,8 +37,8 @@ open class ReceiptMemCache(
}
private val mapping = Caffeine.newBuilder()
.maximumSize(blocks * 200L)
.build<TxId, ByteArray>()
.maximumSize(blocks * 200L)
.build<TxId, ByteArray>()
open fun evict(block: BlockContainer) {
block.transactions.forEach {
@@ -60,5 +60,4 @@ open class ReceiptMemCache(
open fun acceptsRecentBlocks(heightDelta: Long): Boolean {
return blocks <= heightDelta && heightDelta >= 0
}
}
}

View File

@@ -16,16 +16,15 @@
package io.emeraldpay.dshackle.cache
import io.emeraldpay.dshackle.data.DefaultContainer
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.proto.CachesProto
import io.emeraldpay.grpc.Chain
import io.emeraldpay.etherjar.rpc.json.TransactionReceiptJson
import io.emeraldpay.grpc.Chain
import io.lettuce.core.api.reactive.RedisReactiveCommands
import reactor.core.publisher.Mono
open class ReceiptRedisCache(
redis: RedisReactiveCommands<String, ByteArray>,
chain: Chain
redis: RedisReactiveCommands<String, ByteArray>,
chain: Chain
) : OnTxRedisCache<ByteArray>(redis, chain, CachesProto.ValueContainer.ValueType.TX_RECEIPT) {
override fun deserializeValue(value: CachesProto.ValueContainer): ByteArray {
@@ -39,4 +38,4 @@ open class ReceiptRedisCache(
fun add(json: DefaultContainer<TransactionReceiptJson>): Mono<Void> {
return super.add(json.txId!!, json.json!!, null, json.height)
}
}
}

View File

@@ -28,8 +28,8 @@ import reactor.core.publisher.Mono
* Memory cache for transactions
*/
open class TxMemCache(
// usually there is 100-150 tx per block on Ethereum, we keep data for about 32 blocks by default
private val maxSize: Int = 125 * 32
// usually there is 100-150 tx per block on Ethereum, we keep data for about 32 blocks by default
private val maxSize: Int = 125 * 32
) : Reader<TxId, TxContainer> {
companion object {
@@ -37,8 +37,8 @@ open class TxMemCache(
}
private val mapping = Caffeine.newBuilder()
.maximumSize(maxSize.toLong())
.build<TxId, TxContainer>()
.maximumSize(maxSize.toLong())
.build<TxId, TxContainer>()
override fun read(key: TxId): Mono<TxContainer> {
return Mono.justOrEmpty(mapping.getIfPresent(key))
@@ -52,13 +52,13 @@ open class TxMemCache(
open fun evict(block: BlockId) {
val ids = mapping.asMap()
.filter { it.value.blockId == block }
.map { it.key }
.filter { it.value.blockId == block }
.map { it.key }
mapping.invalidateAll(ids)
}
open fun add(tx: TxContainer) {
//do not cache fresh transactions
// do not cache fresh transactions
if (tx.blockId == null) {
return
}
@@ -68,4 +68,4 @@ open class TxMemCache(
open fun purge() {
mapping.cleanUp()
}
}
}

View File

@@ -15,31 +15,26 @@
*/
package io.emeraldpay.dshackle.cache
import com.fasterxml.jackson.databind.ObjectMapper
import com.google.protobuf.ByteString
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.proto.CachesProto
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.grpc.Chain
import io.emeraldpay.dshackle.proto.CachesProto
import io.lettuce.core.api.reactive.RedisReactiveCommands
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono
import reactor.util.function.Tuples
import java.time.Instant
import java.util.concurrent.TimeUnit
import kotlin.math.min
/**
* Cache transactions in Redis, up to 24 hours.
*/
open class TxRedisCache(
private val redis: RedisReactiveCommands<String, ByteArray>,
private val chain: Chain
private val redis: RedisReactiveCommands<String, ByteArray>,
private val chain: Chain
) : Reader<TxId, TxContainer>,
OnTxRedisCache<TxContainer>(redis, chain, CachesProto.ValueContainer.ValueType.TX) {
OnTxRedisCache<TxContainer>(redis, chain, CachesProto.ValueContainer.ValueType.TX) {
companion object {
private val log = LoggerFactory.getLogger(TxRedisCache::class.java)
@@ -67,15 +62,14 @@ open class TxRedisCache(
}
val meta = value.txMeta
return TxContainer(
meta.height,
TxId(meta.hash.toByteArray()),
BlockId(meta.blockHash.toByteArray()),
value.value.toByteArray()
meta.height,
TxId(meta.hash.toByteArray()),
BlockId(meta.blockHash.toByteArray()),
value.value.toByteArray()
)
}
open fun add(tx: TxContainer, block: BlockContainer): Mono<Void> {
return super.add(tx.hash, tx, block, tx.height)
}
}
}

View File

@@ -1,7 +1,7 @@
package io.emeraldpay.dshackle.config
class AccessLogConfig(
val enabled: Boolean = false
val enabled: Boolean = false
) {
var filename: String = "./access_log.jsonl"
@@ -14,9 +14,8 @@ class AccessLogConfig(
fun disabled(): AccessLogConfig {
return AccessLogConfig(
enabled = false
enabled = false
)
}
}
}
}

View File

@@ -23,5 +23,4 @@ class AccessLogReader : YamlConfigReader(), ConfigReader<AccessLogConfig> {
}
} ?: AccessLogConfig.default()
}
}
}

View File

@@ -29,14 +29,14 @@ class AuthConfig {
}
class ClientBasicAuth(
val username: String,
val password: String
val username: String,
val password: String
) : ClientAuth()
class ClientTlsAuth(
var ca: String? = null,
var certificate: String? = null,
var key: String? = null
var ca: String? = null,
var certificate: String? = null,
var key: String? = null
) : ClientAuth()
/**
@@ -58,4 +58,4 @@ class AuthConfig {
var clientRequire: Boolean? = null
var clientCa: String? = null
}
}
}

View File

@@ -82,5 +82,4 @@ class AuthConfigReader : YamlConfigReader() {
auth
}
}
}
}

View File

@@ -17,12 +17,12 @@ package io.emeraldpay.dshackle.config
class CacheConfig {
var redis: Redis? = null;
var redis: Redis? = null
class Redis(
var host: String = "127.0.0.1",
var port: Int = 6379,
var db: Int? = 0,
var password: String? = null
var host: String = "127.0.0.1",
var port: Int = 6379,
var db: Int? = 0,
var password: String? = null
)
}
}

View File

@@ -58,4 +58,4 @@ class CacheConfigReader : YamlConfigReader(), ConfigReader<CacheConfig> {
config
}
}
}
}

View File

@@ -20,5 +20,4 @@ import org.yaml.snakeyaml.nodes.MappingNode
interface ConfigReader<T> {
fun read(input: MappingNode?): T?
}
}

View File

@@ -31,4 +31,4 @@ class EnvVariables {
} ?: ""
}
}
}
}

View File

@@ -19,11 +19,11 @@ package io.emeraldpay.dshackle.config
import org.yaml.snakeyaml.error.Mark
open class InvalidConfigException(
message: String
message: String
) : Exception(message)
class InvalidConfigYamlException(
filename: String,
mark: Mark,
message: String
) : InvalidConfigException("Invalid YAML configuration ${message}, at ${filename}:${mark.line}")
filename: String,
mark: Mark,
message: String
) : InvalidConfigException("Invalid YAML configuration $message, at $filename:${mark.line}")

View File

@@ -25,4 +25,4 @@ class MainConfig {
var tokens: TokensConfig? = null
var monitoring: MonitoringConfig = MonitoringConfig.default()
var accessLogConfig: AccessLogConfig = AccessLogConfig.default()
}
}

View File

@@ -21,7 +21,7 @@ import org.yaml.snakeyaml.nodes.MappingNode
import java.io.InputStream
class MainConfigReader(
fileResolver: FileResolver
fileResolver: FileResolver
) : YamlConfigReader(), ConfigReader<MainConfig> {
companion object {
@@ -73,5 +73,4 @@ class MainConfigReader(
}
return config
}
}
}

View File

@@ -16,14 +16,15 @@
package io.emeraldpay.dshackle.config
class MonitoringConfig(
val enabled: Boolean,
val prometheus: PrometheusConfig
val enabled: Boolean,
val prometheus: PrometheusConfig
) {
companion object {
fun default(): MonitoringConfig {
return MonitoringConfig(true, PrometheusConfig.default())
}
fun disabled(): MonitoringConfig {
return MonitoringConfig(false, PrometheusConfig.disabled())
}
@@ -33,19 +34,19 @@ class MonitoringConfig(
var enableExtended: Boolean = false
data class PrometheusConfig(
val enabled: Boolean,
val path: String,
val host: String,
val port: Int
val enabled: Boolean,
val path: String,
val host: String,
val port: Int
) {
companion object {
fun default(): PrometheusConfig {
return PrometheusConfig(true, "/metrics", "127.0.0.1", 8081)
}
fun disabled(): PrometheusConfig {
return PrometheusConfig(false, "/", "127.0.0.1", 0)
}
}
}
}
}

View File

@@ -19,7 +19,7 @@ import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.nodes.MappingNode
import java.io.InputStream
class MonitoringConfigReader: YamlConfigReader(), ConfigReader<MonitoringConfig> {
class MonitoringConfigReader : YamlConfigReader(), ConfigReader<MonitoringConfig> {
companion object {
private val log = LoggerFactory.getLogger(MonitoringConfigReader::class.java)
@@ -64,5 +64,4 @@ class MonitoringConfigReader: YamlConfigReader(), ConfigReader<MonitoringConfig>
val port = getValueAsInt(input, "port") ?: default.port
return MonitoringConfig.PrometheusConfig(enabled, path, host, port)
}
}
}

View File

@@ -24,7 +24,7 @@ import io.emeraldpay.grpc.Chain
open class ProxyConfig {
companion object {
public const val CONFIG_ID = "parsed.proxy"
const val CONFIG_ID = "parsed.proxy"
}
var enabled: Boolean = true
@@ -50,13 +50,13 @@ open class ProxyConfig {
var routes: List<Route> = ArrayList()
class Route(
/**
* URL binding for the route. http://$host:$port/$id
*/
val id: String,
/**
* Blockchain to dispatch requests
*/
val blockchain: Chain
/**
* URL binding for the route. http://$host:$port/$id
*/
val id: String,
/**
* Blockchain to dispatch requests
*/
val blockchain: Chain
)
}
}

View File

@@ -19,10 +19,8 @@ package io.emeraldpay.dshackle.config
import io.emeraldpay.grpc.Chain
import org.apache.commons.lang3.StringUtils
import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.Yaml
import org.yaml.snakeyaml.nodes.MappingNode
import java.io.InputStream
import java.io.InputStreamReader
/**
* Read YAML config, part related to Proxy configuration
@@ -84,5 +82,4 @@ class ProxyConfigReader : YamlConfigReader(), ConfigReader<ProxyConfig> {
config.tls = authConfigReader.readServerTls(input)
return config
}
}
}

View File

@@ -15,12 +15,12 @@
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.etherjar.domain.Address
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import io.emeraldpay.etherjar.domain.Address
class TokensConfig(
val tokens: List<Token>
val tokens: List<Token>
) {
class Token {
@@ -30,7 +30,7 @@ class TokensConfig(
// coin name
var name: String? = null
var type: Type? = null;
var type: Type? = null
var address: String? = null
fun validate(): String? {
@@ -40,9 +40,9 @@ class TokensConfig(
name.isNullOrBlank() -> "name"
type == null -> type
address.isNullOrBlank() -> "address"
blockchain != null
&& BlockchainType.from(blockchain!!) == BlockchainType.ETHEREUM
&& !Address.isValidAddress(address) -> "address"
blockchain != null &&
BlockchainType.from(blockchain!!) == BlockchainType.ETHEREUM &&
!Address.isValidAddress(address) -> "address"
else -> null
}
}
@@ -51,5 +51,4 @@ class TokensConfig(
enum class Type {
ERC20
}
}
}

View File

@@ -18,7 +18,7 @@ package io.emeraldpay.dshackle.config
import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.nodes.MappingNode
import java.io.InputStream
import java.util.*
import java.util.Locale
class TokensConfigReader : YamlConfigReader(), ConfigReader<TokensConfig> {
@@ -60,5 +60,4 @@ class TokensConfigReader : YamlConfigReader(), ConfigReader<TokensConfig> {
TokensConfig(it)
}
}
}
}

View File

@@ -17,11 +17,9 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Defaults
import java.lang.ClassCastException
import java.net.URI
import java.util.*
import kotlin.collections.ArrayList
import kotlin.collections.HashMap
import java.util.Arrays
import java.util.Locale
open class UpstreamsConfig {
var defaultOptions: MutableList<DefaultOptions> = ArrayList<DefaultOptions>()
@@ -46,8 +44,10 @@ open class UpstreamsConfig {
}
val copy = Options()
copy.minPeers = if (this.minPeers != null) this.minPeers else additional.minPeers
copy.disableValidation = if (this.disableValidation != null) this.disableValidation else additional.disableValidation
copy.providesBalance = if (this.providesBalance != null) this.providesBalance else additional.providesBalance
copy.disableValidation =
if (this.disableValidation != null) this.disableValidation else additional.disableValidation
copy.providesBalance =
if (this.providesBalance != null) this.providesBalance else additional.providesBalance
return copy
}
@@ -60,7 +60,6 @@ open class UpstreamsConfig {
return options
}
}
}
class DefaultOptions : Options() {
@@ -125,14 +124,14 @@ open class UpstreamsConfig {
var msgSize: Int? = null
}
//TODO make it unmodifiable after initial load
class Labels: HashMap<String, String>() {
// TODO make it unmodifiable after initial load
class Labels : HashMap<String, String>() {
companion object {
@JvmStatic fun fromMap(map: Map<String, String>): Labels {
@JvmStatic
fun fromMap(map: Map<String, String>): Labels {
val labels = Labels()
map.entries.forEach() { kv ->
map.entries.forEach { kv ->
labels.put(kv.key, kv.value)
}
return labels
@@ -140,7 +139,7 @@ open class UpstreamsConfig {
}
}
enum class UpstreamType private constructor(vararg code: String) {
enum class UpstreamType(vararg code: String) {
ETHEREUM_JSON_RPC("ethereum"),
BITCOIN_JSON_RPC("bitcoin"),
DSHACKLE("dshackle", "grpc"),
@@ -168,12 +167,12 @@ open class UpstreamsConfig {
}
class Methods(
val enabled: Set<Method>,
val disabled: Set<Method>
val enabled: Set<Method>,
val disabled: Set<Method>
)
class Method(
val name: String,
val quorum: String? = null
val name: String,
val quorum: String? = null
)
}
}

View File

@@ -23,14 +23,12 @@ import org.yaml.snakeyaml.nodes.MappingNode
import org.yaml.snakeyaml.nodes.ScalarNode
import reactor.util.function.Tuples
import java.io.InputStream
import java.lang.IllegalArgumentException
import java.net.URI
import java.time.Duration
import java.util.*
import kotlin.collections.ArrayList
import java.util.Locale
class UpstreamsConfigReader(
private val fileResolver: FileResolver
private val fileResolver: FileResolver
) : YamlConfigReader(), ConfigReader<UpstreamsConfig> {
private val log = LoggerFactory.getLogger(UpstreamsConfigReader::class.java)
@@ -198,7 +196,10 @@ class UpstreamsConfigReader(
upstream.methods = tryReadMethods(upNode)
}
internal fun readUpstreamGrpc(upNode: MappingNode, upstream: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>) {
internal fun readUpstreamGrpc(
upNode: MappingNode,
upstream: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>
) {
if (hasAny(upNode, "labels")) {
log.warn("Labels should be not applied to gRPC upstream")
}
@@ -221,13 +222,13 @@ class UpstreamsConfigReader(
if (hasAny(upNode, "labels")) {
getMapping(upNode, "labels")?.let { labels ->
labels.value.stream()
.filter { n -> n.keyNode is ScalarNode && n.valueNode is ScalarNode }
.map { n -> Tuples.of((n.keyNode as ScalarNode).value, (n.valueNode as ScalarNode).value) }
.map { kv -> Tuples.of(kv.t1.trim(), kv.t2.trim()) }
.filter { kv -> StringUtils.isNotEmpty(kv.t1) && StringUtils.isNotEmpty(kv.t2) }
.forEach { kv ->
upstream.labels[kv.t1] = kv.t2
}
.filter { n -> n.keyNode is ScalarNode && n.valueNode is ScalarNode }
.map { n -> Tuples.of((n.keyNode as ScalarNode).value, (n.valueNode as ScalarNode).value) }
.map { kv -> Tuples.of(kv.t1.trim(), kv.t2.trim()) }
.filter { kv -> StringUtils.isNotEmpty(kv.t1) && StringUtils.isNotEmpty(kv.t2) }
.forEach { kv ->
upstream.labels[kv.t1] = kv.t2
}
}
}
}
@@ -247,21 +248,21 @@ class UpstreamsConfigReader(
val enabled = getList<MappingNode>(mnode, "enabled")?.value?.map { m ->
getValueAsString(m, "name")?.let { name ->
UpstreamsConfig.Method(
name = name,
quorum = getValueAsString(m, "quorum")
name = name,
quorum = getValueAsString(m, "quorum")
)
}
}?.filterNotNull()?.toSet() ?: emptySet()
val disabled = getList<MappingNode>(mnode, "disabled")?.value?.map { m ->
getValueAsString(m, "name")?.let { name ->
UpstreamsConfig.Method(
name = name
name = name
)
}
}?.filterNotNull()?.toSet() ?: emptySet()
UpstreamsConfig.Methods(
enabled, disabled
enabled, disabled
)
}
}
@@ -282,5 +283,4 @@ class UpstreamsConfigReader(
}
return options
}
}
}

View File

@@ -24,7 +24,7 @@ import org.yaml.snakeyaml.nodes.Node
import org.yaml.snakeyaml.nodes.ScalarNode
import java.io.InputStream
import java.io.InputStreamReader
import java.util.*
import java.util.Locale
abstract class YamlConfigReader {
private val envVariables = EnvVariables()
@@ -43,12 +43,12 @@ abstract class YamlConfigReader {
return false
}
return mappingNode.value
.stream()
.filter { n -> n.keyNode is ScalarNode }
.filter { n ->
val sn = n.keyNode as ScalarNode
key == sn.value
}.count() > 0
.stream()
.filter { n -> n.keyNode is ScalarNode }
.filter { n ->
val sn = n.keyNode as ScalarNode
key == sn.value
}.count() > 0
}
@Suppress("UNCHECKED_CAST")
@@ -57,20 +57,20 @@ abstract class YamlConfigReader {
return null
}
return mappingNode.value
.stream()
.filter { n -> n.keyNode is ScalarNode && type.isAssignableFrom(n.valueNode.javaClass) }
.filter { n ->
val sn = n.keyNode as ScalarNode
key == sn.value
}
.map { n -> n.valueNode as T }
.findFirst().let {
if (it.isPresent) {
it.get()
} else {
null
}
.stream()
.filter { n -> n.keyNode is ScalarNode && type.isAssignableFrom(n.valueNode.javaClass) }
.filter { n ->
val sn = n.keyNode as ScalarNode
key == sn.value
}
.map { n -> n.valueNode as T }
.findFirst().let {
if (it.isPresent) {
it.get()
} else {
null
}
}
}
protected fun getMapping(mappingNode: MappingNode?, key: String): MappingNode? {
@@ -89,8 +89,8 @@ abstract class YamlConfigReader {
protected fun getListOfString(mappingNode: MappingNode?, key: String): List<String>? {
return getList<ScalarNode>(mappingNode, key)?.value
?.map { it.value }
?.map(envVariables::postProcess)
?.map { it.value }
?.map(envVariables::postProcess)
}
protected fun getValueAsString(mappingNode: MappingNode?, key: String): String? {
@@ -130,7 +130,7 @@ abstract class YamlConfigReader {
fun getValueAsBytes(mappingNode: MappingNode?, key: String): Int? {
return getValueAsString(mappingNode, key)?.let(envVariables::postProcess)?.let {
val m = Regex("^(\\d+)(m|mb|k|kb|b)?$").find(it.lowercase().trim())
?: throw IllegalArgumentException("Not a data size: ${it}. Example of correct values: '1024', '1kb', '5mb'")
?: throw IllegalArgumentException("Not a data size: $it. Example of correct values: '1024', '1kb', '5mb'")
val multiplier = m.groups[2]?.let {
when (it.value) {
"k", "kb" -> 1024
@@ -147,9 +147,9 @@ abstract class YamlConfigReader {
fun getBlockchain(id: String): Chain {
return Chain.values().find { chain ->
chain.name == id.uppercase(Locale.getDefault())
|| chain.chainCode.uppercase(Locale.getDefault()) == id.uppercase(Locale.getDefault())
|| chain.id.toString() == id
chain.name == id.uppercase(Locale.getDefault()) ||
chain.chainCode.uppercase(Locale.getDefault()) == id.uppercase(Locale.getDefault()) ||
chain.id.toString() == id
} ?: Chain.UNSPECIFIED
}
}
}

View File

@@ -23,14 +23,14 @@ import java.math.BigInteger
import java.time.Instant
class BlockContainer(
val height: Long,
val hash: BlockId,
val difficulty: BigInteger,
val timestamp: Instant,
val full: Boolean,
json: ByteArray?,
val parsed: Any?,
val transactions: List<TxId> = emptyList()
val height: Long,
val hash: BlockId,
val difficulty: BigInteger,
val timestamp: Instant,
val full: Boolean,
json: ByteArray?,
val parsed: Any?,
val transactions: List<TxId> = emptyList()
) : SourceContainer(json, parsed) {
companion object {
@@ -38,14 +38,14 @@ class BlockContainer(
fun from(block: BlockJson<*>, raw: ByteArray): BlockContainer {
val hasTransactions = block.transactions?.filterIsInstance<TransactionJson>()?.count() ?: 0 > 0
return BlockContainer(
block.number,
BlockId.from(block),
block.totalDifficulty,
block.timestamp,
hasTransactions,
raw,
block,
block.transactions?.map { TxId.from(it.hash) } ?: emptyList()
block.number,
BlockId.from(block),
block.totalDifficulty,
block.timestamp,
hasTransactions,
raw,
block,
block.transactions?.map { TxId.from(it.hash) } ?: emptyList()
)
}
@@ -88,6 +88,4 @@ class BlockContainer(
result = 31 * result + hash.hashCode()
return result
}
}
}

View File

@@ -21,7 +21,7 @@ import io.emeraldpay.etherjar.rpc.json.BlockJson
import org.bouncycastle.util.encoders.Hex
class BlockId(
value: ByteArray
value: ByteArray
) : HashId(value) {
companion object {
@@ -51,6 +51,4 @@ class BlockId(
return BlockId(bytes)
}
}
}
}

View File

@@ -18,14 +18,14 @@ package io.emeraldpay.dshackle.data
import org.slf4j.LoggerFactory
class DefaultContainer<T>(
val txId: TxId?,
val blockId: BlockId?,
val height: Long?,
json: ByteArray,
parsed: T
val txId: TxId?,
val blockId: BlockId?,
val height: Long?,
json: ByteArray,
parsed: T
) : SourceContainer(json, parsed) {
companion object {
private val log = LoggerFactory.getLogger(DefaultContainer::class.java)
}
}
}

View File

@@ -17,7 +17,7 @@
package io.emeraldpay.dshackle.data
open class HashId(
val value: ByteArray
val value: ByteArray
) {
companion object {
@@ -56,6 +56,4 @@ open class HashId(
override fun hashCode(): Int {
return value.contentHashCode()
}
}
}

View File

@@ -36,7 +36,7 @@ class RawJsonBuilder {
buf.write(START)
buf.write(COMMA)
buf.write(ID_START)
buf.write(id.toString().toByteArray());
buf.write(id.toString().toByteArray())
buf.write(COMMA)
buf.write(RESULT_START)
buf.write(data)
@@ -44,6 +44,4 @@ class RawJsonBuilder {
return buf.toByteArray()
}
}
}

View File

@@ -16,11 +16,9 @@
*/
package io.emeraldpay.dshackle.data
import java.lang.ClassCastException
abstract class SourceContainer(
val json: ByteArray?,
private val parsed: Any?
val json: ByteArray?,
private val parsed: Any?
) {
@Suppress("UNCHECKED_CAST")
@@ -34,7 +32,6 @@ abstract class SourceContainer(
throw ClassCastException("Cannot cast ${parsed.javaClass} to $clazz")
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other !is SourceContainer) return false
@@ -50,4 +47,4 @@ abstract class SourceContainer(
override fun hashCode(): Int {
return json?.contentHashCode() ?: 0
}
}
}

View File

@@ -20,11 +20,11 @@ import io.emeraldpay.dshackle.Global
import io.emeraldpay.etherjar.rpc.json.TransactionJson
class TxContainer(
val height: Long?,
val hash: TxId,
val blockId: BlockId?,
json: ByteArray?,
parsed: Any? = null
val height: Long?,
val hash: TxId,
val blockId: BlockId?,
json: ByteArray?,
parsed: Any? = null
) : SourceContainer(json, parsed) {
companion object {
@@ -41,11 +41,11 @@ class TxContainer(
fun from(tx: TransactionJson, raw: ByteArray): TxContainer {
return TxContainer(
tx.blockNumber,
TxId.from(tx.hash),
tx.blockHash?.let { BlockId.from(it) },
raw,
tx
tx.blockNumber,
TxId.from(tx.hash),
tx.blockHash?.let { BlockId.from(it) },
raw,
tx
)
}
}
@@ -70,6 +70,4 @@ class TxContainer(
result = 31 * result + hash.hashCode()
return result
}
}
}

View File

@@ -19,10 +19,9 @@ package io.emeraldpay.dshackle.data
import io.emeraldpay.etherjar.domain.TransactionId
import io.emeraldpay.etherjar.rpc.json.TransactionJson
import org.bouncycastle.util.encoders.Hex
import java.math.BigInteger
class TxId(
value: ByteArray
value: ByteArray
) : HashId(value) {
companion object {
@@ -47,4 +46,4 @@ class TxId(
return TxId(bytes)
}
}
}
}

View File

@@ -26,20 +26,18 @@ import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics
import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics
import io.micrometer.core.instrument.binder.system.ProcessorMetrics
import io.micrometer.core.instrument.config.MeterFilter
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service
import io.micrometer.prometheus.PrometheusConfig
import io.micrometer.prometheus.PrometheusMeterRegistry
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
import java.io.IOException
import java.net.InetSocketAddress
import javax.annotation.PostConstruct
@Service
class MonitoringSetup(
@Autowired private val monitoringConfig: MonitoringConfig
@Autowired private val monitoringConfig: MonitoringConfig
) {
companion object {
@@ -50,7 +48,7 @@ class MonitoringSetup(
fun setup() {
val prometheusRegistry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT)
Metrics.globalRegistry.add(prometheusRegistry)
Metrics.globalRegistry.config().meterFilter(object: MeterFilter {
Metrics.globalRegistry.config().meterFilter(object : MeterFilter {
override fun map(id: Meter.Id): Meter.Id {
if (id.name.startsWith("jvm") || id.name.startsWith("process") || id.name.startsWith("system")) {
return id
@@ -76,18 +74,24 @@ class MonitoringSetup(
// prometheus is a single thread periodic call, no reason to setup anything complex
try {
log.info("Run Prometheus metrics on ${monitoringConfig.prometheus.host}:${monitoringConfig.prometheus.port}${monitoringConfig.prometheus.path}")
val server = HttpServer.create(InetSocketAddress(monitoringConfig.prometheus.host, monitoringConfig.prometheus.port), 0);
val server = HttpServer.create(
InetSocketAddress(
monitoringConfig.prometheus.host,
monitoringConfig.prometheus.port
),
0
)
server.createContext(monitoringConfig.prometheus.path) { httpExchange ->
val response = prometheusRegistry.scrape()
httpExchange.sendResponseHeaders(200, response.toByteArray().size.toLong());
httpExchange.sendResponseHeaders(200, response.toByteArray().size.toLong())
httpExchange.responseBody.use { os ->
os.write(response.toByteArray())
}
}
Thread(server::start).start();
Thread(server::start).start()
} catch (e: IOException) {
log.error("Failed to start Prometheus Server", e)
}
}
}
}
}

View File

@@ -15,14 +15,20 @@
*/
package io.emeraldpay.dshackle.monitoring.accesslog
import io.grpc.*
import io.grpc.ForwardingServerCall
import io.grpc.ForwardingServerCallListener
import io.grpc.Metadata
import io.grpc.MethodDescriptor
import io.grpc.ServerCall
import io.grpc.ServerCallHandler
import io.grpc.ServerInterceptor
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
@Service
class AccessHandlerGrpc(
@Autowired private val accessLogWriter: AccessLogWriter
@Autowired private val accessLogWriter: AccessLogWriter
) : ServerInterceptor {
companion object {
@@ -30,9 +36,10 @@ class AccessHandlerGrpc(
}
override fun <ReqT : Any, RespT : Any> interceptCall(
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>): ServerCall.Listener<ReqT> {
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>
): ServerCall.Listener<ReqT> {
return when (val method = call.methodDescriptor.bareMethodName) {
"SubscribeHead" -> processSubscribeHead(call, headers, next)
@@ -51,103 +58,109 @@ class AccessHandlerGrpc(
}
private fun <ReqT : Any, RespT : Any, E> process(
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>,
builder: EventsBuilder.RequestReply<E, ReqT, RespT>
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>,
builder: EventsBuilder.RequestReply<E, ReqT, RespT>
): ServerCall.Listener<ReqT> {
builder.start(headers, call.attributes)
val callWrapper: ServerCall<ReqT, RespT> = StdCallResponse(
call, builder, accessLogWriter
call, builder, accessLogWriter
)
return StdCallListener(
next.startCall(callWrapper, headers),
builder
next.startCall(callWrapper, headers),
builder
)
}
@Suppress("UNCHECKED_CAST")
private fun <ReqT : Any, RespT : Any> processSubscribeHead(
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>
): ServerCall.Listener<ReqT> {
return process(call, headers, next,
EventsBuilder.SubscribeHead() as EventsBuilder.RequestReply<*, ReqT, RespT>
return process(
call, headers, next,
EventsBuilder.SubscribeHead() as EventsBuilder.RequestReply<*, ReqT, RespT>
)
}
@Suppress("UNCHECKED_CAST")
private fun <ReqT : Any, RespT : Any> processSubscribeBalance(
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>,
subscribe: Boolean
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>,
subscribe: Boolean
): ServerCall.Listener<ReqT> {
return process(call, headers, next,
EventsBuilder.SubscribeBalance(subscribe) as EventsBuilder.RequestReply<*, ReqT, RespT>
return process(
call, headers, next,
EventsBuilder.SubscribeBalance(subscribe) as EventsBuilder.RequestReply<*, ReqT, RespT>
)
}
@Suppress("UNCHECKED_CAST")
private fun <ReqT : Any, RespT : Any> processSubscribeTxStatus(
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>
): ServerCall.Listener<ReqT> {
return process(call, headers, next,
EventsBuilder.TxStatus() as EventsBuilder.RequestReply<*, ReqT, RespT>
return process(
call, headers, next,
EventsBuilder.TxStatus() as EventsBuilder.RequestReply<*, ReqT, RespT>
)
}
@Suppress("UNCHECKED_CAST")
private fun <ReqT : Any, RespT : Any> processNativeCall(
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>
): ServerCall.Listener<ReqT> {
return process(call, headers, next,
EventsBuilder.NativeCall() as EventsBuilder.RequestReply<*, ReqT, RespT>
return process(
call, headers, next,
EventsBuilder.NativeCall() as EventsBuilder.RequestReply<*, ReqT, RespT>
)
}
@Suppress("UNCHECKED_CAST")
private fun <ReqT : Any, RespT : Any> processNativeSubscribe(
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>
): ServerCall.Listener<ReqT> {
return process(call, headers, next,
EventsBuilder.NativeSubscribe() as EventsBuilder.RequestReply<*, ReqT, RespT>
return process(
call, headers, next,
EventsBuilder.NativeSubscribe() as EventsBuilder.RequestReply<*, ReqT, RespT>
)
}
@Suppress("UNCHECKED_CAST")
private fun <ReqT : Any, RespT : Any> processDescribe(
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>
): ServerCall.Listener<ReqT> {
return process(call, headers, next,
EventsBuilder.Describe() as EventsBuilder.RequestReply<*, ReqT, RespT>
return process(
call, headers, next,
EventsBuilder.Describe() as EventsBuilder.RequestReply<*, ReqT, RespT>
)
}
@Suppress("UNCHECKED_CAST")
private fun <ReqT : Any, RespT : Any> processStatus(
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>
): ServerCall.Listener<ReqT> {
return process(call, headers, next,
EventsBuilder.Status() as EventsBuilder.RequestReply<*, ReqT, RespT>
return process(
call, headers, next,
EventsBuilder.Status() as EventsBuilder.RequestReply<*, ReqT, RespT>
)
}
open class StdCallListener<Req, EB : EventsBuilder.RequestReply<*, Req, *>>(
val next: ServerCall.Listener<Req>,
val builder: EB
val next: ServerCall.Listener<Req>,
val builder: EB
) : ForwardingServerCallListener<Req>() {
override fun onMessage(message: Req) {
@@ -161,9 +174,9 @@ class AccessHandlerGrpc(
}
open class StdCallResponse<ReqT : Any, RespT : Any, EB : EventsBuilder.RequestReply<*, ReqT, RespT>>(
val next: ServerCall<ReqT, RespT>,
val builder: EB,
val accessLogWriter: AccessLogWriter
val next: ServerCall<ReqT, RespT>,
val builder: EB,
val accessLogWriter: AccessLogWriter
) : ForwardingServerCall<ReqT, RespT>() {
override fun getMethodDescriptor(): MethodDescriptor<ReqT, RespT> {
@@ -177,9 +190,8 @@ class AccessHandlerGrpc(
override fun sendMessage(message: RespT) {
super.sendMessage(message)
accessLogWriter.submit(
builder.onReply(message)!!
builder.onReply(message)!!
)
}
}
}
}

View File

@@ -19,8 +19,8 @@ import kotlin.concurrent.withLock
*/
@Service
class AccessHandlerHttp(
@Autowired private val mainConfig: MainConfig,
@Autowired accessLogWriter: AccessLogWriter
@Autowired private val mainConfig: MainConfig,
@Autowired accessLogWriter: AccessLogWriter
) {
companion object {
@@ -40,7 +40,7 @@ class AccessHandlerHttp(
fun create(req: HttpServerRequest, blockchain: Chain): RequestHandler
}
class NoOpFactory() : HandlerFactory {
class NoOpFactory : HandlerFactory {
override fun create(req: HttpServerRequest, blockchain: Chain): RequestHandler {
return NoOpHandler()
}
@@ -70,9 +70,9 @@ class AccessHandlerHttp(
}
class StandardHandler(
private val accessLogWriter: AccessLogWriter,
private val httpRequest: HttpServerRequest,
private val blockchain: Chain
private val accessLogWriter: AccessLogWriter,
private val httpRequest: HttpServerRequest,
private val blockchain: Chain
) : RequestHandler {
private var request: BlockchainOuterClass.NativeCallRequest? = null
@@ -89,13 +89,13 @@ class AccessHandlerHttp(
builder.start(httpRequest)
builder.onRequest(request!!)
responses
.map {
builder.onReply(it, Events.Channel.JSONRPC).also { item ->
//since for JSON RPC you get a single response then the timestamp of all items included in it must have the same timestamp
item.ts = responseTime
}
.map {
builder.onReply(it, Events.Channel.JSONRPC).also { item ->
// since for JSON RPC you get a single response then the timestamp of all items included in it must have the same timestamp
item.ts = responseTime
}
.let(accessLogWriter::submit)
}
.let(accessLogWriter::submit)
}
override fun onRequest(request: BlockchainOuterClass.NativeCallRequest) {
@@ -108,4 +108,4 @@ class AccessHandlerHttp(
}
}
}
}
}

View File

@@ -20,7 +20,9 @@ import io.emeraldpay.dshackle.config.MainConfig
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Repository
import java.io.*
import java.io.BufferedOutputStream
import java.io.File
import java.io.FileOutputStream
import java.time.Duration
import java.time.Instant
import java.util.concurrent.ConcurrentLinkedQueue
@@ -30,7 +32,7 @@ import javax.annotation.PostConstruct
@Repository
class AccessLogWriter(
@Autowired mainConfig: MainConfig
@Autowired mainConfig: MainConfig
) {
companion object {
@@ -120,5 +122,4 @@ class AccessLogWriter(
}
}
}
}
}

View File

@@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonInclude
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import java.time.Instant
import java.util.*
import java.util.UUID
class Events {
@@ -32,140 +32,152 @@ class Events {
}
abstract class Base(
val id: UUID,
val method: String,
val channel: Channel
val id: UUID,
val method: String,
val channel: Channel
) {
val version = "accesslog/v1beta"
var ts = Instant.now()
}
abstract class ChainBase(
val blockchain: Chain, method: String, id: UUID, channel: Channel
val blockchain: Chain,
method: String,
id: UUID,
channel: Channel
) : Base(id, method, channel)
@JsonInclude(JsonInclude.Include.NON_NULL)
class SubscribeHead(
blockchain: Chain, id: UUID,
// initial request details
val request: StreamRequestDetails,
// index of the current response
val index: Int
blockchain: Chain,
id: UUID,
// initial request details
val request: StreamRequestDetails,
// index of the current response
val index: Int
) : ChainBase(blockchain, "SubscribeHead", id, Channel.GRPC)
@JsonInclude(JsonInclude.Include.NON_NULL)
class SubscribeBalance(
blockchain: Chain, id: UUID, subscribe: Boolean,
// initial request details
val request: StreamRequestDetails,
val balanceRequest: BalanceRequest,
val addressBalance: AddressBalance,
// index of the current response
val index: Int
blockchain: Chain,
id: UUID,
subscribe: Boolean,
// initial request details
val request: StreamRequestDetails,
val balanceRequest: BalanceRequest,
val addressBalance: AddressBalance,
// index of the current response
val index: Int
) : ChainBase(blockchain, if (subscribe) "SubscribeBalance" else "GetBalance", id, Channel.GRPC)
@JsonInclude(JsonInclude.Include.NON_NULL)
class TxStatus(
blockchain: Chain, id: UUID,
val request: StreamRequestDetails,
val txStatusRequest: TxStatusRequest,
val txStatus: TxStatusResponse,
// index of the current response
val index: Int
blockchain: Chain,
id: UUID,
val request: StreamRequestDetails,
val txStatusRequest: TxStatusRequest,
val txStatus: TxStatusResponse,
// index of the current response
val index: Int
) : ChainBase(blockchain, "SubscribeTxStatus", id, Channel.GRPC)
data class TxStatusRequest(
val txId: String
val txId: String
)
data class TxStatusResponse(
val confirmations: Int
val confirmations: Int
)
@JsonInclude(JsonInclude.Include.NON_NULL)
class NativeCall(
blockchain: Chain, id: UUID, channel: Channel,
blockchain: Chain,
id: UUID,
channel: Channel,
// info about the initial request, that may include several native calls
val request: StreamRequestDetails,
// total native calls passes within the initial request
val total: Int,
// index of the call specific for the current response
val index: Int,
val selector: String? = null,
val quorum: Long? = null,
val minAvailability: String? = null,
// info about the initial request, that may include several native calls
val request: StreamRequestDetails,
// total native calls passes within the initial request
val total: Int,
// index of the call specific for the current response
val index: Int,
val selector: String? = null,
val quorum: Long? = null,
val minAvailability: String? = null,
val succeed: Boolean,
val rpcError: Int? = null,
val payloadSizeBytes: Long,
val nativeCall: NativeCallItemDetails
val succeed: Boolean,
val rpcError: Int? = null,
val payloadSizeBytes: Long,
val nativeCall: NativeCallItemDetails
) : ChainBase(blockchain, "NativeCall", id, channel)
@JsonInclude(JsonInclude.Include.NON_NULL)
class NativeSubscribe(
blockchain: Chain, id: UUID, channel: Channel,
blockchain: Chain,
id: UUID,
channel: Channel,
// info about the initial request, that may include several native calls
val request: StreamRequestDetails,
val payloadSizeBytes: Long,
val nativeSubscribe: NativeSubscribeItemDetails
// info about the initial request, that may include several native calls
val request: StreamRequestDetails,
val payloadSizeBytes: Long,
val nativeSubscribe: NativeSubscribeItemDetails
) : ChainBase(blockchain, "NativeSubscribe", id, channel)
@JsonInclude(JsonInclude.Include.NON_NULL)
class Describe(
id: UUID,
val request: StreamRequestDetails
id: UUID,
val request: StreamRequestDetails
) : Base(id, "Describe", Channel.GRPC)
@JsonInclude(JsonInclude.Include.NON_NULL)
class Status(
blockchain: Chain, id: UUID,
val request: StreamRequestDetails
blockchain: Chain,
id: UUID,
val request: StreamRequestDetails
) : ChainBase(blockchain, "Status", id, Channel.GRPC)
data class StreamRequestDetails(
val id: UUID,
val start: Instant,
val remote: Remote
val id: UUID,
val start: Instant,
val remote: Remote
)
data class Remote(
val ips: List<String>,
val ip: String,
val userAgent: String
val ips: List<String>,
val ip: String,
val userAgent: String
)
data class NativeCallItemDetails(
val method: String,
val id: Int,
val payloadSizeBytes: Long
val method: String,
val id: Int,
val payloadSizeBytes: Long
)
data class NativeCallReplyDetails(
val id: Int,
val succeed: Boolean,
val replySizeBytes: Long,
val ts: Instant = Instant.now()
val id: Int,
val succeed: Boolean,
val replySizeBytes: Long,
val ts: Instant = Instant.now()
)
data class NativeSubscribeItemDetails(
val method: String,
val payloadSizeBytes: Long
val method: String,
val payloadSizeBytes: Long
)
data class NativeSubscribeReplyDetails(
val replySizeBytes: Long,
val ts: Instant = Instant.now()
val replySizeBytes: Long,
val ts: Instant = Instant.now()
)
data class BalanceRequest(
val asset: String,
val addressType: String
val asset: String,
val addressType: String
)
data class AddressBalance(
val asset: String,
val address: String
val asset: String,
val address: String
)
}
}

View File

@@ -27,7 +27,8 @@ import reactor.netty.http.server.HttpServerRequest
import java.net.InetAddress
import java.net.InetSocketAddress
import java.time.Instant
import java.util.*
import java.util.Locale
import java.util.UUID
class EventsBuilder {
@@ -48,23 +49,23 @@ class EventsBuilder {
fun onReply(msg: Resp): E
}
abstract class Base<T>() : StartingHttp2Request, StartingHttp1Request {
abstract class Base<T> : StartingHttp2Request, StartingHttp1Request {
companion object {
private val remoteIpHeaders = listOf(
"x-real-ip",
"x-forwarded-for"
"x-real-ip",
"x-forwarded-for"
)
private val remoteIpKeys = listOf(
Metadata.Key.of("x-real-ip", Metadata.ASCII_STRING_MARSHALLER),
Metadata.Key.of("x-forwarded-for", Metadata.ASCII_STRING_MARSHALLER)
Metadata.Key.of("x-real-ip", Metadata.ASCII_STRING_MARSHALLER),
Metadata.Key.of("x-forwarded-for", Metadata.ASCII_STRING_MARSHALLER)
)
private val invalidCharacters = Regex("[\n\t]+")
}
var requestDetails = Events.StreamRequestDetails(
UUID.randomUUID(),
Instant.now(),
Events.Remote(emptyList(), "", "")
UUID.randomUUID(),
Instant.now(),
Events.Remote(emptyList(), "", "")
)
var chainId: Int = Chain.UNSPECIFIED.id
@@ -84,35 +85,37 @@ class EventsBuilder {
private fun findBestIp(ips: List<InetAddress>): InetAddress? {
// check if a real remote address is provided, otherwise use any local address
return ips.sortedWith(kotlin.Comparator { a, b ->
val aLocal = a.isLoopbackAddress || a.isSiteLocalAddress
val bLocal = b.isLoopbackAddress || b.isSiteLocalAddress
when {
aLocal && bLocal -> 0
aLocal -> 1
else -> -1
return ips.sortedWith(
kotlin.Comparator { a, b ->
val aLocal = a.isLoopbackAddress || a.isSiteLocalAddress
val bLocal = b.isLoopbackAddress || b.isSiteLocalAddress
when {
aLocal && bLocal -> 0
aLocal -> 1
else -> -1
}
}
}).firstOrNull()
).firstOrNull()
}
private fun clean(s: String): String {
return StringUtils.truncate(s, 128)
.replace(invalidCharacters, " ")
.trim()
.replace(invalidCharacters, " ")
.trim()
}
protected abstract fun getT(): T
override fun start(metadata: Metadata, attributes: Attributes) {
val userAgent = metadata.get(Metadata.Key.of("user-agent", Metadata.ASCII_STRING_MARSHALLER))
?.let(this@Base::clean)
?: ""
?.let(this@Base::clean)
?: ""
val ips = ArrayList<InetAddress>()
remoteIpKeys.forEach { key ->
metadata.get(key)?.let {
it.trim().ifEmpty { null }
?.let(this@Base::toInetAddress)
?.let(ips::add)
?.let(this@Base::toInetAddress)
?.let(ips::add)
}
}
attributes.get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR)?.let { addr ->
@@ -122,24 +125,26 @@ class EventsBuilder {
}
val ip = findBestIp(ips)?.hostAddress ?: ""
this.requestDetails = this.requestDetails
.copy(remote = Events.Remote(
ips = ips.map { it.hostAddress },
ip = ip,
userAgent = userAgent
))
.copy(
remote = Events.Remote(
ips = ips.map { it.hostAddress },
ip = ip,
userAgent = userAgent
)
)
}
override fun start(request: HttpServerRequest) {
val headers = request.requestHeaders()
val userAgent = headers.get("user-agent")
?.let(this@Base::clean)
?: ""
?.let(this@Base::clean)
?: ""
val ips = ArrayList<InetAddress>()
remoteIpHeaders.forEach { key ->
headers.get(key)?.let {
it.trim().ifEmpty { null }
?.let(this@Base::toInetAddress)
?.let(ips::add)
?.let(this@Base::toInetAddress)
?.let(ips::add)
}
}
request.remoteAddress()?.let { addr ->
@@ -147,11 +152,13 @@ class EventsBuilder {
}
val ip = findBestIp(ips)?.hostAddress ?: ""
this.requestDetails = this.requestDetails
.copy(remote = Events.Remote(
ips = ips.map { it.hostAddress },
ip = ip,
userAgent = userAgent
))
.copy(
remote = Events.Remote(
ips = ips.map { it.hostAddress },
ip = ip,
userAgent = userAgent
)
)
}
fun withChain(chain: Int): T {
@@ -161,9 +168,9 @@ class EventsBuilder {
}
}
class SubscribeHead() :
Base<SubscribeHead>(),
RequestReply<Events.SubscribeHead, Common.Chain, BlockchainOuterClass.ChainHead> {
class SubscribeHead :
Base<SubscribeHead>(),
RequestReply<Events.SubscribeHead, Common.Chain, BlockchainOuterClass.ChainHead> {
private var index = 0
@@ -177,14 +184,14 @@ class EventsBuilder {
override fun onReply(msg: BlockchainOuterClass.ChainHead): Events.SubscribeHead {
return Events.SubscribeHead(
chain, UUID.randomUUID(), requestDetails, index++
chain, UUID.randomUUID(), requestDetails, index++
)
}
}
class SubscribeBalance(val subscribe: Boolean) :
Base<SubscribeBalance>(),
RequestReply<Events.SubscribeBalance, BlockchainOuterClass.BalanceRequest, BlockchainOuterClass.AddressBalance> {
Base<SubscribeBalance>(),
RequestReply<Events.SubscribeBalance, BlockchainOuterClass.BalanceRequest, BlockchainOuterClass.AddressBalance> {
private var index = 0
private var balanceRequest: Events.BalanceRequest? = null
@@ -195,8 +202,8 @@ class EventsBuilder {
override fun onRequest(msg: BlockchainOuterClass.BalanceRequest) {
balanceRequest = Events.BalanceRequest(
msg.asset.code.uppercase(Locale.getDefault()),
msg.address.addrTypeCase.name
msg.asset.code.uppercase(Locale.getDefault()),
msg.address.addrTypeCase.name
)
}
@@ -207,14 +214,14 @@ class EventsBuilder {
val addressBalance = Events.AddressBalance(msg.asset.code, msg.address.address)
val chain = Chain.byId(msg.asset.chain.number)
return Events.SubscribeBalance(
chain, UUID.randomUUID(), subscribe, requestDetails, balanceRequest!!, addressBalance, index++
chain, UUID.randomUUID(), subscribe, requestDetails, balanceRequest!!, addressBalance, index++
)
}
}
class TxStatus() :
Base<TxStatus>(),
RequestReply<Events.TxStatus, BlockchainOuterClass.TxStatusRequest, BlockchainOuterClass.TxStatus> {
class TxStatus :
Base<TxStatus>(),
RequestReply<Events.TxStatus, BlockchainOuterClass.TxStatusRequest, BlockchainOuterClass.TxStatus> {
private var index = 0
private var txStatusRequest: Events.TxStatusRequest? = null
@@ -225,21 +232,20 @@ class EventsBuilder {
override fun onReply(msg: BlockchainOuterClass.TxStatus): Events.TxStatus {
return Events.TxStatus(
chain, UUID.randomUUID(), requestDetails, txStatusRequest!!,
Events.TxStatusResponse(msg.confirmations),
index++
chain, UUID.randomUUID(), requestDetails, txStatusRequest!!,
Events.TxStatusResponse(msg.confirmations),
index++
)
}
override fun getT(): TxStatus {
return this
}
}
class NativeCall :
Base<NativeCall>(),
RequestReply<Events.NativeCall, BlockchainOuterClass.NativeCallRequest, BlockchainOuterClass.NativeCallReplyItem> {
Base<NativeCall>(),
RequestReply<Events.NativeCall, BlockchainOuterClass.NativeCallRequest, BlockchainOuterClass.NativeCallReplyItem> {
val items = ArrayList<Events.NativeCallItemDetails>()
val replies = HashMap<Int, Events.NativeCallReplyDetails>()
private var index = 0
@@ -252,11 +258,11 @@ class EventsBuilder {
withChain(msg.chain.number)
msg.itemsList.forEach { item ->
this.items.add(
Events.NativeCallItemDetails(
item.method,
item.id,
item.payload.size().toLong()
)
Events.NativeCallItemDetails(
item.method,
item.id,
item.payload.size().toLong()
)
)
}
}
@@ -264,38 +270,40 @@ class EventsBuilder {
override fun onReply(msg: BlockchainOuterClass.NativeCallReplyItem): Events.NativeCall {
val item = items.find { it.id == msg.id }!!
return Events.NativeCall(
request = requestDetails,
total = items.size,
index = index++,
succeed = msg.succeed,
blockchain = chain,
nativeCall = item,
payloadSizeBytes = item.payloadSizeBytes,
id = UUID.randomUUID(),
channel = Events.Channel.GRPC
request = requestDetails,
total = items.size,
index = index++,
succeed = msg.succeed,
blockchain = chain,
nativeCall = item,
payloadSizeBytes = item.payloadSizeBytes,
id = UUID.randomUUID(),
channel = Events.Channel.GRPC
)
}
fun onReply(reply: io.emeraldpay.dshackle.rpc.NativeCall.CallResult,
channel: Events.Channel): Events.NativeCall {
fun onReply(
reply: io.emeraldpay.dshackle.rpc.NativeCall.CallResult,
channel: Events.Channel
): Events.NativeCall {
val item = items.find { it.id == reply.id }!!
return Events.NativeCall(
request = requestDetails,
total = items.size,
index = index++,
succeed = !reply.isError(),
blockchain = chain,
nativeCall = item,
payloadSizeBytes = item.payloadSizeBytes,
id = UUID.randomUUID(),
channel = channel
request = requestDetails,
total = items.size,
index = index++,
succeed = !reply.isError(),
blockchain = chain,
nativeCall = item,
payloadSizeBytes = item.payloadSizeBytes,
id = UUID.randomUUID(),
channel = channel
)
}
}
class NativeSubscribe :
Base<NativeSubscribe>(),
RequestReply<Events.NativeSubscribe, BlockchainOuterClass.NativeSubscribeRequest, BlockchainOuterClass.NativeSubscribeReplyItem> {
Base<NativeSubscribe>(),
RequestReply<Events.NativeSubscribe, BlockchainOuterClass.NativeSubscribeRequest, BlockchainOuterClass.NativeSubscribeReplyItem> {
var item: Events.NativeSubscribeItemDetails? = null
val replies = HashMap<Int, Events.NativeSubscribeReplyDetails>()
@@ -306,26 +314,26 @@ class EventsBuilder {
override fun onRequest(msg: BlockchainOuterClass.NativeSubscribeRequest) {
withChain(msg.chain.number)
this.item = Events.NativeSubscribeItemDetails(
msg.method,
msg.payload.size().toLong()
msg.method,
msg.payload.size().toLong()
)
}
override fun onReply(msg: BlockchainOuterClass.NativeSubscribeReplyItem): Events.NativeSubscribe {
return Events.NativeSubscribe(
request = requestDetails,
blockchain = chain,
nativeSubscribe = item!!,
payloadSizeBytes = msg.payload?.size()?.toLong() ?: 0L,
id = UUID.randomUUID(),
channel = Events.Channel.GRPC
request = requestDetails,
blockchain = chain,
nativeSubscribe = item!!,
payloadSizeBytes = msg.payload?.size()?.toLong() ?: 0L,
id = UUID.randomUUID(),
channel = Events.Channel.GRPC
)
}
}
class Describe :
Base<Describe>(),
RequestReply<Events.Describe, BlockchainOuterClass.DescribeRequest, BlockchainOuterClass.DescribeResponse> {
Base<Describe>(),
RequestReply<Events.Describe, BlockchainOuterClass.DescribeRequest, BlockchainOuterClass.DescribeResponse> {
override fun getT(): Describe {
return this
@@ -336,15 +344,15 @@ class EventsBuilder {
override fun onReply(msg: BlockchainOuterClass.DescribeResponse): Events.Describe {
return Events.Describe(
id = UUID.randomUUID(),
request = requestDetails
id = UUID.randomUUID(),
request = requestDetails
)
}
}
class Status :
Base<Status>(),
RequestReply<Events.Status, BlockchainOuterClass.StatusRequest, BlockchainOuterClass.ChainStatus> {
Base<Status>(),
RequestReply<Events.Status, BlockchainOuterClass.StatusRequest, BlockchainOuterClass.ChainStatus> {
override fun getT(): Status {
return this
}
@@ -355,11 +363,10 @@ class EventsBuilder {
override fun onReply(msg: BlockchainOuterClass.ChainStatus): Events.Status {
val chain = Chain.byId(msg.chainValue)
return Events.Status(
blockchain = chain,
request = requestDetails,
id = UUID.randomUUID()
blockchain = chain,
request = requestDetails,
id = UUID.randomUUID()
)
}
}
}
}

View File

@@ -23,10 +23,10 @@ import org.slf4j.LoggerFactory
* JSON RPC call to the proxy
*/
class ProxyCall(
/**
* Type of the request. The response format depends on it
*/
val type: RpcType
/**
* Type of the request. The response format depends on it
*/
val type: RpcType
) {
companion object {
@@ -55,4 +55,4 @@ class ProxyCall(
*/
BATCH
}
}
}

View File

@@ -24,8 +24,8 @@ import io.emeraldpay.dshackle.config.ProxyConfig
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp
import io.emeraldpay.dshackle.rpc.NativeCall
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.grpc.Chain
import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Timer
@@ -38,16 +38,14 @@ import org.slf4j.LoggerFactory
import org.springframework.http.HttpHeaders
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.netty.DisposableServer
import reactor.netty.http.server.HttpServer
import reactor.netty.http.server.HttpServerRequest
import reactor.netty.http.server.HttpServerResponse
import reactor.netty.http.server.HttpServerRoutes
import java.util.*
import java.util.EnumMap
import java.util.concurrent.TimeUnit
import java.util.concurrent.locks.ReentrantReadWriteLock
import java.util.function.BiFunction
import kotlin.collections.HashMap
import kotlin.concurrent.read
import kotlin.concurrent.write
@@ -55,12 +53,12 @@ import kotlin.concurrent.write
* HTTP Proxy Server
*/
class ProxyServer(
private var config: ProxyConfig,
private val readRpcJson: ReadRpcJson,
private val writeRpcJson: WriteRpcJson,
private val nativeCall: NativeCall,
private val tlsSetup: TlsSetup,
private val accessHandler: AccessHandlerHttp.HandlerFactory
private var config: ProxyConfig,
private val readRpcJson: ReadRpcJson,
private val writeRpcJson: WriteRpcJson,
private val nativeCall: NativeCall,
private val tlsSetup: TlsSetup,
private val accessHandler: AccessHandlerHttp.HandlerFactory
) {
companion object {
@@ -101,19 +99,19 @@ class ProxyServer(
}
log.info("Listening Proxy on ${config.host}:${config.port}")
var serverBuilder = HttpServer.create()
.doOnChannelInit { _, channel, _ ->
channel.pipeline().addFirst(errorHandler)
}
.host(config.host)
.port(config.port)
.doOnChannelInit { _, channel, _ ->
channel.pipeline().addFirst(errorHandler)
}
.host(config.host)
.port(config.port)
tlsSetup.setupServer("proxy", config.tls, false)?.let { sslContext ->
serverBuilder = serverBuilder.secure { secure -> secure.sslContext(sslContext) }
}
serverBuilder
.route(this::setupRoutes)
.bindNow()
.route(this::setupRoutes)
.bindNow()
}
fun setupRoutes(routes: HttpServerRoutes) {
@@ -139,26 +137,26 @@ class ProxyServer(
}
}
val request = BlockchainOuterClass.NativeCallRequest.newBuilder()
.setChain(Common.ChainRef.forNumber(chain.id))
.addAllItems(call.items)
.build()
.setChain(Common.ChainRef.forNumber(chain.id))
.addAllItems(call.items)
.build()
handler.onRequest(request)
val jsons = nativeCall
.nativeCallResult(Mono.just(request))
.doOnNext {
metricById(it.id)?.requestMetric?.increment()
.nativeCallResult(Mono.just(request))
.doOnNext {
metricById(it.id)?.requestMetric?.increment()
}
.doOnNext {
handler.onResponse(it)
metricById(it.id)?.callMetric?.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS)
}
.doOnError {
// when error happened the whole flux is stopped and no result is produced, so we should mark all the requests as failed
call.items.forEach { item ->
requestMetrics.get(chain, item.method).errorMetric.increment()
}
.doOnNext {
handler.onResponse(it)
metricById(it.id)?.callMetric?.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS)
}
.doOnError {
//when error happened the whole flux is stopped and no result is produced, so we should mark all the requests as failed
call.items.forEach { item ->
requestMetrics.get(chain, item.method).errorMetric.increment()
}
}
.transform(writeRpcJson.toJsons(call))
}
.transform(writeRpcJson.toJsons(call))
return if (call.type == ProxyCall.RpcType.SINGLE) {
jsons.next()
} else {
@@ -166,21 +164,25 @@ class ProxyServer(
}
}
fun processRequest(chain: Chain, request: Mono<ByteArray>, handler: AccessHandlerHttp.RequestHandler): Flux<ByteBuf> {
fun processRequest(
chain: Chain,
request: Mono<ByteArray>,
handler: AccessHandlerHttp.RequestHandler
): Flux<ByteBuf> {
return request
.map(readRpcJson)
.flatMapMany { call ->
execute(chain, call, handler)
}
.onErrorResume(RpcException::class.java) { err ->
val id = err.details?.let {
if (it is JsonRpcResponse.Id) it else JsonRpcResponse.NumberId(-1)
} ?: JsonRpcResponse.NumberId(-1)
.map(readRpcJson)
.flatMapMany { call ->
execute(chain, call, handler)
}
.onErrorResume(RpcException::class.java) { err ->
val id = err.details?.let {
if (it is JsonRpcResponse.Id) it else JsonRpcResponse.NumberId(-1)
} ?: JsonRpcResponse.NumberId(-1)
val json = JsonRpcResponse.error(err.code, err.rpcMessage, id)
Mono.just(Global.objectMapper.writeValueAsString(json))
}
.map { Unpooled.wrappedBuffer(it.toByteArray()) }
val json = JsonRpcResponse.error(err.code, err.rpcMessage, id)
Mono.just(Global.objectMapper.writeValueAsString(json))
}
.map { Unpooled.wrappedBuffer(it.toByteArray()) }
}
fun proxy(routeConfig: ProxyConfig.Route): BiFunction<HttpServerRequest, HttpServerResponse, Publisher<Void>> {
@@ -188,13 +190,13 @@ class ProxyServer(
// handle access events
val eventHandler = accessHandler.create(req, routeConfig.blockchain)
val request = req.receive()
.aggregate()
.asByteArray()
.aggregate()
.asByteArray()
val results = processRequest(routeConfig.blockchain, request, eventHandler)
// make sure that the access log handler is closed at the end, so it can render the logs
.doFinally { eventHandler.close() }
// make sure that the access log handler is closed at the end, so it can render the logs
.doFinally { eventHandler.close() }
resp.addHeader(HttpHeaders.CONTENT_TYPE, "application/json")
.send(results)
.send(results)
}
}
@@ -251,25 +253,25 @@ class ProxyServer(
class RequestMetricsBasic(chain: Chain) : RequestMetrics {
override val callMetric = Timer.builder("request.jsonrpc.call")
.tags("chain", chain.chainCode)
.register(Metrics.globalRegistry)
.tags("chain", chain.chainCode)
.register(Metrics.globalRegistry)
override val errorMetric = Counter.builder("request.jsonrpc.err")
.tags("chain", chain.chainCode)
.register(Metrics.globalRegistry)
.tags("chain", chain.chainCode)
.register(Metrics.globalRegistry)
override val requestMetric = Counter.builder("request.jsonrpc.request.total")
.tags("chain", chain.chainCode)
.register(Metrics.globalRegistry)
.tags("chain", chain.chainCode)
.register(Metrics.globalRegistry)
}
class RequestMetricsWithMethod(chain: Chain, method: String) : RequestMetrics {
override val callMetric = Timer.builder("request.jsonrpc.call")
.tags("chain", chain.chainCode, "method", method)
.register(Metrics.globalRegistry)
.tags("chain", chain.chainCode, "method", method)
.register(Metrics.globalRegistry)
override val errorMetric = Counter.builder("request.jsonrpc.err")
.tags("chain", chain.chainCode, "method", method)
.register(Metrics.globalRegistry)
.tags("chain", chain.chainCode, "method", method)
.register(Metrics.globalRegistry)
override val requestMetric = Counter.builder("request.jsonrpc.request.total")
.tags("chain", chain.chainCode, "method", method)
.register(Metrics.globalRegistry)
.tags("chain", chain.chainCode, "method", method)
.register(Metrics.globalRegistry)
}
}

View File

@@ -25,19 +25,16 @@ import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.etherjar.rpc.RpcResponseError
import io.emeraldpay.etherjar.rpc.json.RequestJson
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
import java.io.IOException
import java.util.*
import java.util.function.Function
import java.util.stream.Collectors
/**
* Reader for JSON RPC request
*/
@Service
open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
open class ReadRpcJson : Function<ByteArray, ProxyCall> {
companion object {
private val log = LoggerFactory.getLogger(ReadRpcJson::class.java)
@@ -55,21 +52,37 @@ open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
val id = json["id"]
if ("2.0" != json["jsonrpc"]) {
if (json["jsonrpc"] == null) {
throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "jsonrpc version is not set", id?.let { JsonRpcResponse.Id.from(it) })
throw RpcException(
RpcResponseError.CODE_INVALID_REQUEST,
"jsonrpc version is not set",
id?.let { JsonRpcResponse.Id.from(it) }
)
}
throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "Unsupported JSON RPC version: " + json["jsonrpc"].toString(), id?.let { JsonRpcResponse.Id.from(it) })
throw RpcException(
RpcResponseError.CODE_INVALID_REQUEST,
"Unsupported JSON RPC version: " + json["jsonrpc"].toString(),
id?.let { JsonRpcResponse.Id.from(it) }
)
}
if (!(json["method"] != null && json["method"] is String)) {
throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "Method is not set", id?.let { JsonRpcResponse.Id.from(it) })
throw RpcException(
RpcResponseError.CODE_INVALID_REQUEST,
"Method is not set",
id?.let { JsonRpcResponse.Id.from(it) }
)
}
if (json.containsKey("params") && json["params"] !is List<*>) {
throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "Params must be an array", id?.let { JsonRpcResponse.Id.from(it) })
throw RpcException(
RpcResponseError.CODE_INVALID_REQUEST,
"Params must be an array",
id?.let { JsonRpcResponse.Id.from(it) }
)
}
RequestJson<Any>(
json["method"].toString(),
//params MAY be omitted
(json["params"] ?: emptyList<Any>()) as List<*>,
id
json["method"].toString(),
// params MAY be omitted
(json["params"] ?: emptyList<Any>()) as List<*>,
id
)
}
}
@@ -81,7 +94,7 @@ open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
fun getStartOfJson(buf: ByteArray): Byte {
val count = buf.size
var i = 0
//if cannot find anything in the first 255 bytes, just consider it as invalid
// if cannot find anything in the first 255 bytes, just consider it as invalid
while (i < 256 && i < count) {
if (buf[i] != spaces[0] && buf[i] != spaces[1] && buf[i] != spaces[2]) {
return buf[i]
@@ -127,17 +140,17 @@ open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
// our internal ids for calls
var seq = 0
val batch = list.stream()
.map<RequestJson<Any>>(jsonExtractor)
.map { json ->
val id = seq++
context.ids[id] = json.id
BlockchainOuterClass.NativeCallItem.newBuilder()
.setId(id)
.setMethod(json.method)
.setPayload(ByteString.copyFrom(objectMapper.writeValueAsBytes(json.params)))
.build()
}
.collect(Collectors.toList())
.map<RequestJson<Any>>(jsonExtractor)
.map { json ->
val id = seq++
context.ids[id] = json.id
BlockchainOuterClass.NativeCallItem.newBuilder()
.setId(id)
.setMethod(json.method)
.setPayload(ByteString.copyFrom(objectMapper.writeValueAsBytes(json.params)))
.build()
}
.collect(Collectors.toList())
context.items.addAll(batch)
return context
} catch (e: RpcException) {
@@ -147,5 +160,4 @@ open class ReadRpcJson() : Function<ByteArray, ProxyCall> {
throw RpcException(RpcResponseError.CODE_INVALID_JSON, e.message)
}
}
}
}

View File

@@ -17,7 +17,6 @@
package io.emeraldpay.dshackle.proxy
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.rpc.NativeCall
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
@@ -31,7 +30,7 @@ import java.util.function.Function
* Writer for JSON RPC requests
*/
@Service
open class WriteRpcJson() {
open class WriteRpcJson {
companion object {
private val log = LoggerFactory.getLogger(WriteRpcJson::class.java)
@@ -45,35 +44,35 @@ open class WriteRpcJson() {
open fun toJsons(call: ProxyCall): Function<Flux<NativeCall.CallResult>, Flux<String>> {
return Function { flux ->
flux
.flatMap { response ->
if (!call.ids.containsKey(response.id)) {
log.warn("ID wasn't requested: ${response.id}")
return@flatMap Flux.empty<String>()
}
val json = toJson(call, response)
if (json == null) {
Flux.empty<String>()
} else {
Flux.just(json)
}
.flatMap { response ->
if (!call.ids.containsKey(response.id)) {
log.warn("ID wasn't requested: ${response.id}")
return@flatMap Flux.empty<String>()
}
.onErrorResume { t ->
if (t is NativeCall.CallFailure) {
Mono.just(toJson(call, t)!!)
} else {
Mono.empty()
}
val json = toJson(call, response)
if (json == null) {
Flux.empty<String>()
} else {
Flux.just(json)
}
.onErrorContinue { t, _ ->
log.warn("Failed to convert to JSON", t)
}
.onErrorResume { t ->
if (t is NativeCall.CallFailure) {
Mono.just(toJson(call, t)!!)
} else {
Mono.empty()
}
}
.onErrorContinue { t, _ ->
log.warn("Failed to convert to JSON", t)
}
}
}
open fun toJson(call: ProxyCall, response: NativeCall.CallResult): String? {
val id = call.ids[response.id]?.let {
JsonRpcResponse.Id.from(it)
} ?: return null;
} ?: return null
val json = if (response.isError()) {
val error = response.error!!
error.upstreamError?.let { upstreamError ->
@@ -86,7 +85,7 @@ open class WriteRpcJson() {
}
fun toJson(call: ProxyCall, error: NativeCall.CallFailure): String? {
val id = call.ids[error.id] ?: return null;
val id = call.ids[error.id] ?: return null
val json = JsonRpcResponse.error(-32003, error.reason.message ?: "", JsonRpcResponse.Id.from(id))
return objectMapper.writeValueAsString(json)
}
@@ -97,18 +96,18 @@ open class WriteRpcJson() {
fun asArray(): Function<Flux<String>, Flux<String>> {
return Function { flux ->
val body = flux.zipWith(Flux.concat(Mono.just(false), Flux.just(true).repeat()))
.map {
if (it.t2) {
"," + it.t1
} else {
it.t1
}
.map {
if (it.t2) {
"," + it.t1
} else {
it.t1
}
}
Flux.concat(
Mono.just("["),
body,
Mono.just("]")
Mono.just("["),
body,
Mono.just("]")
)
}
}
}
}

View File

@@ -20,9 +20,8 @@ import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.etherjar.rpc.RpcException
open class AlwaysQuorum: CallQuorum {
open class AlwaysQuorum : CallQuorum {
private var resolved = false
private var result: ByteArray? = null
@@ -60,4 +59,4 @@ open class AlwaysQuorum: CallQuorum {
override fun toString(): String {
return "Quorum: Accept Any"
}
}
}

View File

@@ -20,7 +20,7 @@ import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream
open class BroadcastQuorum(
val quorum: Int = 3
val quorum: Int = 3
) : CallQuorum, ValueAwareQuorum<String>(String::class.java) {
private var result: ByteArray? = null
@@ -61,4 +61,4 @@ open class BroadcastQuorum(
override fun toString(): String {
return "Quorum: Broadcast to $quorum upstreams"
}
}
}

View File

@@ -20,10 +20,6 @@ import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.etherjar.domain.TransactionId
import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.etherjar.rpc.json.BlockJson
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
import reactor.util.function.Tuple2
import java.util.function.BiFunction
import java.util.function.Predicate
@@ -54,4 +50,4 @@ interface CallQuorum {
}
}
}
}
}

View File

@@ -16,15 +16,11 @@
*/
package io.emeraldpay.dshackle.quorum
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.etherjar.rpc.JacksonRpcConverter
import io.emeraldpay.etherjar.rpc.RpcException
open class NonEmptyQuorum(
val maxTries: Int = 3
val maxTries: Int = 3
) : CallQuorum, ValueAwareQuorum<Any>(Any::class.java) {
private var result: ByteArray? = null
@@ -59,4 +55,4 @@ open class NonEmptyQuorum(
override fun toString(): String {
return "Quorum: Accept Non Error Result"
}
}
}

View File

@@ -16,17 +16,14 @@
*/
package io.emeraldpay.dshackle.quorum
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.etherjar.hex.HexQuantity
import io.emeraldpay.etherjar.rpc.JacksonRpcConverter
import io.emeraldpay.etherjar.rpc.RpcException
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
open class NonceQuorum(
val tries: Int = 3
val tries: Int = 3
) : CallQuorum, ValueAwareQuorum<String>(String::class.java) {
private val lock = ReentrantLock()
@@ -74,4 +71,4 @@ open class NonceQuorum(
override fun toString(): String {
return "Quorum: Confirm with $tries upstreams"
}
}
}

View File

@@ -20,7 +20,6 @@ import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.etherjar.rpc.RpcException
import java.util.concurrent.atomic.AtomicReference
/**
@@ -28,7 +27,7 @@ import java.util.concurrent.atomic.AtomicReference
*
* NOTE: NativeCall checks the quorums and applies a HeightSelector if NotLaggingQuorum is enabled for a call
*/
class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum {
class NotLaggingQuorum(val maxLag: Long = 0) : CallQuorum {
private val result: AtomicReference<ByteArray> = AtomicReference()
private val failed = AtomicReference(false)
@@ -73,4 +72,4 @@ class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum {
override fun toString(): String {
return "Quorum: late <= $maxLag blocks"
}
}
}

View File

@@ -35,4 +35,4 @@ interface QuorumReaderFactory {
return QuorumRpcReader(apis, quorum)
}
}
}
}

View File

@@ -30,8 +30,8 @@ import reactor.util.function.Tuples
* Makes request with applying Quorum
*/
class QuorumRpcReader(
private val apis: ApiSource,
private val quorum: CallQuorum
private val apis: ApiSource,
private val quorum: CallQuorum
) : Reader<JsonRpcRequest, QuorumRpcReader.Result> {
companion object {
@@ -58,8 +58,8 @@ class QuorumRpcReader(
val defaultResult: Mono<Result> = Mono.just(quorum).flatMap { q ->
if (q.isFailed()) {
Mono.error<Result>(
q.getError()?.asException(JsonRpcResponse.NumberId(1))
?: RpcException(-32000, "Unknown Upstream error")
q.getError()?.asException(JsonRpcResponse.NumberId(1))
?: RpcException(-32000, "Unknown Upstream error")
)
} else {
log.warn("Did not get any result from upstream. Method [${key.method}] using [$q]")
@@ -68,72 +68,71 @@ class QuorumRpcReader(
}
return Flux.from(apis)
.takeUntil {
quorum.isFailed() || quorum.isResolved()
}
.flatMap { api ->
api.getApi()
.read(key)
.flatMap { response ->
response.requireResult()
.onErrorResume { err ->
if (err is RpcException || err is JsonRpcException) {
// on error notify quorum, it may use error message or other details
val cleanErr: JsonRpcException = when (err) {
is RpcException -> JsonRpcException.from(err)
is JsonRpcException -> err
else -> throw IllegalStateException("Cannot convert from exception", err)
}
quorum.record(cleanErr, api)
// it it's failed after that, then we don't need more calls, stop api source
if (quorum.isFailed()) {
apis.resolve()
} else {
apis.request(1)
}
} else {
log.warn("Result processing error", err)
}
Mono.empty()
}
.takeUntil {
quorum.isFailed() || quorum.isResolved()
}
.flatMap { api ->
api.getApi()
.read(key)
.flatMap { response ->
response.requireResult()
.onErrorResume { err ->
if (err is RpcException || err is JsonRpcException) {
// on error notify quorum, it may use error message or other details
val cleanErr: JsonRpcException = when (err) {
is RpcException -> JsonRpcException.from(err)
is JsonRpcException -> err
else -> throw IllegalStateException("Cannot convert from exception", err)
}
quorum.record(cleanErr, api)
// it it's failed after that, then we don't need more calls, stop api source
if (quorum.isFailed()) {
apis.resolve()
} else {
apis.request(1)
}
} else {
log.warn("Result processing error", err)
}
Mono.empty()
}
.map { Tuples.of(it, api) }
}
.retryWhen(retrySpec)
// record all correct responses until quorum reached
.reduce(quorum, { res, a ->
if (res.record(a.t1, a.t2)) {
apis.resolve()
} else {
apis.request(1)
}
res
})
// if last call resulted in error it's still possible that request was resolved correctly. i.e. for BroadcastQuorum
.onErrorResume { err ->
if (quorum.isResolved()) {
Mono.just(quorum)
} else {
Mono.error(err)
}
.map { Tuples.of(it, api) }
}
.retryWhen(retrySpec)
// record all correct responses until quorum reached
.reduce(quorum, { res, a ->
if (res.record(a.t1, a.t2)) {
apis.resolve()
} else {
apis.request(1)
}
.doOnNext {
if (!it.isResolved() && !it.isFailed()) {
log.debug("No quorum for ${key.method} using [${quorum}]. Error: ${it.getError()?.message ?: ""}")
}
res
})
// if last call resulted in error it's still possible that request was resolved correctly. i.e. for BroadcastQuorum
.onErrorResume { err ->
if (quorum.isResolved()) {
Mono.just(quorum)
} else {
Mono.error(err)
}
// return nothing if not resolved
.filter { it.isResolved() }
.map {
// TODO find actual quorum number
QuorumRpcReader.Result(it.getResult()!!, 1)
}
.doOnNext {
if (!it.isResolved() && !it.isFailed()) {
log.debug("No quorum for ${key.method} using [$quorum]. Error: ${it.getError()?.message ?: ""}")
}
.switchIfEmpty(defaultResult)
}
// return nothing if not resolved
.filter { it.isResolved() }
.map {
// TODO find actual quorum number
QuorumRpcReader.Result(it.getResult()!!, 1)
}
.switchIfEmpty(defaultResult)
}
class Result(
val value: ByteArray,
val quorum: Int
val value: ByteArray,
val quorum: Int
)
}
}

View File

@@ -16,18 +16,16 @@
*/
package io.emeraldpay.dshackle.quorum
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.etherjar.rpc.JacksonRpcConverter
import io.emeraldpay.etherjar.rpc.RpcException
import org.slf4j.LoggerFactory
abstract class ValueAwareQuorum<T>(
val clazz: Class<T>
): CallQuorum {
val clazz: Class<T>
) : CallQuorum {
private val log = LoggerFactory.getLogger(ValueAwareQuorum::class.java)
private var rpcError: JsonRpcError? = null
@@ -45,7 +43,7 @@ abstract class ValueAwareQuorum<T>(
} catch (e: Exception) {
recordError(response, e.message, upstream)
}
return isResolved();
return isResolved()
}
override fun record(error: JsonRpcException, upstream: Upstream) {
@@ -60,4 +58,4 @@ abstract class ValueAwareQuorum<T>(
override fun getError(): JsonRpcError? {
return rpcError
}
}
}

View File

@@ -20,15 +20,14 @@ import io.emeraldpay.dshackle.Defaults
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import java.time.Duration
/**
* Composition of multiple readers.
* Reader returns first value returned by any of the source readers by checking one by one until one of them returns a non-empty result.
*/
class CompoundReader<K, D>(
private vararg val readers: Reader<K, D>
): Reader<K, D> {
private vararg val readers: Reader<K, D>
) : Reader<K, D> {
companion object {
private val log = LoggerFactory.getLogger(CompoundReader::class.java)
@@ -39,13 +38,12 @@ class CompoundReader<K, D>(
return Mono.empty()
}
return Flux.fromIterable(readers.asIterable())
.flatMap({ rdr ->
rdr.read(key)
.timeout(Defaults.timeoutInternal, Mono.empty())
.doOnError { t -> log.warn("Failed to read from $rdr", t) }
.onErrorResume { Mono.empty() }
}, 1)
.next()
.flatMap({ rdr ->
rdr.read(key)
.timeout(Defaults.timeoutInternal, Mono.empty())
.doOnError { t -> log.warn("Failed to read from $rdr", t) }
.onErrorResume { Mono.empty() }
}, 1)
.next()
}
}
}

View File

@@ -18,9 +18,9 @@ package io.emeraldpay.dshackle.reader
import reactor.core.publisher.Mono
class EmptyReader<K, D>: Reader<K, D> {
class EmptyReader<K, D> : Reader<K, D> {
override fun read(key: K): Mono<D> {
return Mono.empty()
}
}
}

View File

@@ -21,5 +21,4 @@ import reactor.core.publisher.Mono
interface Reader<in K, D> {
fun read(key: K): Mono<D>
}
}

View File

@@ -22,22 +22,21 @@ import java.util.function.Function
* Reader wrapper that maps the input key from ne value to another (ex. convert from Long to String)
*/
class RekeyingReader<K, K1, D>(
/**
* Mapping between original Key and Key supported by the reader
*/
private val rekey: Function<K, K1>,
/**
* Actual reader
*/
private val reader: Reader<K1, D>
/**
* Mapping between original Key and Key supported by the reader
*/
private val rekey: Function<K, K1>,
/**
* Actual reader
*/
private val reader: Reader<K1, D>
) : Reader<K, D> {
override fun read(key: K): Mono<D> {
return Mono.just(key)
.map(rekey)
.flatMap {
reader.read(it)
}
.map(rekey)
.flatMap {
reader.read(it)
}
}
}
}

View File

@@ -25,8 +25,8 @@ import reactor.core.publisher.Mono
* Reader that requests data through upstream RPC using provided JSON RPC request builder
*/
class RpcReader<T>(
private val up: Multistream,
private val paramsBuilder: (T) -> JsonRpcRequest
private val up: Multistream,
private val paramsBuilder: (T) -> JsonRpcRequest
) : Reader<T, ByteArray> {
companion object {
@@ -45,11 +45,10 @@ class RpcReader<T>(
override fun read(key: T): Mono<ByteArray> {
return up.getDirectApi(Selector.empty)
.flatMap { rdr ->
rdr.read(paramsBuilder(key)).flatMap {
it.requireResult()
}
.flatMap { rdr ->
rdr.read(paramsBuilder(key)).flatMap {
it.requireResult()
}
}
}
}
}

View File

@@ -22,18 +22,17 @@ import java.util.function.Function
* Reader wrapper that transforms output of the reader to a different format
*/
class TransformingReader<K, D0, D>(
/**
* Actual reader
*/
private val reader: Reader<K, D0>,
/**
* Result transformation
*/
private val transformer: Function<in D0, out D>
/**
* Actual reader
*/
private val reader: Reader<K, D0>,
/**
* Result transformation
*/
private val transformer: Function<in D0, out D>
) : Reader<K, D> {
override fun read(key: K): Mono<D> {
return reader.read(key).map(transformer)
}
}
}

View File

@@ -31,44 +31,45 @@ import org.springframework.context.annotation.DependsOn
import org.springframework.stereotype.Service
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import java.util.*
import java.util.Locale
import java.util.concurrent.TimeUnit
@Service @DependsOn("monitoringSetup")
@Service
@DependsOn("monitoringSetup")
class BlockchainRpc(
@Autowired private val nativeCall: NativeCall,
@Autowired private val nativeSubscribe: NativeSubscribe,
@Autowired private val streamHead: StreamHead,
@Autowired private val trackTx: List<TrackTx>,
@Autowired private val trackAddress: List<TrackAddress>,
@Autowired private val describe: Describe,
@Autowired private val subscribeStatus: SubscribeStatus
): ReactorBlockchainGrpc.BlockchainImplBase() {
@Autowired private val nativeCall: NativeCall,
@Autowired private val nativeSubscribe: NativeSubscribe,
@Autowired private val streamHead: StreamHead,
@Autowired private val trackTx: List<TrackTx>,
@Autowired private val trackAddress: List<TrackAddress>,
@Autowired private val describe: Describe,
@Autowired private val subscribeStatus: SubscribeStatus
) : ReactorBlockchainGrpc.BlockchainImplBase() {
private val log = LoggerFactory.getLogger(BlockchainRpc::class.java)
private val describeMetric = Counter.builder("request.grpc.request")
.tag("type", "describe")
.tag("chain", "NA")
.register(Metrics.globalRegistry)
.tag("type", "describe")
.tag("chain", "NA")
.register(Metrics.globalRegistry)
private val subscribeStatusMetric = Counter.builder("request.grpc.request")
.tag("type", "subscribeStatus")
.tag("chain", "NA")
.register(Metrics.globalRegistry)
.tag("type", "subscribeStatus")
.tag("chain", "NA")
.register(Metrics.globalRegistry)
private val errorMetric = Counter.builder("request.grpc.err")
.register(Metrics.globalRegistry)
.register(Metrics.globalRegistry)
private val chainMetrics = ChainValue { chain -> RequestMetrics(chain) }
override fun nativeCall(request: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<BlockchainOuterClass.NativeCallReplyItem> {
var startTime = 0L
var metrics: RequestMetrics? = null
return nativeCall.nativeCall(
request
.doOnNext {
metrics = chainMetrics.get(it.chain)
metrics!!.nativeCallMetric.increment()
startTime = System.currentTimeMillis()
}
request
.doOnNext {
metrics = chainMetrics.get(it.chain)
metrics!!.nativeCallMetric.increment()
startTime = System.currentTimeMillis()
}
).doOnNext {
metrics?.nativeCallRespMetric?.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS)
}.doOnError { errorMetric.increment() }
@@ -77,11 +78,11 @@ class BlockchainRpc(
override fun nativeSubscribe(request: Mono<BlockchainOuterClass.NativeSubscribeRequest>): Flux<BlockchainOuterClass.NativeSubscribeReplyItem> {
var metrics: RequestMetrics? = null
return nativeSubscribe.nativeSubscribe(
request
.doOnNext {
metrics = chainMetrics.get(it.chain)
metrics!!.nativeSubscribeMetric.increment()
}
request
.doOnNext {
metrics = chainMetrics.get(it.chain)
metrics!!.nativeSubscribeMetric.increment()
}
).doOnNext {
metrics?.nativeSubscribeRespMetric?.increment()
}.doOnError { errorMetric.increment() }
@@ -89,8 +90,8 @@ class BlockchainRpc(
override fun subscribeHead(request: Mono<Common.Chain>): Flux<BlockchainOuterClass.ChainHead> {
return streamHead.add(
request
.doOnNext { chainMetrics.get(it.type).subscribeHeadMetric.increment() }
request
.doOnNext { chainMetrics.get(it.type).subscribeHeadMetric.increment() }
).doOnError { errorMetric.increment() }
}
@@ -102,8 +103,8 @@ class BlockchainRpc(
try {
trackTx.find { it.isSupported(chain) }?.let { track ->
track.subscribe(request)
.doOnNext { metrics.subscribeHeadRespMetric.increment() }
.doOnError { errorMetric.increment() }
.doOnNext { metrics.subscribeHeadRespMetric.increment() }
.doOnError { errorMetric.increment() }
} ?: Flux.error(SilentException.UnsupportedBlockchain(chain))
} catch (t: Throwable) {
log.error("Internal error during Tx Subscription", t)
@@ -122,12 +123,12 @@ class BlockchainRpc(
try {
trackAddress.find { it.isSupported(chain, asset) }?.let { track ->
track.subscribe(request)
.doOnNext { metrics.subscribeBalanceRespMetric.increment() }
.doOnError { errorMetric.increment() }
.doOnNext { metrics.subscribeBalanceRespMetric.increment() }
.doOnError { errorMetric.increment() }
} ?: Flux.error<BlockchainOuterClass.AddressBalance>(SilentException.UnsupportedBlockchain(chain))
.doOnSubscribe {
log.error("Balance for $chain:$asset is not supported")
}
.doOnSubscribe {
log.error("Balance for $chain:$asset is not supported")
}
} catch (t: Throwable) {
log.error("Internal error during Balance Subscription", t)
errorMetric.increment()
@@ -146,13 +147,16 @@ class BlockchainRpc(
try {
trackAddress.find { it.isSupported(chain, asset) }?.let { track ->
track.getBalance(request)
.doOnNext {
metrics.getBalanceRespMetric.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS)
}
.doOnNext {
metrics.getBalanceRespMetric.record(
System.currentTimeMillis() - startTime,
TimeUnit.MILLISECONDS
)
}
} ?: Flux.error<BlockchainOuterClass.AddressBalance>(SilentException.UnsupportedBlockchain(chain))
.doOnSubscribe {
log.error("Balance for $chain:$asset is not supported")
}
.doOnSubscribe {
log.error("Balance for $chain:$asset is not supported")
}
} catch (t: Throwable) {
log.error("Internal error during Balance Request", t)
errorMetric.increment()
@@ -164,61 +168,61 @@ class BlockchainRpc(
override fun describe(request: Mono<BlockchainOuterClass.DescribeRequest>): Mono<BlockchainOuterClass.DescribeResponse> {
describeMetric.increment()
return describe.describe(request)
.doOnError { errorMetric.increment() }
.doOnError { errorMetric.increment() }
}
override fun subscribeStatus(request: Mono<BlockchainOuterClass.StatusRequest>): Flux<BlockchainOuterClass.ChainStatus> {
subscribeStatusMetric.increment()
return subscribeStatus.subscribeStatus(request)
.doOnError { errorMetric.increment() }
.doOnError { errorMetric.increment() }
}
class RequestMetrics(chain: Chain) {
val nativeCallMetric = Counter.builder("request.grpc.request")
.tag("type", "nativeCall")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
.tag("type", "nativeCall")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
val nativeCallRespMetric = Timer.builder("request.grpc.response")
.tag("type", "nativeCall")
.tag("chain", chain.chainCode)
.publishPercentileHistogram()
.register(Metrics.globalRegistry)
.tag("type", "nativeCall")
.tag("chain", chain.chainCode)
.publishPercentileHistogram()
.register(Metrics.globalRegistry)
val nativeSubscribeMetric = Counter.builder("request.grpc.request")
.tag("type", "nativeSubscribe")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
.tag("type", "nativeSubscribe")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
val nativeSubscribeRespMetric = Counter.builder("request.grpc.response")
.tag("type", "nativeSubscribe")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
.tag("type", "nativeSubscribe")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
val subscribeHeadMetric = Counter.builder("request.grpc.request")
.tag("type", "subscribeHead")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
.tag("type", "subscribeHead")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
val subscribeHeadRespMetric = Counter.builder("request.grpc.reply")
.tag("type", "subscribeHead")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
.tag("type", "subscribeHead")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
val subscribeTxMetric = Counter.builder("request.grpc.request")
.tag("type", "subscribeTx")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
.tag("type", "subscribeTx")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
val subscribeBalanceMetric = Counter.builder("request.grpc.request")
.tag("type", "subscribeBalance")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
.tag("type", "subscribeBalance")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
val subscribeBalanceRespMetric = Counter.builder("request.grpc.reply")
.tag("type", "subscribeBalance")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
.tag("type", "subscribeBalance")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
val getBalanceMetric = Counter.builder("request.grpc.request")
.tag("type", "getBalance")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
.tag("type", "getBalance")
.tag("chain", chain.chainCode)
.register(Metrics.globalRegistry)
val getBalanceRespMetric = Timer.builder("request.grpc.response")
.tag("type", "getBalance")
.tag("chain", chain.chainCode)
.publishPercentileHistogram()
.register(Metrics.globalRegistry)
.tag("type", "getBalance")
.tag("chain", chain.chainCode)
.publishPercentileHistogram()
.register(Metrics.globalRegistry)
}
}
}

View File

@@ -19,15 +19,17 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
import reactor.core.publisher.Mono
@Service
class Describe(
@Autowired private val multistreamHolder: MultistreamHolder,
@Autowired private val subscribeStatus: SubscribeStatus
@Autowired private val multistreamHolder: MultistreamHolder,
@Autowired private val subscribeStatus: SubscribeStatus
) {
fun describe(requestMono: Mono<BlockchainOuterClass.DescribeRequest>): Mono<BlockchainOuterClass.DescribeResponse> {
@@ -39,9 +41,9 @@ class Describe(
val targets = chainUpstreams.getMethods().getSupportedMethods()
val capabilities: MutableSet<Capability> = mutableSetOf()
val chainDescription = BlockchainOuterClass.DescribeChain.newBuilder()
.setChain(Common.ChainRef.forNumber(chain.id))
.addAllSupportedMethods(targets)
.setStatus(status)
.setChain(Common.ChainRef.forNumber(chain.id))
.addAllSupportedMethods(targets)
.setStatus(status)
chainUpstreams.getAll().let { ups ->
ups.forEach { up ->
val nodes = QuorumForLabels()
@@ -50,25 +52,27 @@ class Describe(
}
nodes.getAll().forEach { node ->
val nodeDetails = BlockchainOuterClass.NodeDetails.newBuilder()
.setQuorum(node.quorum)
.addAllLabels(node.labels.entries.map { label ->
.setQuorum(node.quorum)
.addAllLabels(
node.labels.entries.map { label ->
BlockchainOuterClass.Label.newBuilder()
.setName(label.key)
.setValue(label.value)
.build()
})
.setName(label.key)
.setValue(label.value)
.build()
}
)
chainDescription.addNodes(nodeDetails)
}
capabilities.addAll(up.getCapabilities())
}
}
chainDescription.addAllCapabilities(
capabilities.map {
when (it) {
Capability.RPC -> BlockchainOuterClass.Capabilities.CAP_CALLS
Capability.BALANCE -> BlockchainOuterClass.Capabilities.CAP_BALANCE
}
capabilities.map {
when (it) {
Capability.RPC -> BlockchainOuterClass.Capabilities.CAP_CALLS
Capability.BALANCE -> BlockchainOuterClass.Capabilities.CAP_BALANCE
}
}
)
resp.addChains(chainDescription.build())
}
@@ -76,5 +80,4 @@ class Describe(
resp.build()
}
}
}
}

View File

@@ -17,12 +17,11 @@ class EthereumAddresses {
Flux.just(Address.from(addresses.addressSingle.address))
Common.AnyAddress.AddrTypeCase.ADDRESS_MULTI ->
Flux.fromIterable(addresses.addressMulti.addressesList)
.map { Address.from(it.address) }
.map { Address.from(it.address) }
else -> {
log.error("Unsupported address type: ${addresses.addrTypeCase}")
Flux.empty()
}
}
}
}
}

View File

@@ -21,33 +21,35 @@ import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.dshackle.quorum.AlwaysQuorum
import io.emeraldpay.dshackle.quorum.CallQuorum
import io.emeraldpay.dshackle.quorum.NotLaggingQuorum
import io.emeraldpay.dshackle.quorum.QuorumReaderFactory
import io.emeraldpay.dshackle.upstream.ApiSource
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.calls.EthereumCallSelector
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.etherjar.rpc.RpcResponseError
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import org.apache.commons.lang3.StringUtils
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
import reactor.core.publisher.*
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.kotlin.core.publisher.toMono
import java.lang.Exception
import java.util.*
import java.util.EnumMap
@Service
open class NativeCall(
@Autowired private val multistreamHolder: MultistreamHolder
@Autowired private val multistreamHolder: MultistreamHolder
) {
private val log = LoggerFactory.getLogger(NativeCall::class.java)
@@ -69,19 +71,19 @@ open class NativeCall(
open fun nativeCall(requestMono: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<BlockchainOuterClass.NativeCallReplyItem> {
return nativeCallResult(requestMono)
.map(this::buildResponse)
.onErrorResume(this::processException)
.map(this::buildResponse)
.onErrorResume(this::processException)
}
open fun nativeCallResult(requestMono: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<CallResult> {
return requestMono.flatMapMany(this::prepareCall)
.map(this::parseParams)
.parallel()
.flatMap {
this.fetch(it)
.doOnError { e -> log.warn("Error during native call: ${e.message}") }
}
.sequential()
.map(this::parseParams)
.parallel()
.flatMap {
this.fetch(it)
.doOnError { e -> log.warn("Error during native call: ${e.message}") }
}
.sequential()
}
fun parseParams(it: CallContext<RawCallDetails>): CallContext<ParsedCallDetails> {
@@ -91,14 +93,14 @@ open class NativeCall(
fun buildResponse(it: CallResult): BlockchainOuterClass.NativeCallReplyItem {
val result = BlockchainOuterClass.NativeCallReplyItem.newBuilder()
.setSucceed(!it.isError())
.setId(it.id)
.setSucceed(!it.isError())
.setId(it.id)
if (it.isError()) {
it.error?.let { error ->
result.setErrorMessage(error.message)
}
} else {
result.setPayload(ByteString.copyFrom(it.result))
result.payload = ByteString.copyFrom(it.result)
}
return result.build()
@@ -112,11 +114,11 @@ open class NativeCall(
0
}
return BlockchainOuterClass.NativeCallReplyItem.newBuilder()
.setSucceed(false)
.setErrorMessage(it?.message ?: "Internal error")
.setId(id)
.build()
.toMono()
.setSucceed(false)
.setErrorMessage(it?.message ?: "Internal error")
.setId(id)
.build()
.toMono()
}
fun prepareCall(request: BlockchainOuterClass.NativeCallRequest): Flux<CallContext<RawCallDetails>> {
@@ -130,31 +132,35 @@ open class NativeCall(
}
val upstream = multistreamHolder.getUpstream(chain)
?: return Flux.error(CallFailure(0, SilentException.UnsupportedBlockchain(chain)))
?: return Flux.error(CallFailure(0, SilentException.UnsupportedBlockchain(chain)))
return prepareCall(request, upstream)
}
fun prepareCall(request: BlockchainOuterClass.NativeCallRequest, upstream: Multistream): Flux<CallContext<RawCallDetails>> {
fun prepareCall(
request: BlockchainOuterClass.NativeCallRequest,
upstream: Multistream
): Flux<CallContext<RawCallDetails>> {
val chain = Chain.byId(request.chainValue)
return Flux.fromIterable(request.itemsList).flatMap {
val method = it.method
val params = it.payload.toStringUtf8()
// for ethereum the actual block needed for the call may be specified in the call parameters
val callSpecificMatcher: Mono<Selector.Matcher> = if (BlockchainType.from(upstream.chain) == BlockchainType.ETHEREUM) {
ethereumCallSelectors[chain]?.getMatcher(method, params, upstream.getHead())
} else {
null
} ?: Mono.empty()
val callSpecificMatcher: Mono<Selector.Matcher> =
if (BlockchainType.from(upstream.chain) == BlockchainType.ETHEREUM) {
ethereumCallSelectors[chain]?.getMatcher(method, params, upstream.getHead())
} else {
null
} ?: Mono.empty()
callSpecificMatcher.defaultIfEmpty(Selector.empty).map { csm ->
val matcher = Selector.Builder()
.withMatcher(csm)
.forMethod(method)
.forLabels(Selector.convertToMatcher(request.selector))
.withMatcher(csm)
.forMethod(method)
.forLabels(Selector.convertToMatcher(request.selector))
val callQuorum = upstream.getMethods().getQuorumFor(method) ?: AlwaysQuorum() // can be null in tests
val callQuorum = upstream.getMethods().getQuorumFor(method) // can be null in tests
callQuorum.init(upstream.getHead())
// for NotLaggingQuorum it makes sense to select compatible upstreams before the call
@@ -172,22 +178,22 @@ open class NativeCall(
fun fetch(ctx: CallContext<ParsedCallDetails>): Mono<CallResult> {
return ctx.upstream.getRoutedApi(ctx.matcher)
.flatMap { api ->
api.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
.flatMap(JsonRpcResponse::requireResult)
.map {
CallResult.ok(ctx.id, it)
}
}.switchIfEmpty(
Mono.just(ctx).flatMap(this::executeOnRemote)
)
.onErrorResume {
if (it is CallFailure) {
Mono.just(CallResult.fail(it.id, it.reason))
} else {
Mono.just(CallResult.fail(ctx.id, it))
.flatMap { api ->
api.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
.flatMap(JsonRpcResponse::requireResult)
.map {
CallResult.ok(ctx.id, it)
}
}.switchIfEmpty(
Mono.just(ctx).flatMap(this::executeOnRemote)
)
.onErrorResume {
if (it is CallFailure) {
Mono.just(CallResult.fail(it.id, it.reason))
} else {
Mono.just(CallResult.fail(ctx.id, it))
}
}
}
fun executeOnRemote(ctx: CallContext<ParsedCallDetails>): Mono<CallResult> {
@@ -196,21 +202,21 @@ open class NativeCall(
}
val reader = quorumReaderFactory.create(ctx.getApis(), ctx.callQuorum)
return reader
.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
.map {
CallResult(ctx.id, it.value, null)
.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
.map {
CallResult(ctx.id, it.value, null)
}
.onErrorResume { t ->
val failure = if (t is CallFailure) {
CallResult.fail(t.id, t.reason)
} else {
CallResult.fail(ctx.id, t)
}
.onErrorResume { t ->
val failure = if (t is CallFailure) {
CallResult.fail(t.id, t.reason)
} else {
CallResult.fail(ctx.id, t)
}
Mono.just(failure)
}
.switchIfEmpty(
Mono.just(CallResult.fail(ctx.id, 1, "No response or no available upstream for ${ctx.payload.method}"))
)
Mono.just(failure)
}
.switchIfEmpty(
Mono.just(CallResult.fail(ctx.id, 1, "No response or no available upstream for ${ctx.payload.method}"))
)
}
@Suppress("UNCHECKED_CAST")
@@ -222,11 +228,13 @@ open class NativeCall(
return req as List<Any>
}
open class CallContext<T>(val id: Int,
val upstream: Multistream,
val matcher: Selector.Matcher,
val callQuorum: CallQuorum,
val payload: T) {
open class CallContext<T>(
val id: Int,
val upstream: Multistream,
val matcher: Selector.Matcher,
val callQuorum: CallQuorum,
val payload: T
) {
fun <X> withPayload(payload: X): CallContext<X> {
return CallContext(id, upstream, matcher, callQuorum, payload)
}
@@ -273,4 +281,4 @@ open class NativeCall(
class RawCallDetails(val method: String, val params: String)
class ParsedCallDetails(val method: String, val params: List<Any>)
}
}

View File

@@ -34,7 +34,7 @@ import reactor.core.publisher.Mono
@Service
class NativeSubscribe(
@Autowired private val multistreamHolder: MultistreamHolder
@Autowired private val multistreamHolder: MultistreamHolder
) {
companion object {
@@ -45,9 +45,9 @@ class NativeSubscribe(
fun nativeSubscribe(request: Mono<BlockchainOuterClass.NativeSubscribeRequest>): Flux<BlockchainOuterClass.NativeSubscribeReplyItem> {
return request
.flatMapMany(this@NativeSubscribe::start)
.map(this@NativeSubscribe::convertToProto)
.onErrorMap(this@NativeSubscribe::convertToStatus)
.flatMapMany(this@NativeSubscribe::start)
.map(this@NativeSubscribe::convertToProto)
.onErrorMap(this@NativeSubscribe::convertToStatus)
}
fun start(it: BlockchainOuterClass.NativeSubscribeRequest): Publisher<out Any> {
@@ -68,15 +68,15 @@ class NativeSubscribe(
fun convertToStatus(t: Throwable) = when (t) {
is SilentException.UnsupportedBlockchain -> StatusException(
Status.UNAVAILABLE.withDescription("BLOCKCHAIN UNAVAILABLE: ${t.blockchainId}")
Status.UNAVAILABLE.withDescription("BLOCKCHAIN UNAVAILABLE: ${t.blockchainId}")
)
is UnsupportedOperationException -> StatusException(
Status.UNIMPLEMENTED.withDescription(t.message)
Status.UNIMPLEMENTED.withDescription(t.message)
)
else -> {
log.warn("Unhandled error", t)
StatusException(
Status.INTERNAL.withDescription(t.message)
Status.INTERNAL.withDescription(t.message)
)
}
}
@@ -84,15 +84,14 @@ class NativeSubscribe(
fun subscribe(chain: Chain, method: String, params: Any?): Flux<out Any> {
val up = multistreamHolder.getUpstream(chain) ?: return Flux.error(SilentException.UnsupportedBlockchain(chain))
return (up as EthereumMultistream)
.getSubscribe()
.subscribe(method, params)
.getSubscribe()
.subscribe(method, params)
}
fun convertToProto(value: Any): BlockchainOuterClass.NativeSubscribeReplyItem {
val result = objectMapper.writeValueAsBytes(value)
return BlockchainOuterClass.NativeSubscribeReplyItem.newBuilder()
.setPayload(ByteString.copyFrom(result))
.build()
.setPayload(ByteString.copyFrom(result))
.build()
}
}
}

View File

@@ -30,7 +30,7 @@ import reactor.core.publisher.Mono
@Service
class StreamHead(
@Autowired private val multistreamHolder: MultistreamHolder
@Autowired private val multistreamHolder: MultistreamHolder
) {
private val log = LoggerFactory.getLogger(StreamHead::class.java)
@@ -40,24 +40,23 @@ class StreamHead(
Chain.byId(request.type.number)
}.flatMapMany { chain ->
val up = multistreamHolder.getUpstream(chain)
?: return@flatMapMany Flux.error<BlockchainOuterClass.ChainHead>(Exception("Unavailable chain: $chain"))
?: return@flatMapMany Flux.error<BlockchainOuterClass.ChainHead>(Exception("Unavailable chain: $chain"))
up.getHead()
.getFlux()
.map { asProto(chain, it!!) }
.onErrorContinue { t, _ ->
log.warn("Head subscription error: ${t.message}")
}
.getFlux()
.map { asProto(chain, it!!) }
.onErrorContinue { t, _ ->
log.warn("Head subscription error: ${t.message}")
}
}
}
fun asProto(chain: Chain, block: BlockContainer): BlockchainOuterClass.ChainHead {
return BlockchainOuterClass.ChainHead.newBuilder()
.setChainValue(chain.id)
.setHeight(block.height)
.setTimestamp(block.timestamp.toEpochMilli())
.setWeight(ByteString.copyFrom(block.difficulty.toByteArray()))
.setBlockId(block.hash.toHex())
.build()
.setChainValue(chain.id)
.setHeight(block.height)
.setTimestamp(block.timestamp.toEpochMilli())
.setWeight(ByteString.copyFrom(block.difficulty.toByteArray()))
.setBlockId(block.hash.toHex())
.build()
}
}
}

View File

@@ -18,7 +18,9 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.grpc.Chain
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
@@ -27,7 +29,7 @@ import reactor.core.publisher.Mono
@Service
class SubscribeStatus(
@Autowired private val multistreamHolder: MultistreamHolder
@Autowired private val multistreamHolder: MultistreamHolder
) {
fun subscribeStatus(requestMono: Mono<BlockchainOuterClass.StatusRequest>): Flux<BlockchainOuterClass.ChainStatus> {
@@ -52,10 +54,10 @@ class SubscribeStatus(
fun chainUnavailable(chain: Chain): BlockchainOuterClass.ChainStatus {
return BlockchainOuterClass.ChainStatus.newBuilder()
.setAvailability(BlockchainOuterClass.AvailabilityEnum.AVAIL_UNAVAILABLE)
.setChain(Common.ChainRef.forNumber(chain.id))
.setQuorum(0)
.build()
.setAvailability(BlockchainOuterClass.AvailabilityEnum.AVAIL_UNAVAILABLE)
.setChain(Common.ChainRef.forNumber(chain.id))
.setQuorum(0)
.build()
}
fun chainStatus(chain: Chain, available: UpstreamAvailability, ups: Multistream): BlockchainOuterClass.ChainStatus {
@@ -67,12 +69,11 @@ class SubscribeStatus(
0
}
return BlockchainOuterClass.ChainStatus.newBuilder()
.setAvailability(BlockchainOuterClass.AvailabilityEnum.forNumber(available.grpcId))
.setChain(Common.ChainRef.forNumber(chain.id))
.setQuorum(quorum)
.build()
.setAvailability(BlockchainOuterClass.AvailabilityEnum.forNumber(available.grpcId))
.setChain(Common.ChainRef.forNumber(chain.id))
.setQuorum(quorum)
.build()
}
class ChainSubscription(val chain: Chain, val up: Multistream, val avail: UpstreamAvailability)
}
}

View File

@@ -18,7 +18,6 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.grpc.Chain
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
/**
* Base interface to tracking balance on a single blockchain
@@ -28,5 +27,4 @@ interface TrackAddress {
fun isSupported(chain: Chain, asset: String): Boolean
fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance>
fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance>
}
}

View File

@@ -18,7 +18,6 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.upstream.Capability
@@ -27,6 +26,7 @@ import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream
import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent
import io.emeraldpay.dshackle.upstream.grpc.BitcoinGrpcUpstream
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import org.apache.commons.lang3.StringUtils
import org.bitcoinj.params.MainNetParams
@@ -37,14 +37,12 @@ import org.springframework.stereotype.Service
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import java.math.BigInteger
import java.time.Duration
import java.util.concurrent.ConcurrentHashMap
import javax.annotation.PostConstruct
import kotlin.collections.HashMap
@Service
class TrackBitcoinAddress(
@Autowired private val multistreamHolder: MultistreamHolder
@Autowired private val multistreamHolder: MultistreamHolder
) : TrackAddress {
companion object {
@@ -52,8 +50,8 @@ class TrackBitcoinAddress(
}
override fun isSupported(chain: Chain, asset: String): Boolean {
return (asset == "bitcoin" || asset == "btc" || asset == "satoshi")
&& BlockchainType.from(chain) == BlockchainType.BITCOIN && multistreamHolder.isAvailable(chain)
return (asset == "bitcoin" || asset == "btc" || asset == "satoshi") &&
BlockchainType.from(chain) == BlockchainType.BITCOIN && multistreamHolder.isAvailable(chain)
}
/**
@@ -65,8 +63,8 @@ class TrackBitcoinAddress(
* Criteria for a remote grpc upstream that can provide a balance
*/
private val balanceUpstreamMatcher = Selector.LocalAndMatcher(
Selector.GrpcMatcher(),
Selector.CapabilityMatcher(Capability.BALANCE)
Selector.GrpcMatcher(),
Selector.CapabilityMatcher(Capability.BALANCE)
)
@PostConstruct
@@ -99,7 +97,7 @@ class TrackBitcoinAddress(
return when {
request.address.hasAddressXpub() -> {
val xpubAddresses = api.getXpubAddresses()
?: return Flux.error(IllegalStateException("Xpub verification is not available"))
?: return Flux.error(IllegalStateException("Xpub verification is not available"))
val addressXpub = request.address.addressXpub
if (StringUtils.isEmpty(addressXpub.xpub)) {
@@ -109,45 +107,52 @@ class TrackBitcoinAddress(
val start = Math.max(0, addressXpub.start).toInt()
val limit = Math.min(100, Math.max(1, addressXpub.limit)).toInt()
xpubAddresses.activeAddresses(xpub, start, limit)
.map { it.toString() }
.doOnError { t -> log.error("Failed to process xpub. ${t.javaClass}:${t.message}") }
.map { it.toString() }
.doOnError { t -> log.error("Failed to process xpub. ${t.javaClass}:${t.message}") }
}
request.address.hasAddressSingle() -> {
Flux.just(request.address.addressSingle.address)
}
request.address.hasAddressMulti() -> {
Flux.fromIterable(
request.address.addressMulti.addressesList
.map { addr -> addr.address }
//TODO why sorted?
.sorted()
request.address.addressMulti.addressesList
.map { addr -> addr.address }
// TODO why sorted?
.sorted()
)
}
else -> Flux.error(IllegalArgumentException("Unsupported address type"))
}
}
fun requestBalances(chain: Chain, api: BitcoinMultistream, addresses: Flux<String>, includeUtxo: Boolean): Flux<AddressBalance> {
fun requestBalances(
chain: Chain,
api: BitcoinMultistream,
addresses: Flux<String>,
includeUtxo: Boolean
): Flux<AddressBalance> {
return addresses
.map { Address(chain, it) }
.flatMap { address ->
balanceForAddress(api, address, includeUtxo)
}
.map { Address(chain, it) }
.flatMap { address ->
balanceForAddress(api, address, includeUtxo)
}
}
fun balanceForAddress(api: BitcoinMultistream, address: Address, includeUtxo: Boolean): Mono<AddressBalance> {
return api.getReader()
.listUnspent(address.bitcoinAddress)
.map { unspent ->
totalUnspent(address, includeUtxo, unspent)
}
.switchIfEmpty(Mono.just(0).map {
.listUnspent(address.bitcoinAddress)
.map { unspent ->
totalUnspent(address, includeUtxo, unspent)
}
.switchIfEmpty(
Mono.just(0).map {
AddressBalance(address, BigInteger.ZERO)
})
.onErrorResume { t ->
log.error("Failed to get unspent", t)
Mono.empty()
}
)
.onErrorResume { t ->
log.error("Failed to get unspent", t)
Mono.empty()
}
}
fun totalUnspent(address: Address, includeUtxo: Boolean, unspent: List<SimpleUnspent>): AddressBalance {
@@ -156,10 +161,10 @@ class TrackBitcoinAddress(
} else {
unspent.map {
AddressBalance(
address,
BigInteger.valueOf(it.value),
if (includeUtxo) listOf(BalanceUtxo(it.txid, it.vout, it.value))
else emptyList()
address,
BigInteger.valueOf(it.value),
if (includeUtxo) listOf(BalanceUtxo(it.txid, it.vout, it.value))
else emptyList()
)
}.reduce { a, b -> a.plus(b) }
}
@@ -169,26 +174,32 @@ class TrackBitcoinAddress(
val ups = api.getApiSource(balanceUpstreamMatcher)
ups.request(1)
return Mono.from(ups)
.map { up ->
up.cast(BitcoinGrpcUpstream::class.java).remote
}
.timeout(Defaults.timeoutInternal, Mono.empty())
.switchIfEmpty(
Mono.just(0)
.doOnNext {
log.warn("No upstream providing balance for ${api.chain}")
}
.then(Mono.error(SilentException.DataUnavailable("BALANCE")))
)
.map { up ->
up.cast(BitcoinGrpcUpstream::class.java).remote
}
.timeout(Defaults.timeoutInternal, Mono.empty())
.switchIfEmpty(
Mono.just(0)
.doOnNext {
log.warn("No upstream providing balance for ${api.chain}")
}
.then(Mono.error(SilentException.DataUnavailable("BALANCE")))
)
}
fun getRemoteBalance(api: BitcoinMultistream, request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
fun getRemoteBalance(
api: BitcoinMultistream,
request: BlockchainOuterClass.BalanceRequest
): Flux<BlockchainOuterClass.AddressBalance> {
return getBalanceGrpc(api).flatMapMany { remote ->
remote.getBalance(request)
}
}
fun subscribeRemoteBalance(api: BitcoinMultistream, request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
fun subscribeRemoteBalance(
api: BitcoinMultistream,
request: BlockchainOuterClass.BalanceRequest
): Flux<BlockchainOuterClass.AddressBalance> {
return getBalanceGrpc(api).flatMapMany { remote ->
remote.subscribeBalance(request)
}
@@ -197,43 +208,43 @@ class TrackBitcoinAddress(
override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
val chain = Chain.byId(request.asset.chainValue)
val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java)
?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue))
?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue))
return if (isBalanceAvailable(chain)) {
val addresses = allAddresses(upstream, request)
requestBalances(chain, upstream, addresses, request.includeUtxo)
.map(this@TrackBitcoinAddress::buildResponse)
.doOnError { t ->
log.error("Failed to get balance", t)
}
.map(this@TrackBitcoinAddress::buildResponse)
.doOnError { t ->
log.error("Failed to get balance", t)
}
} else {
getRemoteBalance(upstream, request)
.doOnError { t ->
log.error("Failed to get balance from remote", t)
}
.doOnError { t ->
log.error("Failed to get balance from remote", t)
}
}
}
override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
val chain = Chain.byId(request.asset.chainValue)
val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java)
?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue))
?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue))
if (isBalanceAvailable(chain)) {
val addresses = allAddresses(upstream, request).cache()
val following = upstream.getHead().getFlux()
.flatMap {
requestBalances(chain, upstream, Flux.from(addresses), request.includeUtxo)
}
.flatMap {
requestBalances(chain, upstream, Flux.from(addresses), request.includeUtxo)
}
val last = HashMap<String, BigInteger>()
val result = following
.filter { curr ->
val prev = last[curr.address.address]
//TODO utxo can change without changing balance
val changed = prev == null || curr.balance != prev
if (changed) {
last[curr.address.address] = curr.balance
}
changed
.filter { curr ->
val prev = last[curr.address.address]
// TODO utxo can change without changing balance
val changed = prev == null || curr.balance != prev
if (changed) {
last[curr.address.address] = curr.balance
}
changed
}
return result.map(this@TrackBitcoinAddress::buildResponse)
} else {
@@ -243,24 +254,30 @@ class TrackBitcoinAddress(
private fun buildResponse(address: AddressBalance): BlockchainOuterClass.AddressBalance {
return BlockchainOuterClass.AddressBalance.newBuilder()
.setBalance(address.balance.toString(10))
.setAsset(Common.Asset.newBuilder()
.setChainValue(address.address.chain.id)
.setCode("BTC"))
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.address))
.addAllUtxo(
address.utxo.map { utxo ->
BlockchainOuterClass.Utxo.newBuilder()
.setBalance(utxo.value.toString())
.setIndex(utxo.vout.toLong())
.setTxId(utxo.txid)
.build()
}
)
.build()
.setBalance(address.balance.toString(10))
.setAsset(
Common.Asset.newBuilder()
.setChainValue(address.address.chain.id)
.setCode("BTC")
)
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.address))
.addAllUtxo(
address.utxo.map { utxo ->
BlockchainOuterClass.Utxo.newBuilder()
.setBalance(utxo.value.toString())
.setIndex(utxo.vout.toLong())
.setTxId(utxo.txid)
.build()
}
)
.build()
}
open class AddressBalance(val address: Address, var balance: BigInteger = BigInteger.ZERO, var utxo: List<BalanceUtxo> = emptyList()) {
open class AddressBalance(
val address: Address,
var balance: BigInteger = BigInteger.ZERO,
var utxo: List<BalanceUtxo> = emptyList()
) {
constructor(chain: Chain, address: String, balance: BigInteger) : this(Address(chain, address), balance)
fun plus(other: AddressBalance) = AddressBalance(address, balance + other.balance, utxo.plus(other.utxo))
@@ -268,7 +285,7 @@ class TrackBitcoinAddress(
open class BalanceUtxo(val txid: String, val vout: Int, val value: Long)
//TODO use bitcoin class for address
// TODO use bitcoin class for address
class Address(val chain: Chain, val address: String) {
val network = if (chain == Chain.BITCOIN) {
MainNetParams()
@@ -276,7 +293,7 @@ class TrackBitcoinAddress(
TestNet3Params()
}
val bitcoinAddress = org.bitcoinj.core.Address.fromString(
network, address
network, address
)
}
}
}

View File

@@ -18,11 +18,11 @@ package io.emeraldpay.dshackle.rpc
import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream
import io.emeraldpay.dshackle.upstream.bitcoin.ExtractBlock
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
@@ -37,7 +37,7 @@ import kotlin.math.min
@Service
class TrackBitcoinTx(
@Autowired private val multistreamHolder: MultistreamHolder
@Autowired private val multistreamHolder: MultistreamHolder
) : TrackTx {
companion object {
@@ -51,61 +51,69 @@ class TrackBitcoinTx(
override fun subscribe(request: BlockchainOuterClass.TxStatusRequest): Flux<BlockchainOuterClass.TxStatus> {
val chain = Chain.byId(request.chainValue)
val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java)
?: return Flux.error(SilentException.UnsupportedBlockchain(chain))
?: return Flux.error(SilentException.UnsupportedBlockchain(chain))
val txid = request.txId
val confirmations = max(min(1, request.confirmationLimit), 12)
return subscribe(chain, upstream, txid)
.takeUntil { tx ->
tx.confirmations >= confirmations
}.map(this::asProto)
.takeUntil { tx ->
tx.confirmations >= confirmations
}.map(this::asProto)
}
fun subscribe(chain: Chain, upstream: BitcoinMultistream, txid: String): Flux<TxStatus> {
return loadExisting(upstream, txid)
.flatMapMany { status ->
if (status.mined) {
//Head almost always knows the current height, so it can continue with calculating confirmations
//without publishing an empty TxStatus first
continueWithMined(upstream, status)
} else {
loadMempool(upstream, txid)
.flatMapMany { tx ->
val next = if (tx.found) {
untilMined(upstream, tx)
} else {
untilFound(chain, upstream, txid)
}
//fist provide the current status, then updates
Flux.concat(Mono.just(tx), next)
}
}
.flatMapMany { status ->
if (status.mined) {
// Head almost always knows the current height, so it can continue with calculating confirmations
// without publishing an empty TxStatus first
continueWithMined(upstream, status)
} else {
loadMempool(upstream, txid)
.flatMapMany { tx ->
val next = if (tx.found) {
untilMined(upstream, tx)
} else {
untilFound(chain, upstream, txid)
}
// fist provide the current status, then updates
Flux.concat(Mono.just(tx), next)
}
}
}
}
fun continueWithMined(upstream: BitcoinMultistream, status: TxStatus): Flux<TxStatus> {
return upstream.getReader().getBlock(status.blockHash!!)
.map { block ->
TxStatus(status.txid, true, ExtractBlock.getHeight(block), true, status.blockHash, ExtractBlock.getTime(block), ExtractBlock.getDifficulty(block))
}.flatMapMany { tx ->
withConfirmations(upstream, tx)
}
.map { block ->
TxStatus(
status.txid,
true,
ExtractBlock.getHeight(block),
true,
status.blockHash,
ExtractBlock.getTime(block),
ExtractBlock.getDifficulty(block)
)
}.flatMapMany { tx ->
withConfirmations(upstream, tx)
}
}
fun untilFound(chain: Chain, upstream: BitcoinMultistream, txid: String): Flux<TxStatus> {
return Flux.interval(Duration.ofSeconds(1))
.take(Duration.ofMinutes(10))
.flatMap { loadMempool(upstream, txid) }
.skipUntil { it.found }
.flatMap { subscribe(chain, upstream, txid) }
.doOnError { t ->
log.error("Failed to wait until found", t)
}
.take(Duration.ofMinutes(10))
.flatMap { loadMempool(upstream, txid) }
.skipUntil { it.found }
.flatMap { subscribe(chain, upstream, txid) }
.doOnError { t ->
log.error("Failed to wait until found", t)
}
}
fun untilMined(upstream: BitcoinMultistream, tx: TxStatus): Mono<TxStatus> {
return upstream.getHead().getFlux().flatMap {
loadExisting(upstream, tx.txid)
.filter { it.mined }
.filter { it.mined }
}.single()
}
@@ -136,34 +144,36 @@ class TrackBitcoinTx(
private fun asProto(tx: TxStatus): BlockchainOuterClass.TxStatus {
val data = BlockchainOuterClass.TxStatus.newBuilder()
.setTxId(tx.txid)
.setConfirmations(tx.confirmations.toInt())
.setTxId(tx.txid)
.setConfirmations(tx.confirmations.toInt())
data.broadcasted = tx.found
val isMined = tx.mined
data.mined = isMined
if (isMined) {
data.setBlock(
Common.BlockInfo.newBuilder()
.setBlockId(tx.blockHash!!.substring(2))
.setTimestamp(tx.blockTime!!.toEpochMilli())
.setWeight(ByteString.copyFrom(tx.blockTotalDifficulty!!.toByteArray()))
.setHeight(tx.height!!)
Common.BlockInfo.newBuilder()
.setBlockId(tx.blockHash!!.substring(2))
.setTimestamp(tx.blockTime!!.toEpochMilli())
.setWeight(ByteString.copyFrom(tx.blockTotalDifficulty!!.toByteArray()))
.setHeight(tx.height!!)
)
}
return data.build()
}
class TxStatus(
val txid: String,
val found: Boolean = false,
val height: Long? = null,
val mined: Boolean = false,
val blockHash: String? = null,
val blockTime: Instant? = null,
val blockTotalDifficulty: BigInteger? = null,
val confirmations: Long = 0) {
val txid: String,
val found: Boolean = false,
val height: Long? = null,
val mined: Boolean = false,
val blockHash: String? = null,
val blockTime: Instant? = null,
val blockTotalDifficulty: BigInteger? = null,
val confirmations: Long = 0
) {
fun withHead(headHeight: Long) = TxStatus(txid, found, height, mined, blockHash, blockTime, blockTotalDifficulty, headHeight - height!! + 1)
fun withHead(headHeight: Long) =
TxStatus(txid, found, height, mined, blockHash, blockTime, blockTotalDifficulty, headHeight - height!! + 1)
}
}
}

View File

@@ -2,7 +2,6 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.config.TokensConfig
import io.emeraldpay.dshackle.upstream.Head
@@ -11,25 +10,25 @@ import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
import io.emeraldpay.etherjar.domain.Address
import io.emeraldpay.etherjar.erc20.ERC20Token
import io.emeraldpay.etherjar.hex.Hex32
import io.emeraldpay.etherjar.hex.HexQuantity
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import java.math.BigInteger
import java.util.*
import java.util.Locale
import javax.annotation.PostConstruct
import kotlin.collections.HashMap
@Service
class TrackERC20Address(
@Autowired private val multistreamHolder: MultistreamHolder,
@Autowired private val tokensConfig: TokensConfig
@Autowired private val multistreamHolder: MultistreamHolder,
@Autowired private val tokensConfig: TokensConfig
) : TrackAddress {
companion object {
@@ -46,8 +45,8 @@ class TrackERC20Address(
val asset = token.name!!.lowercase(Locale.getDefault())
val id = TokenId(chain, asset)
val definition = TokenDefinition(
chain, asset,
ERC20Token(Address.from(token.address))
chain, asset,
ERC20Token(Address.from(token.address))
)
tokens[id] = definition
log.info("Enable ERC20 balance for $chain:$asset")
@@ -56,7 +55,7 @@ class TrackERC20Address(
override fun isSupported(chain: Chain, asset: String): Boolean {
return tokens.containsKey(TokenId(chain, asset.lowercase(Locale.getDefault()))) &&
BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain)
BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain)
}
override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
@@ -64,9 +63,9 @@ class TrackERC20Address(
val asset = request.asset.code.lowercase(Locale.getDefault())
val tokenDefinition = tokens[TokenId(chain, asset)] ?: return Flux.empty()
return ethereumAddresses.extract(request.address)
.map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) }
.flatMap { addr -> getBalance(addr).map(addr::withBalance) }
.map { buildResponse(it) }
.map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) }
.flatMap { addr -> getBalance(addr).map(addr::withBalance) }
.map { buildResponse(it) }
}
override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
@@ -76,63 +75,65 @@ class TrackERC20Address(
val head = multistreamHolder.getUpstream(chain)?.getHead()?.getFlux() ?: Flux.empty()
return ethereumAddresses.extract(request.address)
.map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) }
.flatMap { addr ->
val current = getBalance(addr)
val updates = head.flatMap { getBalance(addr) }
Flux.concat(current, updates)
.distinctUntilChanged()
.map { addr.withBalance(it) }
}
.map { buildResponse(it) }
.map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) }
.flatMap { addr ->
val current = getBalance(addr)
val updates = head.flatMap { getBalance(addr) }
Flux.concat(current, updates)
.distinctUntilChanged()
.map { addr.withBalance(it) }
}
.map { buildResponse(it) }
}
fun getBalance(addr: TrackedAddress): Mono<BigInteger> {
val upstream = getUpstream(addr.chain)
return upstream
.getDirectApi(Selector.empty)
.flatMap { api ->
api.read(prepareEthCall(addr.token, addr.address, upstream.getHead()))
.flatMap(JsonRpcResponse::requireStringResult)
.map {
Hex32.from(it).asQuantity().value
}
}
.getDirectApi(Selector.empty)
.flatMap { api ->
api.read(prepareEthCall(addr.token, addr.address, upstream.getHead()))
.flatMap(JsonRpcResponse::requireStringResult)
.map {
Hex32.from(it).asQuantity().value
}
}
}
fun prepareEthCall(token: ERC20Token, target: Address, head: Head): JsonRpcRequest {
val call = token
.readBalanceOf(target)
.toJson()
.readBalanceOf(target)
.toJson()
val height = head.getCurrentHeight()?.let { HexQuantity.from(it).toHex() } ?: "latest"
return JsonRpcRequest("eth_call", listOf(call, height))
}
fun getUpstream(chain: Chain): EthereumMultistream {
return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java)
?: throw SilentException.UnsupportedBlockchain(chain)
?: throw SilentException.UnsupportedBlockchain(chain)
}
private fun buildResponse(address: TrackedAddress): BlockchainOuterClass.AddressBalance {
return BlockchainOuterClass.AddressBalance.newBuilder()
.setBalance(address.balance!!.toString(10))
.setAsset(Common.Asset.newBuilder()
.setChainValue(address.chain.id)
.setCode(address.tokenName.uppercase(Locale.getDefault())))
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex()))
.build()
.setBalance(address.balance!!.toString(10))
.setAsset(
Common.Asset.newBuilder()
.setChainValue(address.chain.id)
.setCode(address.tokenName.uppercase(Locale.getDefault()))
)
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex()))
.build()
}
class TrackedAddress(val chain: Chain,
val address: Address,
val token: ERC20Token,
val tokenName: String,
val balance: BigInteger? = null
class TrackedAddress(
val chain: Chain,
val address: Address,
val token: ERC20Token,
val tokenName: String,
val balance: BigInteger? = null
) {
fun withBalance(balance: BigInteger) = TrackedAddress(chain, address, token, tokenName, balance)
}
data class TokenId(val chain: Chain, val name: String)
data class TokenDefinition(val chain: Chain, val name: String, val token: ERC20Token)
}
}

View File

@@ -18,24 +18,24 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.grpc.Chain
import io.emeraldpay.etherjar.domain.Address
import io.emeraldpay.etherjar.domain.Wei
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import java.util.*
import java.util.Locale
@Service
class TrackEthereumAddress(
@Autowired private val multistreamHolder: MultistreamHolder
@Autowired private val multistreamHolder: MultistreamHolder
) : TrackAddress {
private val log = LoggerFactory.getLogger(TrackEthereumAddress::class.java)
@@ -43,46 +43,46 @@ class TrackEthereumAddress(
override fun isSupported(chain: Chain, asset: String): Boolean {
return asset == "ether" &&
BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain)
BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain)
}
override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
return initAddress(request)
.flatMap { a -> getBalance(a).map { a.withBalance(it) } }
.map { buildResponse(it) }
.flatMap { a -> getBalance(a).map { a.withBalance(it) } }
.map { buildResponse(it) }
}
override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {
val chain = Chain.byId(request.asset.chainValue)
val head = multistreamHolder.getUpstream(chain)?.getHead()?.getFlux() ?: Flux.empty()
val balances = initAddress(request)
.flatMap { tracked ->
val current = getBalance(tracked)
.map {
tracked.withBalance(it)
}
val updates = head
.flatMap {
getBalance(tracked)
}.map {
tracked.withBalance(it)
}
Flux.concat(current, updates)
.distinctUntilChanged {
it.balance ?: Wei.ZERO
}
}
.doOnError { t ->
if (t is SilentException) {
if (t is SilentException.UnsupportedBlockchain) {
log.warn("Unsupported blockchain: ${t.blockchainId}")
}
log.debug("Failed to process subscription", t)
} else {
log.warn("Failed to process subscription", t)
.flatMap { tracked ->
val current = getBalance(tracked)
.map {
tracked.withBalance(it)
}
val updates = head
.flatMap {
getBalance(tracked)
}.map {
tracked.withBalance(it)
}
Flux.concat(current, updates)
.distinctUntilChanged {
it.balance ?: Wei.ZERO
}
}
.doOnError { t ->
if (t is SilentException) {
if (t is SilentException.UnsupportedBlockchain) {
log.warn("Unsupported blockchain: ${t.blockchainId}")
}
log.debug("Failed to process subscription", t)
} else {
log.warn("Failed to process subscription", t)
}
}
return balances.map {
buildResponse(it)
@@ -91,7 +91,7 @@ class TrackEthereumAddress(
fun getUpstream(chain: Chain): EthereumMultistream {
return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java)
?: throw SilentException.UnsupportedBlockchain(chain)
?: throw SilentException.UnsupportedBlockchain(chain)
}
private fun initAddress(request: BlockchainOuterClass.BalanceRequest): Flux<TrackedAddress> {
@@ -110,33 +110,36 @@ class TrackEthereumAddress(
private fun createAddress(address: Common.SingleAddress, chain: Chain): TrackedAddress {
val addressParsed = Address.from(address.address)
return TrackedAddress(
chain,
addressParsed
chain,
addressParsed
)
}
fun getBalance(addr: TrackedAddress): Mono<Wei> {
return getUpstream(addr.chain)
.getReader()
.balance()
.read(addr.address)
.timeout(Defaults.timeout)
.getReader()
.balance()
.read(addr.address)
.timeout(Defaults.timeout)
}
private fun buildResponse(address: TrackedAddress): BlockchainOuterClass.AddressBalance {
return BlockchainOuterClass.AddressBalance.newBuilder()
.setBalance(address.balance!!.amount!!.toString(10))
.setAsset(Common.Asset.newBuilder()
.setChainValue(address.chain.id)
.setCode("ETHER"))
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex()))
.build()
.setBalance(address.balance!!.amount!!.toString(10))
.setAsset(
Common.Asset.newBuilder()
.setChainValue(address.chain.id)
.setCode("ETHER")
)
.setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex()))
.build()
}
class TrackedAddress(val chain: Chain,
val address: Address,
val balance: Wei? = null
class TrackedAddress(
val chain: Chain,
val address: Address,
val balance: Wei? = null
) {
fun withBalance(balance: Wei) = TrackedAddress(chain, address, balance)
}
}
}

View File

@@ -19,19 +19,19 @@ package io.emeraldpay.dshackle.rpc
import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.grpc.Chain
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.domain.TransactionId
import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.etherjar.rpc.json.BlockJson
import io.emeraldpay.etherjar.rpc.json.TransactionJson
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
@@ -48,7 +48,7 @@ import kotlin.math.min
@Service
class TrackEthereumTx(
@Autowired private val multistreamHolder: MultistreamHolder
@Autowired private val multistreamHolder: MultistreamHolder
) : TrackTx {
companion object {
@@ -70,63 +70,62 @@ class TrackEthereumTx(
val base = prepareTracking(request)
val up = getUpstream(base.chain)
return update(base)
.defaultIfEmpty(base)
.flatMapMany {
Flux.concat(Mono.just(it), subscribe(it, up))
.distinctUntilChanged(TxDetails::status)
.map(this@TrackEthereumTx::asProto)
.subscribeOn(scheduler)
}
.doOnError { t ->
log.error("Subscription error", t)
}
.defaultIfEmpty(base)
.flatMapMany {
Flux.concat(Mono.just(it), subscribe(it, up))
.distinctUntilChanged(TxDetails::status)
.map(this@TrackEthereumTx::asProto)
.subscribeOn(scheduler)
}
.doOnError { t ->
log.error("Subscription error", t)
}
}
fun getUpstream(chain: Chain): EthereumMultistream {
return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java)
?: throw SilentException.UnsupportedBlockchain(chain)
?: throw SilentException.UnsupportedBlockchain(chain)
}
fun subscribe(base: TxDetails, up: EthereumMultistream): Flux<TxDetails> {
var latestTx = base
val untilFound = Mono.just(latestTx)
.subscribeOn(scheduler)
.map {
//replace with the latest value, it may be already found
latestTx
.subscribeOn(scheduler)
.map {
// replace with the latest value, it may be already found
latestTx
}
.flatMap { latest ->
if (!latest.status.found) {
update(latest).defaultIfEmpty(latestTx)
} else {
Mono.just(latest)
}
.flatMap { latest ->
if (!latest.status.found) {
update(latest).defaultIfEmpty(latestTx)
} else {
Mono.just(latest)
}
}
.flatMap { received ->
if (!received.status.found) {
Mono.error(SilentException("Retry not found"))
} else {
Mono.just(received)
}
.flatMap { received ->
if (!received.status.found) {
Mono.error(SilentException("Retry not found"))
} else {
Mono.just(received)
}
}
.retryWhen(
Retry.fixedDelay(10, Duration.ofSeconds(2))
)
.onErrorResume { Mono.empty() }
}
.retryWhen(
Retry.fixedDelay(10, Duration.ofSeconds(2))
)
.onErrorResume { Mono.empty() }
val inBlocks = up.getHead().getFlux()
.subscribeOn(scheduler)
.flatMap { block ->
onNewBlock(latestTx, block)
}
.subscribeOn(scheduler)
.flatMap { block ->
onNewBlock(latestTx, block)
}
return Flux.merge(untilFound, inBlocks)
.takeUntil(TxDetails::shouldClose)
.doOnNext { newTx ->
latestTx = newTx
}
.takeUntil(TxDetails::shouldClose)
.doOnNext { newTx ->
latestTx = newTx
}
}
fun onNewBlock(tx: TxDetails, block: BlockContainer): Mono<TxDetails> {
@@ -134,7 +133,8 @@ class TrackEthereumTx(
if (!tx.status.mined) {
val justMined = block.transactions.contains(txid)
return if (justMined) {
Mono.just(tx.withStatus(
Mono.just(
tx.withStatus(
mined = true,
found = true,
confirmations = 1,
@@ -142,13 +142,14 @@ class TrackEthereumTx(
blockTime = block.timestamp,
blockTotalDifficulty = block.difficulty,
blockHash = BlockHash(block.hash.value)
))
)
)
} else {
update(tx)
}
} else {
//verify if it's still on chain
//TODO head is supposed to erase block when it was replaced, so can safely recalc here
// verify if it's still on chain
// TODO head is supposed to erase block when it was replaced, so can safely recalc here
return update(tx)
}
}
@@ -157,19 +158,19 @@ class TrackEthereumTx(
val initialStatus = tx.status
val upstream = getUpstream(tx.chain)
return upstream.getReader()
.txByHash().read(tx.txid)
.onErrorResume(RpcException::class.java) { t ->
log.warn("Upstream error, ignoring. {}", t.rpcMessage)
Mono.empty<TransactionJson>()
}
.flatMap { updateFromBlock(upstream, tx, it) }
.doOnError { t ->
log.error("Failed to load tx block", t)
}
.switchIfEmpty(Mono.just(tx.withStatus(found = false)))
.filter { current ->
initialStatus != current.status || current.shouldClose()
}
.txByHash().read(tx.txid)
.onErrorResume(RpcException::class.java) { t ->
log.warn("Upstream error, ignoring. {}", t.rpcMessage)
Mono.empty<TransactionJson>()
}
.flatMap { updateFromBlock(upstream, tx, it) }
.doOnError { t ->
log.error("Failed to load tx block", t)
}
.switchIfEmpty(Mono.just(tx.withStatus(found = false)))
.filter { current ->
initialStatus != current.status || current.shouldClose()
}
}
fun prepareTracking(request: BlockchainOuterClass.TxStatusRequest): TxDetails {
@@ -178,10 +179,10 @@ class TrackEthereumTx(
throw SilentException.UnsupportedBlockchain(request.chainValue)
}
val details = TxDetails(
chain,
Instant.now(),
TransactionId.from(request.txId),
min(max(1, request.confirmationLimit), 100)
chain,
Instant.now(),
TransactionId.from(request.txId),
min(max(1, request.confirmationLimit), 100)
)
return details
}
@@ -189,12 +190,12 @@ class TrackEthereumTx(
fun setBlockDetails(tx: TxDetails, block: BlockJson<TransactionRefJson>): TxDetails {
return if (block.number != null && block.totalDifficulty != null) {
tx.withStatus(
blockTotalDifficulty = block.totalDifficulty,
blockTime = block.timestamp
blockTotalDifficulty = block.totalDifficulty,
blockTime = block.timestamp
)
} else {
tx.withStatus(
mined = false
mined = false
)
}
}
@@ -205,22 +206,22 @@ class TrackEthereumTx(
return Mono.empty()
}
return upstream.getReader()
.blocksByHashParsed().read(tx.status.blockHash)
.map { block ->
setBlockDetails(tx, block)
}.doOnError { t ->
log.warn("Failed to update weight", t)
}
.blocksByHashParsed().read(tx.status.blockHash)
.map { block ->
setBlockDetails(tx, block)
}.doOnError { t ->
log.warn("Failed to update weight", t)
}
}
fun updateFromBlock(upstream: EthereumMultistream, tx: TxDetails, blockTx: TransactionJson): Mono<TxDetails> {
return if (blockTx.blockNumber != null && blockTx.blockHash != null && blockTx.blockHash != ZERO_BLOCK) {
val updated = tx.withStatus(
blockHash = blockTx.blockHash,
height = blockTx.blockNumber,
found = true,
mined = true,
confirmations = 1
blockHash = blockTx.blockHash,
height = blockTx.blockNumber,
found = true,
mined = true,
confirmations = 1
)
upstream.getHead().getFlux().next().map { head ->
val height = updated.status.height
@@ -228,72 +229,89 @@ class TrackEthereumTx(
updated
} else {
updated.withStatus(
confirmations = head.height - height + 1
confirmations = head.height - height + 1
)
}
}.doOnError { t ->
log.error("Unable to load head details", t)
}.flatMap(this::loadWeight)
} else {
Mono.just(tx.withStatus(
Mono.just(
tx.withStatus(
found = true,
mined = false
))
)
)
}
}
private fun asProto(tx: TxDetails): BlockchainOuterClass.TxStatus {
val data = BlockchainOuterClass.TxStatus.newBuilder()
.setTxId(tx.txid.toHex())
.setConfirmations(tx.status.confirmations.toInt())
.setTxId(tx.txid.toHex())
.setConfirmations(tx.status.confirmations.toInt())
data.broadcasted = tx.status.found
val isMined = tx.status.mined
data.mined = isMined
if (isMined) {
data.setBlock(
Common.BlockInfo.newBuilder()
.setBlockId(tx.status.blockHash!!.toHex().substring(2))
.setTimestamp(tx.status.blockTime!!.toEpochMilli())
.setWeight(ByteString.copyFrom(tx.status.blockTotalDifficulty!!.toByteArray()))
.setHeight(tx.status.height!!)
Common.BlockInfo.newBuilder()
.setBlockId(tx.status.blockHash!!.toHex().substring(2))
.setTimestamp(tx.status.blockTime!!.toEpochMilli())
.setWeight(ByteString.copyFrom(tx.status.blockTotalDifficulty!!.toByteArray()))
.setHeight(tx.status.height!!)
)
}
return data.build()
}
class TxDetails(val chain: Chain,
val since: Instant,
val txid: TransactionId,
val maxConfirmations: Int,
val status: TxStatus
class TxDetails(
val chain: Chain,
val since: Instant,
val txid: TransactionId,
val maxConfirmations: Int,
val status: TxStatus
) {
constructor(chain: Chain,
since: Instant,
txid: TransactionId,
maxConfirmations: Int) : this(chain, since, txid, maxConfirmations, TxStatus())
constructor(
chain: Chain,
since: Instant,
txid: TransactionId,
maxConfirmations: Int
) : this(chain, since, txid, maxConfirmations, TxStatus())
fun copy(
since: Instant = this.since,
status: TxStatus = this.status
since: Instant = this.since,
status: TxStatus = this.status
) = TxDetails(chain, since, txid, maxConfirmations, status)
fun withStatus(found: Boolean = this.status.found,
height: Long? = this.status.height,
mined: Boolean = this.status.mined,
blockHash: BlockHash? = this.status.blockHash,
blockTime: Instant? = this.status.blockTime,
blockTotalDifficulty: BigInteger? = this.status.blockTotalDifficulty,
confirmations: Long = this.status.confirmations): TxDetails {
return copy(status = this.status.copy(found, height, mined, blockHash, blockTime, blockTotalDifficulty, confirmations))
fun withStatus(
found: Boolean = this.status.found,
height: Long? = this.status.height,
mined: Boolean = this.status.mined,
blockHash: BlockHash? = this.status.blockHash,
blockTime: Instant? = this.status.blockTime,
blockTotalDifficulty: BigInteger? = this.status.blockTotalDifficulty,
confirmations: Long = this.status.confirmations
): TxDetails {
return copy(
status = this.status.copy(
found,
height,
mined,
blockHash,
blockTime,
blockTotalDifficulty,
confirmations
)
)
}
fun shouldClose(): Boolean {
return maxConfirmations <= this.status.confirmations
|| since.isBefore(Instant.now().minus(TRACK_TTL))
|| (!status.found && since.isBefore(Instant.now().minus(NOT_FOUND_TRACK_TTL)))
|| (!status.mined && since.isBefore(Instant.now().minus(NOT_MINED_TRACK_TTL)))
return maxConfirmations <= this.status.confirmations ||
since.isBefore(Instant.now().minus(TRACK_TTL)) ||
(!status.found && since.isBefore(Instant.now().minus(NOT_FOUND_TRACK_TTL))) ||
(!status.mined && since.isBefore(Instant.now().minus(NOT_MINED_TRACK_TTL)))
}
override fun toString(): String {
@@ -320,25 +338,27 @@ class TrackEthereumTx(
result = 31 * result + status.hashCode()
return result
}
}
class TxStatus(val found: Boolean = false,
val height: Long? = null,
val mined: Boolean = false,
val blockHash: BlockHash? = null,
val blockTime: Instant? = null,
val blockTotalDifficulty: BigInteger? = null,
val confirmations: Long = 0) {
class TxStatus(
val found: Boolean = false,
val height: Long? = null,
val mined: Boolean = false,
val blockHash: BlockHash? = null,
val blockTime: Instant? = null,
val blockTotalDifficulty: BigInteger? = null,
val confirmations: Long = 0
) {
fun copy(found: Boolean = this.found,
height: Long? = this.height,
mined: Boolean = this.mined,
blockHash: BlockHash? = this.blockHash,
blockTime: Instant? = this.blockTime,
blockTotalDifficulty: BigInteger? = this.blockTotalDifficulty,
confirmation: Long = this.confirmations)
= TxStatus(found, height, mined, blockHash, blockTime, blockTotalDifficulty, confirmation)
fun copy(
found: Boolean = this.found,
height: Long? = this.height,
mined: Boolean = this.mined,
blockHash: BlockHash? = this.blockHash,
blockTime: Instant? = this.blockTime,
blockTotalDifficulty: BigInteger? = this.blockTotalDifficulty,
confirmation: Long = this.confirmations
) = TxStatus(found, height, mined, blockHash, blockTime, blockTotalDifficulty, confirmation)
fun clean() = TxStatus(false, null, false, null, null, null, 0)
@@ -369,7 +389,5 @@ class TrackEthereumTx(
override fun toString(): String {
return "TxStatus(found=$found, height=$height, mined=$mined, blockHash=$blockHash, blockTime=$blockTime, blockTotalDifficulty=$blockTotalDifficulty, confirmations=$confirmations)"
}
}
}
}

View File

@@ -22,4 +22,4 @@ import reactor.core.publisher.Flux
interface TrackTx {
fun isSupported(chain: Chain): Boolean
fun subscribe(request: BlockchainOuterClass.TxStatusRequest): Flux<BlockchainOuterClass.TxStatus>
}
}

View File

@@ -16,7 +16,6 @@
*/
package io.emeraldpay.dshackle.startup
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.dshackle.FileResolver
import io.emeraldpay.dshackle.cache.CachesFactory
import io.emeraldpay.dshackle.config.UpstreamsConfig
@@ -34,49 +33,48 @@ import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcHttpClient
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import io.micrometer.core.instrument.Counter
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Repository
import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Tag
import io.micrometer.core.instrument.Timer
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Repository
import java.net.URI
import java.util.*
import java.util.concurrent.atomic.AtomicInteger
import javax.annotation.PostConstruct
import kotlin.collections.HashMap
@Repository
open class ConfiguredUpstreams(
@Autowired private val currentUpstreams: CurrentMultistreamHolder,
@Autowired private val fileResolver: FileResolver,
@Autowired private val config: UpstreamsConfig,
@Autowired private val cachesFactory: CachesFactory
@Autowired private val currentUpstreams: CurrentMultistreamHolder,
@Autowired private val fileResolver: FileResolver,
@Autowired private val config: UpstreamsConfig,
@Autowired private val cachesFactory: CachesFactory
) {
private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java)
private var seq = AtomicInteger(0)
private val chainNames = mapOf(
"ethereum" to Chain.ETHEREUM,
"ethereum-classic" to Chain.ETHEREUM_CLASSIC,
"eth" to Chain.ETHEREUM,
"polygon" to Chain.MATIC,
"matic" to Chain.MATIC,
"etc" to Chain.ETHEREUM_CLASSIC,
"morden" to Chain.TESTNET_MORDEN,
"kovan" to Chain.TESTNET_KOVAN,
"kovan-testnet" to Chain.TESTNET_KOVAN,
"goerli" to Chain.TESTNET_GOERLI,
"goerli-testnet" to Chain.TESTNET_GOERLI,
"rinkeby" to Chain.TESTNET_RINKEBY,
"rinkeby-testnet" to Chain.TESTNET_RINKEBY,
"ropsten" to Chain.TESTNET_ROPSTEN,
"ropsten-testnet" to Chain.TESTNET_ROPSTEN,
"bitcoin" to Chain.BITCOIN,
"bitcoin-testnet" to Chain.TESTNET_BITCOIN
"ethereum" to Chain.ETHEREUM,
"ethereum-classic" to Chain.ETHEREUM_CLASSIC,
"eth" to Chain.ETHEREUM,
"polygon" to Chain.MATIC,
"matic" to Chain.MATIC,
"etc" to Chain.ETHEREUM_CLASSIC,
"morden" to Chain.TESTNET_MORDEN,
"kovan" to Chain.TESTNET_KOVAN,
"kovan-testnet" to Chain.TESTNET_KOVAN,
"goerli" to Chain.TESTNET_GOERLI,
"goerli-testnet" to Chain.TESTNET_GOERLI,
"rinkeby" to Chain.TESTNET_RINKEBY,
"rinkeby-testnet" to Chain.TESTNET_RINKEBY,
"ropsten" to Chain.TESTNET_ROPSTEN,
"ropsten-testnet" to Chain.TESTNET_ROPSTEN,
"bitcoin" to Chain.BITCOIN,
"bitcoin-testnet" to Chain.TESTNET_BITCOIN
)
@PostConstruct
@@ -95,7 +93,7 @@ open class ConfiguredUpstreams(
return@forEach
}
val options = (up.options ?: UpstreamsConfig.Options())
.merge(defaultOptions[chain] ?: UpstreamsConfig.Options.getDefaults())
.merge(defaultOptions[chain] ?: UpstreamsConfig.Options.getDefaults())
when (BlockchainType.from(chain)) {
BlockchainType.ETHEREUM -> {
buildEthereumUpstream(up.cast(UpstreamsConfig.EthereumConnection::class.java), chain, options)
@@ -135,9 +133,10 @@ open class ConfiguredUpstreams(
fun buildMethods(config: UpstreamsConfig.Upstream<*>, chain: Chain): CallMethods {
return if (config.methods != null) {
ManagedCallMethods(currentUpstreams.getDefaultMethods(chain),
config.methods!!.enabled.map { it.name }.toSet(),
config.methods!!.disabled.map { it.name }.toSet()
ManagedCallMethods(
currentUpstreams.getDefaultMethods(chain),
config.methods!!.enabled.map { it.name }.toSet(),
config.methods!!.disabled.map { it.name }.toSet()
).also {
config.methods!!.enabled.forEach { m ->
if (m.quorum != null) {
@@ -150,9 +149,11 @@ open class ConfiguredUpstreams(
}
}
private fun buildBitcoinUpstream(config: UpstreamsConfig.Upstream<UpstreamsConfig.BitcoinConnection>,
chain: Chain,
options: UpstreamsConfig.Options) {
private fun buildBitcoinUpstream(
config: UpstreamsConfig.Upstream<UpstreamsConfig.BitcoinConnection>,
chain: Chain,
options: UpstreamsConfig.Options
) {
val conn = config.connection!!
val directApi: Reader<JsonRpcRequest, JsonRpcResponse>? = buildHttpClient(config)
@@ -171,19 +172,24 @@ open class ConfiguredUpstreams(
}
val methods = buildMethods(config, chain)
val upstream = BitcoinRpcUpstream(config.id
?: "bitcoin-${seq.getAndIncrement()}", chain, directApi,
options, config.role,
QuorumForLabels.QuorumItem(1, config.labels),
methods, esplora)
val upstream = BitcoinRpcUpstream(
config.id
?: "bitcoin-${seq.getAndIncrement()}",
chain, directApi,
options, config.role,
QuorumForLabels.QuorumItem(1, config.labels),
methods, esplora
)
upstream.start()
currentUpstreams.update(UpstreamChange(chain, upstream, UpstreamChange.ChangeType.ADDED))
}
private fun buildEthereumUpstream(config: UpstreamsConfig.Upstream<UpstreamsConfig.EthereumConnection>,
chain: Chain,
options: UpstreamsConfig.Options) {
private fun buildEthereumUpstream(
config: UpstreamsConfig.Upstream<UpstreamsConfig.EthereumConnection>,
chain: Chain,
options: UpstreamsConfig.Options
) {
val conn = config.connection!!
val urls = ArrayList<URI>()
@@ -194,8 +200,8 @@ open class ConfiguredUpstreams(
val wsFactoryApi: EthereumWsFactory? = conn.ws?.let { endpoint ->
val wsApi = EthereumWsFactory(
endpoint.url,
endpoint.origin ?: URI("http://localhost"),
endpoint.url,
endpoint.origin ?: URI("http://localhost"),
)
wsApi.config = endpoint
endpoint.basicAuth?.let { auth ->
@@ -208,11 +214,11 @@ open class ConfiguredUpstreams(
log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}")
val ethereumUpstream = if (wsFactoryApi != null && !conn.preferHttp) {
EthereumWsUpstream(
config.id!!,
chain, wsFactoryApi,
options, config.role,
QuorumForLabels.QuorumItem(1, config.labels),
methods
config.id!!,
chain, wsFactoryApi,
options, config.role,
QuorumForLabels.QuorumItem(1, config.labels),
methods
)
} else {
val directApi: Reader<JsonRpcRequest, JsonRpcResponse>? = buildHttpClient(config)
@@ -221,11 +227,11 @@ open class ConfiguredUpstreams(
return
}
EthereumRpcUpstream(
config.id!!,
chain, directApi, wsFactoryApi,
options, config.role,
QuorumForLabels.QuorumItem(1, config.labels),
methods
config.id!!,
chain, directApi, wsFactoryApi,
options, config.role,
QuorumForLabels.QuorumItem(1, config.labels),
methods
)
}
@@ -233,23 +239,26 @@ open class ConfiguredUpstreams(
currentUpstreams.update(UpstreamChange(chain, ethereumUpstream, UpstreamChange.ChangeType.ADDED))
}
private fun buildGrpcUpstream(config: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>, options: UpstreamsConfig.Options) {
private fun buildGrpcUpstream(
config: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>,
options: UpstreamsConfig.Options
) {
val endpoint = config.connection!!
val ds = GrpcUpstreams(
config.id!!,
endpoint.host!!,
endpoint.port ?: 2449,
endpoint.auth,
fileResolver
config.id!!,
endpoint.host!!,
endpoint.port,
endpoint.auth,
fileResolver
).apply {
timeout = options.timeout
}
log.info("Using ALL CHAINS (gRPC) upstream, at ${endpoint.host}:${endpoint.port}")
ds.start()
.doOnNext {
log.info("Chain ${it.chain} has ${it.type} through gRPC at ${endpoint.host}:${endpoint.port}")
}
.subscribe(currentUpstreams::update)
.doOnNext {
log.info("Chain ${it.chain} has ${it.type} through gRPC at ${endpoint.host}:${endpoint.port}")
}
.subscribe(currentUpstreams::update)
}
private fun buildHttpClient(config: UpstreamsConfig.Upstream<out UpstreamsConfig.RpcConnection>): JsonRpcHttpClient? {
@@ -262,29 +271,29 @@ open class ConfiguredUpstreams(
}
}
val metricsTags = listOf(
// "unknown" is not supposed to happen
Tag.of("upstream", config.id ?: "unknown"),
// UNSPECIFIED shouldn't happen too
Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED ).chainCode)
// "unknown" is not supposed to happen
Tag.of("upstream", config.id ?: "unknown"),
// UNSPECIFIED shouldn't happen too
Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED).chainCode)
)
val metrics = RpcMetrics(
Timer.builder("upstream.rpc.conn")
.description("Request time through a HTTP JSON RPC connection")
.tags(metricsTags)
.publishPercentileHistogram()
.register(Metrics.globalRegistry),
Counter.builder("upstream.rpc.err")
.description("Errors received on request through HTTP JSON RPC connection")
.tags(metricsTags)
.register(Metrics.globalRegistry)
Timer.builder("upstream.rpc.conn")
.description("Request time through a HTTP JSON RPC connection")
.tags(metricsTags)
.publishPercentileHistogram()
.register(Metrics.globalRegistry),
Counter.builder("upstream.rpc.err")
.description("Errors received on request through HTTP JSON RPC connection")
.tags(metricsTags)
.register(Metrics.globalRegistry)
)
urls.add(endpoint.url)
JsonRpcHttpClient(
endpoint.url.toString(),
metrics,
conn.rpc?.basicAuth,
tls
endpoint.url.toString(),
metrics,
conn.rpc?.basicAuth,
tls
)
}
}
}
}

View File

@@ -17,9 +17,8 @@
package io.emeraldpay.dshackle.startup
import io.emeraldpay.dshackle.config.UpstreamsConfig
import java.util.*
import java.util.Collections
import java.util.concurrent.locks.ReentrantReadWriteLock
import kotlin.collections.ArrayList
import kotlin.concurrent.read
import kotlin.concurrent.write
@@ -71,7 +70,6 @@ class QuorumForLabels() {
return nodes.hashCode()
}
/**
* Details for a single element (upstream, node or aggregation)
*/
@@ -97,8 +95,5 @@ class QuorumForLabels() {
result = 31 * result + labels.hashCode()
return result
}
}
}
}

Some files were not shown because too many files have changed in this diff Show More