StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

problem: duplicate logic for Head following

Browse Source 
solution: refactor to default implementation
This commit is contained in:
Igor Artamonov
2019-09-04 21:48:22 -04:00
parent ee66704620
commit d553560be2
9 changed files with 220 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt Normal file
View File

@@ -0,0 +1,50 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson
import org.slf4j.LoggerFactory
import reactor.core.Disposable
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.core.publisher.TopicProcessor
import java.util.concurrent.atomic.AtomicReference
open class DefaultEthereumHead: EthereumHead {
private val log = LoggerFactory.getLogger(DefaultEthereumHead::class.java)
private val head = AtomicReference<BlockJson<TransactionId>>(null)
private val stream: TopicProcessor<BlockJson<TransactionId>> = TopicProcessor.create()
fun follow(source: Flux<BlockJson<TransactionId>>): Disposable {
return source.distinctUntilChanged {
it.hash
}.filter { block ->
val curr = head.get()
curr == null || curr.totalDifficulty < block.totalDifficulty
}
.subscribe { block ->
val prev = head.getAndUpdate { curr ->
if (curr == null || curr.totalDifficulty < block.totalDifficulty) {
block
} else {
curr
}
}
if (prev == null || prev.hash != block.hash) {
log.debug("New block ${block.number} ${block.hash}")
stream.onNext(block)
}
}
}
override fun getFlux(): Flux<BlockJson<TransactionId>> {
return Flux.merge(
Mono.justOrEmpty(head.get()),
Flux.from(stream)
).onBackpressureLatest()
}
fun getCurrent(): BlockJson<TransactionId>? {
return head.get()
}
}

2
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DirectEthereumApi.kt
View File

@@ -42,7 +42,7 @@ open class DirectEthereumApi(
}
return result
.doOnError { t ->
log.warn("Upstream error: [${t.message}] for ${method}")
log.warn("Upstream error: [${t.message}] for $method")
}
.map {
val resp = ResponseJson<Any, Int>()

31
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumHeadMerge.kt
View File

@@ -26,42 +26,17 @@ import reactor.core.publisher.Mono
import java.util.concurrent.atomic.AtomicReference
class EthereumHeadMerge(
fluxes: Iterable<Publisher<BlockJson<TransactionId>>>
): EthereumHead, Lifecycle {
private val fluxes: Iterable<Publisher<BlockJson<TransactionId>>>
): DefaultEthereumHead(), Lifecycle {
private val log = LoggerFactory.getLogger(EthereumHeadMerge::class.java)
private val flux: Flux<BlockJson<TransactionId>>
private val head = AtomicReference<BlockJson<TransactionId>>(null)
private var subscription: Disposable? = null
init {
flux = Flux.merge(fluxes)
.distinctUntilChanged {
it.hash
}
.filter {
val curr = head.get()
curr == null || curr.totalDifficulty < it.totalDifficulty
}
.publish()
.autoConnect()
}
override fun isRunning(): Boolean {
return subscription != null
}
override fun start() {
subscription = Flux.from(flux).subscribe {
head.set(it)
}
}
override fun getFlux(): Flux<BlockJson<TransactionId>> {
return Flux.merge(
Mono.justOrEmpty(head.get()),
Flux.from(this.flux)
).onBackpressureLatest()
subscription = super.follow(Flux.merge(fluxes))
}
override fun stop() {

34
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt
View File

@@ -15,55 +15,42 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.Batch
import io.infinitape.etherjar.rpc.Commands
import io.infinitape.etherjar.rpc.json.BlockJson
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.Disposable
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.core.publisher.TopicProcessor
import java.time.Duration
import java.util.concurrent.atomic.AtomicReference
class EthereumRpcHead(
private val api: DirectEthereumApi
): EthereumHead, Lifecycle {
private val api: DirectEthereumApi,
private val interval: Duration = Duration.ofSeconds(10)
): DefaultEthereumHead(), Lifecycle {
private val log = LoggerFactory.getLogger(EthereumRpcHead::class.java)
private val head = AtomicReference<BlockJson<TransactionId>>(null)
private val stream: TopicProcessor<BlockJson<TransactionId>> = TopicProcessor.create()
private var refreshSubscription: Disposable? = null
override fun start() {
refreshSubscription = Flux.interval(Duration.ofSeconds(7))
val base = Flux.interval(interval)
.flatMap {
val batch = Batch()
val f = batch.add(Commands.eth().blockNumber)
api.rpcClient.execute(batch)
Mono.fromCompletionStage(f).timeout(Duration.ofSeconds(5))
Mono.fromCompletionStage(f).timeout(Duration.ofSeconds(5), Mono.empty())
}
.flatMap {
val batch = Batch()
val f = batch.add(Commands.eth().getBlock(it))
api.rpcClient.execute(batch)
Mono.fromCompletionStage(f).timeout(Duration.ofSeconds(5))
Mono.fromCompletionStage(f).timeout(Duration.ofSeconds(5), Mono.empty())
}
.onErrorContinue { err, _ ->
log.debug("RPC error ${err.message}")
}
.distinctUntilChanged { it.hash }
.filter { block ->
val curr = head.get()
curr == null || curr.totalDifficulty < block.totalDifficulty
}
.subscribe { block ->
head.set(block)
stream.onNext(block)
}
refreshSubscription = super.follow(base)
}
override fun isRunning(): Boolean {
@@ -75,11 +62,4 @@ class EthereumRpcHead(
refreshSubscription = null
}
override fun getFlux(): Flux<BlockJson<TransactionId>> {
return Flux.merge(
Mono.justOrEmpty(head.get()),
Flux.from(stream)
).onBackpressureLatest()
}
}

16
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt
View File

@@ -21,6 +21,7 @@ import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.Disposable
import java.time.Duration
open class EthereumUpstream(
private val id: String,
@@ -70,21 +71,20 @@ open class EthereumUpstream(
override fun stop() {
validatorSubscription?.dispose()
validatorSubscription = null
if (head is Lifecycle) {
head.stop()
}
}
open fun createHead(): EthereumHead {
return if (ethereumWs != null) {
// load current block through RPC then listen for following blocks through WS
val ws = EthereumWsHead(ethereumWs).apply {
this.start()
}
val rpc = EthereumRpcHead(api).apply {
val rpc = EthereumRpcHead(api, Duration.ofSeconds(20)).apply {
this.start()
}
val currentHead = rpc.getFlux().next().doFinally {
rpc.stop()
}
EthereumHeadMerge(listOf(currentHead, ws.getFlux())).apply {
EthereumHeadMerge(listOf(rpc.getFlux(), ws.getFlux())).apply {
this.start()
}
} else {
@@ -106,10 +106,6 @@ open class EthereumUpstream(
return api
}
fun getApi(): DirectEthereumApi {
return api
}
override fun getOptions(): UpstreamsConfig.Options {
return options
}

28
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt
View File

@@ -15,48 +15,24 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.Disposable
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import java.util.concurrent.atomic.AtomicReference
class EthereumWsHead(
private val ws: EthereumWs
): EthereumHead, Lifecycle {
): DefaultEthereumHead(), Lifecycle {
private val log = LoggerFactory.getLogger(EthereumWsHead::class.java)
private var subscription: Disposable? = null
private val head = AtomicReference<BlockJson<TransactionId>>(null)
private var stream: Flux<BlockJson<TransactionId>>? = null
override fun getFlux(): Flux<BlockJson<TransactionId>> {
return stream?.let {
Flux.merge(
Mono.justOrEmpty(head.get()),
Flux.from(this.stream)
).onBackpressureLatest()
} ?: Flux.error(Exception("Not started"))
}
override fun isRunning(): Boolean {
return subscription != null
}
override fun start() {
val flux = ws.getFlux()
.distinctUntilChanged { it.hash }
.filter { block ->
val curr = head.get()
curr == null || curr.totalDifficulty < block.totalDifficulty
}.share()
this.subscription = flux.subscribe(head::set)
this.stream = flux
this.subscription = super.follow(ws.getFlux())
}
override fun stop() {

84
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstream.kt
View File

@@ -22,6 +22,7 @@ import io.emeraldpay.api.proto.Common
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.dshackle.upstream.ethereum.DefaultEthereumHead
import io.emeraldpay.dshackle.upstream.ethereum.DirectEthereumApi
import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead
import io.emeraldpay.grpc.Chain
@@ -35,7 +36,6 @@ import org.springframework.context.Lifecycle
import reactor.core.Disposable
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.core.publisher.TopicProcessor
import reactor.core.publisher.toMono
import java.lang.Exception
import java.math.BigInteger
@@ -57,10 +57,8 @@ open class GrpcUpstream(
private val log = LoggerFactory.getLogger(GrpcUpstream::class.java)
private val options = UpstreamsConfig.Options.getDefaults()
private val headBlock = AtomicReference<BlockJson<TransactionId>>(null)
private val streamBlocks: TopicProcessor<BlockJson<TransactionId>> = TopicProcessor.create()
private val nodes = AtomicReference<NodeDetailsList>(NodeDetailsList())
private val head = Head(this)
private val head = DefaultEthereumHead()
private var targets: CallMethods? = null
private var headSubscription: Disposable? = null
@@ -110,40 +108,36 @@ open class GrpcUpstream(
internal fun observeHead(flux: Flux<BlockchainOuterClass.ChainHead>) {
headSubscription = flux.map { value ->
val block = BlockJson<TransactionId>()
block.number = value.height
block.totalDifficulty = BigInteger(1, value.weight.toByteArray())
block.hash = BlockHash.from("0x"+value.blockId)
block
}
.distinctUntilChanged { it.hash }
.filter { block ->
val curr = headBlock.get()
curr == null || curr.totalDifficulty < block.totalDifficulty
}
.flatMap {
getApi(Selector.EmptyMatcher())
.executeAndConvert(Commands.eth().getBlock(it.hash))
.timeout(Duration.ofSeconds(5), Mono.error(Exception("Timeout requesting block from upstream")))
.doOnError { t ->
val msg = "Failed to download block data for chain $chain"
if (t is RpcException) {
log.warn("$msg. Message: ${t.message}")
} else {
log.error(msg, t)
}
}
}
.onErrorContinue { err, _ ->
log.error("Head subscription error: ${err.message}")
}
.subscribe { block ->
log.debug("New block ${block.number} on ${chain}")
setStatus(UpstreamAvailability.OK)
headBlock.set(block)
streamBlocks.onNext(block)
}
val base = flux.map { value ->
val block = BlockJson<TransactionId>()
block.number = value.height
block.totalDifficulty = BigInteger(1, value.weight.toByteArray())
block.hash = BlockHash.from("0x"+value.blockId)
block
}.distinctUntilChanged {
it.hash
}.filter { block ->
val curr = head.getCurrent()
curr == null || curr.totalDifficulty < block.totalDifficulty
}.flatMap {
getApi(Selector.EmptyMatcher())
.executeAndConvert(Commands.eth().getBlock(it.hash))
.timeout(Duration.ofSeconds(5), Mono.error(Exception("Timeout requesting block from upstream")))
.doOnError { t ->
val msg = "Failed to download block data for chain $chain"
if (t is RpcException) {
log.warn("$msg. Message: ${t.message}")
} else {
log.error(msg, t)
}
}
}.onErrorContinue { err, _ ->
log.error("Head subscription error: ${err.message}")
}.doOnNext {
setStatus(UpstreamAvailability.OK)
}
headSubscription = head.follow(base)
}
fun init(conf: BlockchainOuterClass.DescribeChain) {
@@ -191,7 +185,7 @@ open class GrpcUpstream(
}
override fun isAvailable(): Boolean {
return getStatus() == UpstreamAvailability.OK && headBlock.get() != null && nodes.get().getNodes().any {
return getStatus() == UpstreamAvailability.OK && head.getCurrent() != null && nodes.get().getNodes().any {
it.quorum > 0
}
}
@@ -208,16 +202,4 @@ open class GrpcUpstream(
return options
}
class Head(
val upstream: GrpcUpstream
): EthereumHead {
override fun getFlux(): Flux<BlockJson<TransactionId>> {
return Flux.merge(
Mono.justOrEmpty(upstream.headBlock.get()),
Flux.from(upstream.streamBlocks)
)
}
}
}