problem: doesn't update head subscription through remote dshackle, because upstream becomes ready later after chain setup

solution: fix and refactor to avoid such situation in code
This commit is contained in:
Igor Artamonov
2019-08-11 20:22:37 -04:00
parent 29cea2575e
commit 89bd52a267
30 changed files with 356 additions and 259 deletions

View File

@@ -5,7 +5,6 @@ import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.grpc.Chain
import io.infinitape.etherjar.rpc.RpcException
import org.apache.commons.lang3.StringUtils
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
@@ -77,7 +76,7 @@ class NativeCall(
return prepareCall(request, upstream)
}
fun prepareCall(request: BlockchainOuterClass.NativeCallRequest, upstream: AggregatedUpstreams): Flux<CallContext<Tuple2<String, String>>> {
fun prepareCall(request: BlockchainOuterClass.NativeCallRequest, upstream: AggregatedUpstream): Flux<CallContext<Tuple2<String, String>>> {
val matcher = Selector.convertToMatcher(request.selector)
val apis = upstream.getApis(matcher)
return request.itemsList.toFlux().map {
@@ -109,6 +108,7 @@ class NativeCall(
ctx.withPayload(result)
}
.onErrorMap {
log.error("Failed to make a call", it)
if (it is CallFailure) it
else CallFailure(ctx.id, it)
}

View File

@@ -3,8 +3,6 @@ package io.emeraldpay.dshackle.rpc
import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.upstream.AvailableChains
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.UpstreamServices
import io.emeraldpay.dshackle.upstream.Upstreams
import io.emeraldpay.grpc.Chain
@@ -17,16 +15,13 @@ import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.core.publisher.TopicProcessor
import reactor.core.publisher.toFlux
import java.lang.Exception
import java.time.Duration
import java.util.concurrent.ConcurrentLinkedQueue
import javax.annotation.PostConstruct
import kotlin.collections.HashMap
@Service
class StreamHead(
@Autowired private val upstreams: Upstreams,
@Autowired private val availableChains: AvailableChains
@Autowired private val upstreams: Upstreams
) {
private val log = LoggerFactory.getLogger(StreamHead::class.java)
@@ -34,14 +29,17 @@ class StreamHead(
@PostConstruct
fun init() {
availableChains.observe().subscribe { chain ->
upstreams.observeChains().subscribe { chain ->
if (clients.containsKey(chain)) {
return@subscribe
}
clients[chain] = ConcurrentLinkedQueue()
subscribe(chain)
}
}
private fun subscribe(chain: Chain) {
upstreams.getUpstream(chain)?.let { up ->
upstreams.getUpstream(chain)!!.let { up ->
up.getHead()
.getFlux()
.doOnComplete {

View File

@@ -11,13 +11,12 @@ import reactor.core.publisher.Mono
@Service
class SubscribeStatus(
@Autowired private val upstreams: Upstreams,
@Autowired private val availableChains: AvailableChains
@Autowired private val upstreams: Upstreams
) {
fun subscribeStatus(requestMono: Mono<BlockchainOuterClass.StatusRequest>): Flux<BlockchainOuterClass.ChainStatus> {
return requestMono.flatMapMany {
val ups = availableChains.getAll().mapNotNull { chain ->
val ups = upstreams.getAvailable().mapNotNull { chain ->
val chainUpstream = upstreams.getUpstream(chain)
chainUpstream?.observeStatus()?.map { avail ->
ChainSubscription(chain, chainUpstream, avail)
@@ -45,6 +44,6 @@ class SubscribeStatus(
.build()
}
class ChainSubscription(val chain: Chain, val up: AggregatedUpstreams, val avail: UpstreamAvailability)
class ChainSubscription(val chain: Chain, val up: AggregatedUpstream, val avail: UpstreamAvailability)
}

View File

@@ -2,7 +2,6 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.upstream.AvailableChains
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstreams
import io.emeraldpay.grpc.Chain
@@ -30,7 +29,6 @@ import javax.annotation.PostConstruct
@Service
class TrackAddress(
@Autowired private val upstreams: Upstreams,
@Autowired private val availableChains: AvailableChains,
@Autowired private val upstreamScheduler: Scheduler
) {
@@ -40,7 +38,7 @@ class TrackAddress(
@PostConstruct
fun init() {
availableChains.observe().subscribe { chain ->
upstreams.observeChains().subscribe { chain ->
if (!clients.containsKey(chain)) {
clients[chain] = ConcurrentLinkedQueue()
upstreams.getUpstream(chain)?.getHead()?.let { head ->
@@ -53,7 +51,7 @@ class TrackAddress(
@Scheduled(fixedDelay = 120_000)
fun pingOld() {
val period = Duration.ofMinutes(15)
availableChains.getAll().forEach { chain ->
upstreams.getAvailable().forEach { chain ->
clients[chain]?.let { clients ->
clients.toFlux().filter {
it.lastPing < Instant.now().minus(period)
@@ -80,7 +78,7 @@ class TrackAddress(
private fun initializeSimple(request: BlockchainOuterClass.BalanceRequest): Flux<SimpleAddress> {
val chain = Chain.byId(request.asset.chainValue)
if (!availableChains.supports(chain)) {
if (!upstreams.isAvailable(chain)) {
return Flux.error(Exception("Unsupported chain ${request.asset.chainValue}"))
}
if (request.asset.code?.toLowerCase() != "ether") {

View File

@@ -3,7 +3,6 @@ package io.emeraldpay.dshackle.rpc
import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.upstream.AvailableChains
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.Upstreams
@@ -38,7 +37,6 @@ import kotlin.math.min
@Service
class TrackTx(
@Autowired private val upstreams: Upstreams,
@Autowired private val availableChains: AvailableChains,
@Autowired private val upstreamScheduler: Scheduler
) {
@@ -59,7 +57,7 @@ class TrackTx(
@PostConstruct
fun init() {
availableChains.observe().subscribe { chain ->
upstreams.observeChains().subscribe { chain ->
clients[chain] = ConcurrentLinkedQueue()
upstreams.getUpstream(chain)?.getHead()?.let { head ->
head.getFlux().subscribe { verifyAll(chain) }
@@ -239,6 +237,9 @@ class TrackTx(
.executeAndConvert(Commands.eth().getTransaction(tx.txid))
return execution
.flatMap { updateFromBlock(upstream, tx, it) }
.doOnError { t ->
log.error("Failed to load tx block", t)
}
.switchIfEmpty(Mono.just(tx.withStatus(found = false)))
.filter { current ->
initialStatus != current.status || current.shouldNotify() || current.shouldClose()

View File

@@ -7,8 +7,8 @@ import java.time.Instant
import java.util.concurrent.atomic.AtomicReference
import java.util.function.Predicate
abstract class AggregatedUpstreams(
val targets: EthereumTargets
abstract class AggregatedUpstream(
val targets: CallMethods
): Upstream {
abstract fun getAll(): List<Upstream>

View File

@@ -1,52 +0,0 @@
package io.emeraldpay.dshackle.upstream
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.grpc.Chain
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Repository
import reactor.core.publisher.Flux
import reactor.core.publisher.TopicProcessor
import java.util.*
import kotlin.collections.LinkedHashSet
@Repository
class AvailableChains(
@Autowired private val objectMapper: ObjectMapper
) {
private val all = LinkedHashSet<Chain>()
private val bus = TopicProcessor.create<Chain>()
private val callTargets = HashMap<Chain, EthereumTargets>()
fun add(chain: Chain) {
if (all.contains(chain)) {
return
}
all.add(chain)
bus.onNext(chain)
}
fun observe(): Flux<Chain> {
return Flux.merge(
Flux.fromIterable(all),
Flux.from(bus)
)
}
fun supports(chain: Chain): Boolean {
return all.contains(chain)
}
fun getAll(): Set<Chain> {
return Collections.unmodifiableSet(all)
}
fun targetFor(chain: Chain): EthereumTargets {
var current = callTargets[chain]
if (current == null) {
current = EthereumTargets(objectMapper, chain)
callTargets[chain] = current
}
return current
}
}

View File

@@ -0,0 +1,9 @@
package io.emeraldpay.dshackle.upstream
interface CallMethods {
fun getQuorumFor(method: String): CallQuorum
fun isAllowed(method: String): Boolean
fun getSupportedMethods(): Set<String>
fun isHardcoded(method: String): Boolean
fun hardcoded(method: String): Any
}

View File

@@ -9,8 +9,8 @@ import java.time.Duration
class ChainUpstreams (
val chain: Chain,
private val upstreams: MutableList<Upstream>,
targets: EthereumTargets
) : AggregatedUpstreams(targets) {
targets: CallMethods
) : AggregatedUpstream(targets) {
private val log = LoggerFactory.getLogger(ChainUpstreams::class.java)
private var seq = 0

View File

@@ -12,6 +12,8 @@ import org.springframework.beans.factory.annotation.Autowired
import org.springframework.core.env.Environment
import org.springframework.scheduling.annotation.Scheduled
import org.springframework.stereotype.Repository
import reactor.core.publisher.Flux
import reactor.core.publisher.TopicProcessor
import reactor.core.publisher.toFlux
import java.io.File
import java.net.URI
@@ -23,12 +25,13 @@ import kotlin.collections.HashMap
@Repository
open class ConfiguredUpstreams(
@Autowired val env: Environment,
@Autowired private val objectMapper: ObjectMapper,
@Autowired private val availableChains: AvailableChains
@Autowired private val objectMapper: ObjectMapper
) : Upstreams {
private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java)
private val chainMapping = ConcurrentHashMap<Chain, ChainUpstreams>()
private val chainsBus = TopicProcessor.create<Chain>()
private val callTargets = HashMap<Chain, QuorumBasedMethods>()
private val chainNames = mapOf(
"ethereum" to Chain.ETHEREUM,
@@ -108,7 +111,7 @@ open class ConfiguredUpstreams(
rpcClient,
objectMapper,
chain,
availableChains.targetFor(chain)
targetFor(chain)
)
urls.add(endpoint.url)
}
@@ -122,12 +125,7 @@ open class ConfiguredUpstreams(
}
if (rpcApi != null) {
log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}")
getOrCreateUpstream(chain)
.addUpstream(
EthereumUpstream(
chain, rpcApi!!, wsApi, options, NodeDetailsList.NodeDetails(1, labels), availableChains.targetFor(chain)
)
)
addUpstream(chain, EthereumUpstream(chain, rpcApi!!, wsApi, options, NodeDetailsList.NodeDetails(1, labels), targetFor(chain)))
}
}
@@ -139,7 +137,7 @@ open class ConfiguredUpstreams(
objectMapper,
options,
up.auth,
availableChains
this
)
log.info("Using ALL CHAINS (gRPC) upstream, at ${endpoint.host}:${endpoint.port}")
ds.start()
@@ -148,21 +146,24 @@ open class ConfiguredUpstreams(
}
.subscribe {
log.info("Subscribed to $it through gRPC at ${endpoint.host}:${endpoint.port}")
getOrCreateUpstream(it).addUpstream(ds.getOrCreate(it))
addUpstream(it, ds.getOrCreate(it))
}
}
override fun getUpstream(chain: Chain): AggregatedUpstreams? {
override fun getUpstream(chain: Chain): AggregatedUpstream? {
return chainMapping[chain]
}
override fun getOrCreateUpstream(chain: Chain): ChainUpstreams {
override fun addUpstream(chain: Chain, up: Upstream): ChainUpstreams {
val current = chainMapping[chain]
if (current == null) {
availableChains.add(chain)
val created = ChainUpstreams(chain, ArrayList<Upstream>(), availableChains.targetFor(chain))
val created = ChainUpstreams(chain, ArrayList<Upstream>(), targetFor(chain))
created.addUpstream(up)
chainMapping[chain] = created
chainsBus.onNext(chain)
return created
} else {
current.addUpstream(up)
}
return current
}
@@ -175,4 +176,24 @@ open class ConfiguredUpstreams(
override fun getAvailable(): List<Chain> {
return Collections.unmodifiableList(chainMapping.keys.toList())
}
override fun observeChains(): Flux<Chain> {
return Flux.merge(
Flux.fromIterable(getAvailable()),
Flux.from(chainsBus)
)
}
override fun targetFor(chain: Chain): CallMethods {
var current = callTargets[chain]
if (current == null) {
current = QuorumBasedMethods(objectMapper, chain)
callTargets[chain] = current
}
return current
}
override fun isAvailable(chain: Chain): Boolean {
return chainMapping.containsKey(chain) && callTargets.containsKey(chain)
}
}

View File

@@ -5,13 +5,13 @@ import reactor.core.publisher.TopicProcessor
import java.util.concurrent.atomic.AtomicReference
abstract class DefaultUpstream(
lag: Long,
avail: UpstreamAvailability
defaultLag: Long,
defaultAvail: UpstreamAvailability
) : Upstream {
constructor() : this(Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE)
private val status = AtomicReference(Status(lag, avail, statusByLag(lag, avail)))
private val status = AtomicReference(Status(defaultLag, defaultAvail, statusByLag(defaultLag, defaultAvail)))
private val statusStream: TopicProcessor<UpstreamAvailability> = TopicProcessor.create()
override fun getStatus(): UpstreamAvailability {

View File

@@ -0,0 +1,24 @@
package io.emeraldpay.dshackle.upstream
class DirectCallMethods : CallMethods {
override fun getQuorumFor(method: String): CallQuorum {
return AlwaysQuorum()
}
override fun isAllowed(method: String): Boolean {
return true
}
override fun getSupportedMethods(): Set<String> {
return emptySet()
}
override fun isHardcoded(method: String): Boolean {
return false
}
override fun hardcoded(method: String): Any {
return "unsupported"
}
}

View File

@@ -13,11 +13,11 @@ open class EthereumApi(
val rpcClient: RpcClient,
private val objectMapper: ObjectMapper,
private val chain: Chain,
val targets: EthereumTargets,
var upstream: Upstream? = null
val targets: CallMethods
) {
private val jacksonRpcConverter = JacksonRpcConverter(objectMapper)
var upstream: Upstream? = null
private val timeout = Duration.ofSeconds(5)
private val log = LoggerFactory.getLogger(EthereumApi::class.java)

View File

@@ -2,13 +2,7 @@ package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.grpc.Chain
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.publisher.TopicProcessor
import java.util.concurrent.atomic.AtomicLong
import java.util.concurrent.atomic.AtomicReference
open class EthereumUpstream(
val chain: Chain,
@@ -16,22 +10,20 @@ open class EthereumUpstream(
private val ethereumWs: EthereumWs? = null,
private val options: UpstreamsConfig.Options,
val node: NodeDetailsList.NodeDetails,
private val targets: EthereumTargets
private val targets: CallMethods
): DefaultUpstream() {
constructor(chain: Chain, api: EthereumApi): this(chain, api, null,
UpstreamsConfig.Options.getDefaults(), NodeDetailsList.NodeDetails(1, UpstreamsConfig.Labels()),
DirectCallMethods())
override fun getSupportedTargets(): Set<String> {
return targets.getSupportedMethods()
}
private val log = LoggerFactory.getLogger(EthereumUpstream::class.java)
private val head: EthereumHead = if (ethereumWs != null) {
EthereumWsHead(ethereumWs)
} else {
EthereumRpcHead(api).apply {
this.start()
}
}
private val head: EthereumHead = createHead()
private val validator = UpstreamValidator(this, options)
@@ -43,6 +35,16 @@ open class EthereumUpstream(
.subscribe(this::setStatus)
}
open fun createHead(): EthereumHead {
return if (ethereumWs != null) {
EthereumWsHead(ethereumWs)
} else {
EthereumRpcHead(api).apply {
this.start()
}
}
}
override fun isAvailable(matcher: Selector.Matcher): Boolean {
return getStatus() == UpstreamAvailability.OK && matcher.matches(node.labels)
}

View File

@@ -1,30 +1,30 @@
package io.emeraldpay.dshackle.upstream
class FilteringApiIterator(
private val apis: List<Upstream>,
private val upstreams: List<Upstream>,
private var pos: Int,
private val matcher: Selector.Matcher,
private val repeatLimit: Int = 3
): Iterator<EthereumApi> {
private var nextApi: Upstream? = null
private var nextUpstream: Upstream? = null
private var consumed = 0
private fun nextInternal(): Boolean {
if (nextApi != null) {
if (nextUpstream != null) {
return true
}
while (nextApi == null) {
while (nextUpstream == null) {
consumed++
if (consumed > apis.size * repeatLimit) {
if (consumed > upstreams.size * repeatLimit) {
return false
}
val api = apis[pos++ % apis.size]
if (api.isAvailable(matcher)) {
nextApi = api
val upstream = upstreams[pos++ % upstreams.size]
if (upstream.isAvailable(matcher)) {
nextUpstream = upstream
}
}
return nextApi != null
return nextUpstream != null
}
override fun hasNext(): Boolean {
@@ -33,8 +33,8 @@ class FilteringApiIterator(
override fun next(): EthereumApi {
if (nextInternal()) {
val curr = nextApi!!
nextApi = null
val curr = nextUpstream!!
nextUpstream = null
return curr.getApi(matcher)
}
throw IllegalStateException("No upstream API available")

View File

@@ -11,7 +11,6 @@ import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.*
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.BlockTag
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
@@ -29,10 +28,10 @@ open class GrpcUpstream(
private val client: ReactorBlockchainGrpc.ReactorBlockchainStub,
private val objectMapper: ObjectMapper,
private val options: UpstreamsConfig.Options,
private val targets: EthereumTargets
private val targets: CallMethods
): DefaultUpstream() {
constructor(chain: Chain, client: ReactorBlockchainGrpc.ReactorBlockchainStub, objectMapper: ObjectMapper, targets: EthereumTargets)
constructor(chain: Chain, client: ReactorBlockchainGrpc.ReactorBlockchainStub, objectMapper: ObjectMapper, targets: CallMethods)
: this(chain, client, objectMapper, UpstreamsConfig.Options.getDefaults(), targets)
private val log = LoggerFactory.getLogger(GrpcUpstream::class.java)
@@ -48,7 +47,10 @@ open class GrpcUpstream(
open fun createApi(matcher: Selector.Matcher): EthereumApi {
val rpcClient = DefaultRpcClient(grpcTransport.withMatcher(matcher))
return EthereumApi(rpcClient, objectMapper, chain, targets, this)
return EthereumApi(rpcClient, objectMapper, chain, targets).let {
it.upstream = this
it
}
}
open fun connect() {

View File

@@ -22,7 +22,7 @@ class GrpcUpstreams(
private val objectMapper: ObjectMapper,
private val options: UpstreamsConfig.Options,
private val auth: UpstreamsConfig.TlsAuth? = null,
private val availableChains: AvailableChains
private val upstreams: Upstreams
) {
private val log = LoggerFactory.getLogger(GrpcUpstreams::class.java)
@@ -92,9 +92,9 @@ class GrpcUpstreams(
lock.withLock {
val current = known[chain]
return if (current == null) {
val created = GrpcUpstream(chain, client!!, objectMapper, options, availableChains.targetFor(chain))
val created = GrpcUpstream(chain, client!!, objectMapper, options, upstreams.targetFor(chain))
known[chain] = created
availableChains.add(chain)
upstreams.addUpstream(chain, created)
created.connect()
created
} else {

View File

@@ -6,10 +6,10 @@ import io.infinitape.etherjar.rpc.JacksonRpcConverter
import io.infinitape.etherjar.rpc.RpcException
import java.util.*
class EthereumTargets(
class QuorumBasedMethods(
private val objectMapper: ObjectMapper,
private val chain: Chain
) {
) : CallMethods {
private val jacksonRpcConverter = JacksonRpcConverter(objectMapper)
@@ -61,7 +61,7 @@ class EthereumTargets(
"eth_accounts"
)
open fun getQuorumFor(method: String): CallQuorum {
override fun getQuorumFor(method: String): CallQuorum {
return when {
hardcodedMethods.contains(method) -> AlwaysQuorum()
anyResponseMethods.contains(method) -> NotLaggingQuorum(6)
@@ -78,14 +78,14 @@ class EthereumTargets(
}
}
fun isAllowed(method: String): Boolean {
override fun isAllowed(method: String): Boolean {
return allowedMethods.contains(method)
}
fun isHardcoded(method: String): Boolean {
override fun isHardcoded(method: String): Boolean {
return hardcodedMethods.contains(method)
}
fun hardcoded(method: String): Any {
override fun hardcoded(method: String): Any {
if ("net_version" == method) {
if (Chain.ETHEREUM == chain) {
return "1"
@@ -131,7 +131,7 @@ class EthereumTargets(
throw RpcException(-32601, "Method not found")
}
fun getSupportedMethods(): Set<String> {
override fun getSupportedMethods(): Set<String> {
return allowedMethods.plus(hardcodedMethods).toSortedSet()
}
}

View File

@@ -4,7 +4,10 @@ import io.emeraldpay.grpc.Chain
import reactor.core.publisher.Flux
interface Upstreams {
fun getOrCreateUpstream(chain: Chain): AggregatedUpstreams
fun getUpstream(chain: Chain): AggregatedUpstreams?
fun addUpstream(chain: Chain, up: Upstream): AggregatedUpstream
fun getUpstream(chain: Chain): AggregatedUpstream?
fun getAvailable(): List<Chain>
fun observeChains(): Flux<Chain>
fun targetFor(chain: Chain): CallMethods
fun isAvailable(chain: Chain): Boolean
}