StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added filter for native subscribe

Browse Source
This commit is contained in:
Maxksim Fomenkov
2022-09-12 10:46:21 +03:00
parent 7b00adf9d6
commit 6ec0c7ac27
11 changed files with 70 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt
View File

@@ -20,6 +20,7 @@ import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeMultistream
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
@@ -50,20 +51,17 @@ open class NativeSubscribe(
.onErrorMap(this@NativeSubscribe::convertToStatus)
}
fun start(it: BlockchainOuterClass.NativeSubscribeRequest): Publisher<out Any> {
val chain = Chain.byId(it.chainValue)
fun start(request: BlockchainOuterClass.NativeSubscribeRequest): Publisher<out Any> {
val chain = Chain.byId(request.chainValue)
if (BlockchainType.from(chain) != BlockchainType.ETHEREUM) {
return Mono.error(UnsupportedOperationException("Native subscribe is not supported for ${chain.chainCode}"))
}
val method = it.method
val params: Any? = it.payload?.let { payload ->
if (payload.size() > 0) {
objectMapper.readValue(payload.newInput(), Map::class.java)
} else {
null
}
val method = request.method
val params: Any? = request.payload?.takeIf { !it.isEmpty }?.let {
objectMapper.readValue(it.newInput(), Map::class.java)
}
return subscribe(chain, method, params)
val matcher = Selector.convertToMatcher(request.selector)
return subscribe(chain, method, params, matcher)
}
fun convertToStatus(t: Throwable) = when (t) {
@@ -81,11 +79,11 @@ open class NativeSubscribe(
}
}
open fun subscribe(chain: Chain, method: String, params: Any?): Flux<out Any> {
open fun subscribe(chain: Chain, method: String, params: Any?, matcher: Selector.Matcher = Selector.empty): Flux<out Any> {
val up = multistreamHolder.getUpstream(chain) ?: return Flux.error(SilentException.UnsupportedBlockchain(chain))
return (up as EthereumLikeMultistream)
.getSubscribe()
.subscribe(method, params)
.subscribe(method, params, matcher)
}
fun convertToProto(value: Any): BlockchainOuterClass.NativeSubscribeReplyItem {

4
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeMultistream.kt
View File

@@ -1,8 +1,12 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
interface EthereumLikeMultistream : Upstream {
fun getReader(): EthereumReader
fun getSubscribe(): EthereumSubscribe
fun getHead(mather: Selector.Matcher): Head
}

23
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt
View File

@@ -20,6 +20,7 @@ import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.ChainFees
import io.emeraldpay.dshackle.upstream.EmptyHead
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.MergedHead
import io.emeraldpay.dshackle.upstream.Multistream
@@ -32,6 +33,7 @@ import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.publisher.Mono
import java.util.concurrent.ConcurrentHashMap
@Suppress("UNCHECKED_CAST")
open class EthereumMultistream(
@@ -45,6 +47,7 @@ open class EthereumMultistream(
}
private var head: Head? = null
private val filteredHeads: MutableMap<String, Head> = ConcurrentHashMap()
private val reader: EthereumReader = EthereumReader(this, this.caches, getMethodsFactory())
private val subscribe = EthereumSubscribe(this)
@@ -74,6 +77,7 @@ open class EthereumMultistream(
override fun stop() {
super.stop()
reader.stop()
filteredHeads.clear()
}
override fun isRunning(): Boolean {
@@ -118,6 +122,7 @@ open class EthereumMultistream(
lagObserver.start()
newHead
}
filteredHeads[Selector.AnyLabelMatcher().describeInternal()] = head
onHeadUpdated(head)
return head
}
@@ -142,6 +147,24 @@ open class EthereumMultistream(
return subscribe
}
override fun getHead(mather: Selector.Matcher): Head =
filteredHeads.computeIfAbsent(mather.describeInternal()) { _ ->
upstreams.filter { mather.matches(it) }
.apply {
log.debug("Found $size upstreams matching [${mather.describeInternal()}]")
}
.map { it.getHead() }
.let {
when (it.size) {
0 -> EmptyHead()
1 -> it.first()
else -> MergedHead(it, MostWorkForkChoice()).apply {
start()
}
}
}
}//TODO track unused heads and remove
override fun getFeeEstimation(): ChainFees {
return feeEstimation
}

7
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumSubscribe.kt
View File

@@ -1,5 +1,6 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.ConnectLogs
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.ConnectNewHeads
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.ConnectSyncing
@@ -21,9 +22,9 @@ open class EthereumSubscribe(
private val syncing = ConnectSyncing(upstream)
@Suppress("UNCHECKED_CAST")
open fun subscribe(method: String, params: Any?): Flux<out Any> {
open fun subscribe(method: String, params: Any?, matcher: Selector.Matcher): Flux<out Any> {
if (method == "newHeads") {
return newHeads.connect()
return newHeads.connect(matcher)
}
if (method == "logs") {
val paramsMap = try {
@@ -35,7 +36,7 @@ open class EthereumSubscribe(
} catch (t: Throwable) {
return Flux.error(UnsupportedOperationException("Invalid parameter for $method. Error: ${t.message}"))
}
return logs.start(paramsMap.address, paramsMap.topics)
return logs.start(paramsMap.address, paramsMap.topics, matcher)
}
if (method == "syncing") {
return syncing.connect()

24
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdates.kt
View File

@@ -19,12 +19,14 @@ import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeMultistream
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.scheduler.Schedulers
import java.time.Duration
import java.util.LinkedList
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.locks.ReentrantLock
import java.util.concurrent.locks.ReentrantReadWriteLock
import kotlin.concurrent.read
@@ -46,30 +48,18 @@ class ConnectBlockUpdates(
*/
private val history = LinkedList<BlockContainer>()
private val historyUpdateLock = ReentrantReadWriteLock()
private val connected: MutableMap<String, Flux<Update>> = ConcurrentHashMap()
private var connected: Flux<Update>? = null
private val connectLock = ReentrantLock()
fun connect(): Flux<Update> {
val current = connected
if (current != null) {
return current
}
connectLock.withLock {
val currentRecheck = connected
if (currentRecheck != null) {
return currentRecheck
}
val created = extract(upstream.getHead())
fun connect(matcher: Selector.Matcher): Flux<Update> {
return connected.computeIfAbsent(matcher.describeInternal()) { key ->
extract(upstream.getHead(matcher))
.publishOn(Schedulers.boundedElastic())
.publish()
.refCount(1, Duration.ofSeconds(60))
.doFinally {
// forget it on disconnect, so next time it's recreated
connected = null
connected.remove(key)
}
connected = created
return created
}
}

11
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogs.kt
View File

@@ -15,6 +15,7 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum.subscribe
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeMultistream
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.LogMessage
import io.emeraldpay.etherjar.domain.Address
@@ -40,17 +41,17 @@ open class ConnectLogs(
private val produceLogs = ProduceLogs(upstream)
fun start(): Flux<LogMessage> {
return produceLogs.produce(connectBlockUpdates.connect())
fun start(matcher: Selector.Matcher): Flux<LogMessage> {
return produceLogs.produce(connectBlockUpdates.connect(matcher))
}
open fun start(addresses: List<Address>, topics: List<Hex32>): Flux<LogMessage> {
open fun start(addresses: List<Address>, topics: List<Hex32>, matcher: Selector.Matcher): Flux<LogMessage> {
// shortcut to the whole output if we don't have any filters
if (addresses.isEmpty() && topics.isEmpty()) {
return start()
return start(matcher)
}
// filtered output
return start()
return start(matcher)
.transform(filtered(addresses, topics))
}

27
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectNewHeads.kt
View File

@@ -15,14 +15,14 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum.subscribe
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeMultistream
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.NewHeadMessage
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.scheduler.Schedulers
import java.time.Duration
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
import java.util.concurrent.ConcurrentHashMap
/**
* Connects/reconnects to the upstream to produce NewHeads messages
@@ -35,30 +35,19 @@ class ConnectNewHeads(
private val log = LoggerFactory.getLogger(ConnectNewHeads::class.java)
}
private var connected: Flux<NewHeadMessage>? = null
private val connectLock = ReentrantLock()
private val connected: MutableMap<String, Flux<NewHeadMessage>> = ConcurrentHashMap()
fun connect(): Flux<NewHeadMessage> {
val current = connected
if (current != null) {
return current
}
connectLock.withLock {
val currentRecheck = connected
if (currentRecheck != null) {
return currentRecheck
}
val created = ProduceNewHeads(upstream.getHead())
fun connect(matcher: Selector.Matcher): Flux<NewHeadMessage> =
connected.computeIfAbsent(matcher.describeInternal()) { key ->
ProduceNewHeads(upstream.getHead(matcher))
.start()
.publishOn(Schedulers.boundedElastic())
.publish()
.refCount(1, Duration.ofSeconds(60))
.doFinally {
// forget it on disconnect, so next time it's recreated
connected = null
connected.remove(key)
}
connected = created
return created
}
}
}

3
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosMultiStream.kt
View File

@@ -137,6 +137,9 @@ open class EthereumPosMultiStream(
return subscribe
}
override fun getHead(mather: Selector.Matcher): Head =
getHead() //TODO
override fun getFeeEstimation(): ChainFees {
return feeEstimation
}