StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

solution: subscribe to "logs" events

Browse Source
This commit is contained in:
Igor Artamonov
2021-10-17 20:41:45 -04:00
parent e37076936f
commit 06811060cc
21 changed files with 1552 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/kotlin/io/emeraldpay/dshackle/cache/ReceiptRedisCache.kt vendored
View File

@@ -23,7 +23,7 @@ import io.emeraldpay.etherjar.rpc.json.TransactionReceiptJson
import io.lettuce.core.api.reactive.RedisReactiveCommands
import reactor.core.publisher.Mono
class ReceiptRedisCache(
open class ReceiptRedisCache(
redis: RedisReactiveCommands<String, ByteArray>,
chain: Chain
) : OnTxRedisCache<ByteArray>(redis, chain, CachesProto.ValueContainer.ValueType.TX_RECEIPT) {

55
src/main/kotlin/io/emeraldpay/dshackle/reader/RpcReader.kt Normal file
View File

@@ -0,0 +1,55 @@
/**
* Copyright (c) 2021 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.reader
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono
/**
* Reader that requests data through upstream RPC using provided JSON RPC request builder
*/
class RpcReader<T>(
private val up: Multistream,
private val paramsBuilder: (T) -> JsonRpcRequest
) : Reader<T, ByteArray> {
companion object {
private val log = LoggerFactory.getLogger(RpcReader::class.java)
/**
* Common reader that just passes key as a parameter with the specified method. The key must be serializable to JSON.
* @param method RPC method to use
*/
fun <T> basicRequest(up: Multistream, method: String): RpcReader<T> {
return RpcReader(up) { key ->
JsonRpcRequest(method, listOf(key))
}
}
}
override fun read(key: T): Mono<ByteArray> {
return up.getDirectApi(Selector.empty)
.flatMap { rdr ->
rdr.read(paramsBuilder(key)).flatMap {
it.requireResult()
}
}
}
}

10
src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt
View File

@@ -56,13 +56,13 @@ class NativeSubscribe(
return Mono.error(UnsupportedOperationException("Native subscribe is not supported for ${chain.chainCode}"))
}
val method = it.method
val params: List<*> = it.payload?.let { payload ->
val params: Any? = it.payload?.let { payload ->
if (payload.size() > 0) {
listOf(objectMapper.readValue(payload.newInput(), Map::class.java))
objectMapper.readValue(payload.newInput(), Map::class.java)
} else {
emptyList<Any>()
null
}
} ?: emptyList<Any>()
}
return subscribe(chain, method, params)
}
@@ -81,7 +81,7 @@ class NativeSubscribe(
}
}
fun subscribe(chain: Chain, method: String, params: List<*>): Flux<out Any> {
fun subscribe(chain: Chain, method: String, params: Any?): Flux<out Any> {
val up = multistreamHolder.getUpstream(chain) ?: return Flux.error(SilentException.UnsupportedBlockchain(chain))
return (up as EthereumMultistream)
.getSubscribe()

11
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumReader.kt
View File

@@ -23,7 +23,9 @@ import io.emeraldpay.dshackle.cache.HeightByHashAdding
import io.emeraldpay.dshackle.data.*
import io.emeraldpay.dshackle.reader.*
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.etherjar.domain.Address
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.domain.TransactionId
@@ -142,7 +144,14 @@ open class EthereumReader(
}
fun receipts(): Reader<TxId, ByteArray> {
return caches.getReceipts()
//TODO put into cache
val requested = RekeyingReader(
{ txid: TxId -> txid.toHexWithPrefix() },
RpcReader.basicRequest(up, "eth_getTransactionReceipt"))
return CompoundReader(
caches.getReceipts(),
requested
)
}
fun heightByHash(): Reader<BlockId, Long> {

71
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumSubscribe.kt
View File

@@ -1,6 +1,10 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.ConnectLogs
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.ConnectNewHeads
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.ProduceLogs
import io.emeraldpay.etherjar.domain.Address
import io.emeraldpay.etherjar.hex.Hex32
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
@@ -13,11 +17,76 @@ open class EthereumSubscribe(
}
private val newHeads = ConnectNewHeads(upstream)
private val logs = ConnectLogs(upstream)
open fun subscribe(method: String, params: List<*>): Flux<out Any> {
@Suppress("UNCHECKED_CAST")
open fun subscribe(method: String, params: Any?): Flux<out Any> {
if (method == "newHeads") {
return newHeads.connect()
}
if (method == "logs") {
val paramsMap = try {
if (params != null && Map::class.java.isAssignableFrom(params.javaClass)) {
readLogsRequest(params as Map<String, Any?>)
} else {
LogsRequest(emptyList(), emptyList())
}
} catch (t: Throwable) {
return Flux.error(UnsupportedOperationException("Invalid parameter for $method. Error: ${t.message}"))
}
return logs.start(paramsMap.address, paramsMap.topics)
}
return Flux.error(UnsupportedOperationException("Method $method is not supported"))
}
data class LogsRequest(
val address: List<Address>,
val topics: List<Hex32>
)
fun readLogsRequest(params: Map<String, Any?>): LogsRequest {
val addresses: List<Address> = if (params.containsKey("address")) {
when (val address = params["address"]) {
is String -> try {
listOf(Address.from(address))
} catch (t: Throwable) {
log.debug("Ignore invalid address: $address with error ${t.message}")
emptyList()
}
is Collection<*> -> address.mapNotNull {
try {
Address.from(it.toString())
} catch (t: Throwable) {
log.debug("Ignore invalid address: $address with error ${t.message}")
null
}
}
else -> throw IllegalArgumentException("Invalid type of address field. Must be string or list of strings")
}
} else {
emptyList()
}
val topics: List<Hex32> = if (params.containsKey("topics")) {
when (val topics = params["topics"]) {
is String -> try {
listOf(Hex32.from(topics))
} catch (t: Throwable) {
log.debug("Ignore invalid topic: $topics with error ${t.message}")
emptyList()
}
is Collection<*> -> topics.mapNotNull {
try {
Hex32.from(it.toString())
} catch (t: Throwable) {
log.debug("Ignore invalid topic: $topics with error ${t.message}")
null
}
}
else -> throw IllegalArgumentException("Invalid type of topics field. Must be string or list of strings")
}
} else {
emptyList()
}
return LogsRequest(addresses, topics)
}
}

151
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdates.kt Normal file
View File

@@ -0,0 +1,151 @@
/**
* Copyright (c) 2021 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.upstream.ethereum.subscribe
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.scheduler.Schedulers
import java.time.Duration
import java.util.*
import java.util.concurrent.locks.ReentrantLock
import java.util.concurrent.locks.ReentrantReadWriteLock
import kotlin.concurrent.read
import kotlin.concurrent.withLock
import kotlin.concurrent.write
class ConnectBlockUpdates(
private val upstream: EthereumMultistream
) {
companion object {
private val log = LoggerFactory.getLogger(ConnectBlockUpdates::class.java)
private const val HISTORY_LIMIT = 6 * 3
}
/**
* Need to keep history of few last blocks in case we have got a conflicting blocks on the same height.
* In this case it produces a list of updates for transactions that are missing from the new version of the block.
*/
private val history = LinkedList<BlockContainer>()
private val historyUpdateLock = ReentrantReadWriteLock()
private var connected: Flux<Update>? = null
private val connectLock = ReentrantLock()
fun connect(): Flux<Update> {
val current = connected
if (current != null) {
return current
}
connectLock.withLock {
val currentRecheck = connected
if (currentRecheck != null) {
return currentRecheck
}
val created = extract(upstream.getHead())
.publishOn(Schedulers.boundedElastic())
.publish()
.refCount(1, Duration.ofSeconds(60))
.doFinally {
//forget it on disconnect, so next time it's recreated
connected = null
}
connected = created
return created
}
}
fun extract(head: Head): Flux<Update> {
return head.getFlux()
.flatMap(this@ConnectBlockUpdates::extract)
}
fun extract(block: BlockContainer): Flux<Update> {
val prev = findPrevious(block)
remember(block)
val removed = if (prev != null) {
whenReplaced(prev)
} else {
Flux.empty()
}
val added = extractUpdates(block)
return Flux.concat(removed, added)
}
fun findPrevious(block: BlockContainer): BlockContainer? {
historyUpdateLock.read {
val existing = history.find { it.height == block.height }
if (existing != null) {
historyUpdateLock.write {
history.removeIf { it.hash == existing.hash }
}
}
return existing
}
}
fun remember(block: BlockContainer) {
historyUpdateLock.write {
history.add(block)
if (history.size > HISTORY_LIMIT) {
history.removeFirst()
}
}
}
/**
* Produce updates for transactions when a block is replaces with a different one on the same height.
*/
fun whenReplaced(prev: BlockContainer): Flux<Update> {
return Flux.fromIterable(prev.transactions).map {
Update(
prev.hash,
prev.height,
UpdateType.DROP,
it
)
}
}
fun extractUpdates(block: BlockContainer): Flux<Update> {
return Flux.fromIterable(block.transactions)
.map {
Update(
block.hash,
block.height,
UpdateType.NEW,
it
)
}
}
data class Update(
val blockHash: BlockId,
val blockNumber: Long,
val type: UpdateType,
val transactionId: TxId,
)
enum class UpdateType {
NEW,
DROP
}
}

69
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogs.kt Normal file
View File

@@ -0,0 +1,69 @@
/**
* Copyright (c) 2021 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.upstream.ethereum.subscribe
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.LogMessage
import io.emeraldpay.etherjar.domain.Address
import io.emeraldpay.etherjar.hex.Hex32
import io.emeraldpay.etherjar.hex.HexDataComparator
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import java.util.function.Function
class ConnectLogs(
upstream: EthereumMultistream,
private val connectBlockUpdates: ConnectBlockUpdates,
) {
companion object {
private val log = LoggerFactory.getLogger(ConnectLogs::class.java)
private val ADDR_COMPARATOR = HexDataComparator()
private val TOPIC_COMPARATOR = HexDataComparator()
}
constructor(upstream: EthereumMultistream) : this(upstream, ConnectBlockUpdates(upstream))
private val produceLogs = ProduceLogs(upstream)
fun start(): Flux<LogMessage> {
return produceLogs.produce(connectBlockUpdates.connect())
}
fun start(addresses: List<Address>, topics: List<Hex32>): Flux<LogMessage> {
// shortcut to the whole output if we don't have any filters
if (addresses.isEmpty() && topics.isEmpty()) {
return start()
}
// filtered output
return start()
.transform(filtered(addresses, topics))
}
fun filtered(addresses: List<Address>, topics: List<Hex32>): Function<Flux<LogMessage>, Flux<LogMessage>> {
//sort search criteria to use binary search later
val sortedAddresses: List<Address> = addresses.sortedWith(ADDR_COMPARATOR)
val sortedTopics: List<Hex32> = topics.sortedWith(TOPIC_COMPARATOR)
return Function { logs ->
logs.filter {
val goodAddress = sortedAddresses.isEmpty() || sortedAddresses.binarySearch(it.address, ADDR_COMPARATOR) >= 0
val goodTopic = sortedTopics.isEmpty() || (it.topics.isNotEmpty() && sortedTopics.binarySearch(it.topics[0], TOPIC_COMPARATOR) >= 0)
goodAddress && goodTopic
}
}
}
}

6
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectNewHeads.kt
View File

@@ -16,7 +16,7 @@
package io.emeraldpay.dshackle.upstream.ethereum.subscribe
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.NewHead
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.NewHeadMessage
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.scheduler.Schedulers
@@ -35,10 +35,10 @@ class ConnectNewHeads(
private val log = LoggerFactory.getLogger(ConnectNewHeads::class.java)
}
private var connected: Flux<NewHead>? = null
private var connected: Flux<NewHeadMessage>? = null
private val connectLock = ReentrantLock()
fun connect(): Flux<NewHead> {
fun connect(): Flux<NewHeadMessage> {
val current = connected
if (current != null) {
return current

107
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceLogs.kt Normal file
View File

@@ -0,0 +1,107 @@
/**
* Copyright (c) 2021 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.upstream.ethereum.subscribe
import com.google.common.cache.CacheBuilder
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.LogMessage
import io.emeraldpay.etherjar.hex.HexData
import io.emeraldpay.etherjar.rpc.json.TransactionReceiptJson
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.kotlin.core.publisher.switchIfEmpty
import java.util.concurrent.TimeUnit
class ProduceLogs(
private val receipts: Reader<TxId, ByteArray>
) {
companion object {
private val log = LoggerFactory.getLogger(ProduceLogs::class.java)
}
constructor(upstream: EthereumMultistream) : this(upstream.getReader().receipts())
private val objectMapper = Global.objectMapper
// need to keep history of recent messages in case they get removed. cannot rely on
// any other cache or upstream because if when it gets removed it's unavailable in any other source
private val oldMessages = CacheBuilder.newBuilder()
.expireAfterWrite(5, TimeUnit.HOURS)
.build<LogReference, List<LogMessage>>()
fun produce(block: Flux<ConnectBlockUpdates.Update>): Flux<LogMessage> {
return block.flatMap { update ->
if (update.type == ConnectBlockUpdates.UpdateType.DROP) {
produceRemoved(update)
} else {
produceAdded(update)
}
}
}
fun produceRemoved(update: ConnectBlockUpdates.Update): Flux<LogMessage> {
val old = oldMessages.getIfPresent(LogReference(update.blockHash, update.transactionId))
if (old == null) {
log.warn("No old message to produce removal messages for tx ${update.transactionId} at block ${update.blockHash}")
return Flux.empty()
}
return Flux.fromIterable(old)
.map { it.copy(removed = true) }
}
fun produceAdded(update: ConnectBlockUpdates.Update): Flux<LogMessage> {
return receipts.read(update.transactionId)
.switchIfEmpty {
log.warn("Cannot find receipt for tx ${update.transactionId}")
Mono.empty()
}
.map { objectMapper.readValue(it, TransactionReceiptJson::class.java) }
.flatMapMany { receipt ->
try {
val messages = receipt.logs
.map { txlog ->
LogMessage(
txlog.address,
txlog.blockHash,
txlog.blockNumber,
txlog.data ?: HexData.empty(),
txlog.logIndex,
txlog.topics,
txlog.transactionHash,
txlog.transactionIndex,
false
)
}
oldMessages.put(LogReference(update.blockHash, update.transactionId), messages)
Flux.fromIterable(messages)
} catch (t: Throwable) {
log.warn("Invalid Receipt ${update.transactionId}. ${t.javaClass}: ${t.message}")
Flux.empty<LogMessage>()
}
}
}
private data class LogReference(
val block: BlockId,
val tx: TxId
)
}

9
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceNewHeads.kt
View File

@@ -17,7 +17,7 @@ package io.emeraldpay.dshackle.upstream.ethereum.subscribe
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.NewHead
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.NewHeadMessage
import io.emeraldpay.etherjar.rpc.json.BlockJson
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
import org.slf4j.LoggerFactory
@@ -26,7 +26,7 @@ import reactor.core.publisher.Flux
/**
* Produces NewHead messages by transforming blocks received from Head
* @see Head
* @see NewHead
* @see NewHeadMessage
*/
class ProduceNewHeads(
val head: Head
@@ -38,7 +38,7 @@ class ProduceNewHeads(
private val objectMapper = Global.objectMapper
fun start(): Flux<NewHead> {
fun start(): Flux<NewHeadMessage> {
return head.getFlux()
.map {
if (it.parsed != null) {
@@ -48,7 +48,7 @@ class ProduceNewHeads(
}
}
.map { block ->
NewHead(
NewHeadMessage(
block.number,
block.hash,
block.parentHash,
@@ -61,7 +61,6 @@ class ProduceNewHeads(
block.baseFeePerGas?.amount
)
}
}
}

44
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/LogMessage.kt Normal file
View File

@@ -0,0 +1,44 @@
/**
* Copyright (c) 2021 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.upstream.ethereum.subscribe.json
import com.fasterxml.jackson.databind.annotation.JsonSerialize
import io.emeraldpay.etherjar.domain.Address
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.domain.TransactionId
import io.emeraldpay.etherjar.hex.Hex32
import io.emeraldpay.etherjar.hex.HexData
import io.emeraldpay.etherjar.rpc.json.HexDataSerializer
data class LogMessage(
@get:JsonSerialize(using = HexDataSerializer::class)
val address: Address,
@get:JsonSerialize(using = HexDataSerializer::class)
val blockHash: BlockHash,
@get:JsonSerialize(using = NumberAsHexSerializer::class)
val blockNumber: Long,
@get:JsonSerialize(using = HexDataSerializer::class)
val data: HexData,
@get:JsonSerialize(using = NumberAsHexSerializer::class)
val logIndex: Long,
@get:JsonSerialize(contentUsing = HexDataSerializer::class)
val topics: List<Hex32>,
@get:JsonSerialize(using = HexDataSerializer::class)
val transactionHash: TransactionId,
@get:JsonSerialize(using = NumberAsHexSerializer::class)
val transactionIndex: Long,
val removed: Boolean
)

2
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NewHead.kt → src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NewHeadMessage.kt
View File

@@ -29,7 +29,7 @@ import java.time.Instant
* list of transactions. Also, our JSON doesn't include rarely used fields such as extraData, sha3uncles, stateRoot,
* transactionRoot and some others.
*/
data class NewHead(
data class NewHeadMessage(
@get:JsonSerialize(using = NumberAsHexSerializer::class)
val number: Long,
@get:JsonSerialize(using = HexDataSerializer::class)