StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move from homemade solutions to Caffeine

Browse Source
This commit is contained in:
a10zn8
2022-12-23 11:37:11 +04:00
parent 9fb302729f
commit 08b47d128c
6 changed files with 26 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
src/main/kotlin/io/emeraldpay/dshackle/commons/ExpiringSet.kt
View File

@@ -1,152 +0,0 @@
package io.emeraldpay.dshackle.commons
import org.apache.commons.collections4.iterators.UnmodifiableIterator
import org.slf4j.LoggerFactory
import java.time.Duration
import java.util.LinkedList
import java.util.TreeSet
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
/**
* A naive implementation of a Set with a limit for elements and an expiration time. Supposed to be used a filter for uniqueness.
* Internally it uses a TreeSet and a journal of added elements, which is used ot remove elements when they expire or the list grows too large.
* It's tread safe, but may be suboptimal to use in multithreaded scenario because of internal locks.
*/
class ExpiringSet<T>(
ttl: Duration,
comparator: Comparator<T>,
val limit: Int,
) : MutableSet<T> {
companion object {
private val log = LoggerFactory.getLogger(ExpiringSet::class.java)
}
private val tree = TreeSet<T>(comparator)
private val lock = ReentrantLock()
private val journal = LinkedList<JournalItem<T>>()
private var count = 0
private val ttl = ttl.toMillis()
data class JournalItem<T>(
val since: Long = System.currentTimeMillis(),
val value: T
) {
fun isExpired(ttl: Long): Boolean {
return System.currentTimeMillis() > since + ttl
}
}
override val size: Int
get() = count
override fun clear() {
lock.withLock {
tree.clear()
journal.clear()
count = 0
}
}
override fun addAll(elements: Collection<T>): Boolean {
var changed = false
elements.forEach {
changed = changed || add(it)
}
return changed
}
override fun add(element: T): Boolean {
lock.withLock {
val added = tree.add(element)
if (added) {
journal.offer(JournalItem(value = element))
count++
shrink()
}
return added
}
}
override fun isEmpty(): Boolean {
return count == 0
}
override fun iterator(): MutableIterator<T> {
// not mutable
return UnmodifiableIterator.unmodifiableIterator(tree.iterator())
}
override fun retainAll(elements: Collection<T>): Boolean {
lock.withLock {
var changed = false
val iter = tree.iterator()
while (iter.hasNext()) {
val next = iter.next()
if (!elements.contains(next)) {
changed = true
iter.remove()
count--
}
}
return changed
}
}
override fun removeAll(elements: Collection<T>): Boolean {
var changed = false
elements.forEach {
changed = changed || remove(it)
}
return changed
}
override fun remove(element: T): Boolean {
lock.withLock {
val changed = tree.remove(element)
if (changed) {
count--
}
return changed
}
}
override fun containsAll(elements: Collection<T>): Boolean {
return elements.all { contains(it) }
}
override fun contains(element: T): Boolean {
lock.withLock {
return tree.contains(element)
}
}
fun shrink() {
lock.withLock {
val iter = journal.iterator()
val removeAtLeast = (count - limit).coerceAtLeast(0)
var removed = 0
var stop = false
while (!stop && iter.hasNext()) {
val next = iter.next()
val overflow = removeAtLeast > removed
val expired = next.isExpired(ttl)
if (overflow || expired) {
iter.remove()
if (tree.remove(next.value)) {
removed++
}
}
// we always delete expired elements so don't stop on that
if (!expired) {
// but if we already deleted all non-expired element (i.e., started because it grew too large)
// then we stop as soon as we don't have any overflow
stop = !overflow
}
}
count -= removed
}
}
}

40
src/main/kotlin/io/emeraldpay/dshackle/data/RingSet.kt
View File

@@ -1,40 +0,0 @@
package io.emeraldpay.dshackle.data
import java.util.concurrent.atomic.AtomicReference
class RingSet<T>(
private val maxSize: Int
) : Set<T> {
private var setRef: AtomicReference<LinkedHashSet<T>> = AtomicReference(LinkedHashSet<T>())
override val size: Int
get() = setRef.get().size
fun add(element: T) {
setRef.getAndUpdate { set ->
if (!set.contains(element)) {
val copyset = LinkedHashSet<T>(set)
copyset.add(element)
if (copyset.size > maxSize) {
copyset.remove(set.elementAt(0))
}
copyset
} else {
set
}
}
}
override fun isEmpty(): Boolean {
return setRef.get().isEmpty()
}
override fun contains(element: @UnsafeVariance T): Boolean {
return setRef.get().contains(element)
}
override fun iterator(): Iterator<T> {
return setRef.get().iterator()
}
override fun containsAll(elements: Collection<@UnsafeVariance T>): Boolean {
return setRef.get().containsAll(elements)
}
}

23
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/AggregatedPendingTxes.kt
View File

@@ -15,10 +15,9 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum.subscribe
import io.emeraldpay.dshackle.commons.ExpiringSet
import com.github.benmanes.caffeine.cache.Caffeine
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.etherjar.domain.TransactionId
import io.emeraldpay.etherjar.hex.HexDataComparator
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import java.time.Duration
@@ -31,15 +30,21 @@ class AggregatedPendingTxes(
private val log = LoggerFactory.getLogger(AggregatedPendingTxes::class.java)
}
private val track = ExpiringSet<TransactionId>(
Duration.ofSeconds(30),
HexDataComparator() as Comparator<TransactionId>,
10_000
)
private val track = Caffeine.newBuilder()
.expireAfterWrite(Duration.ofSeconds(30))
.maximumSize(10_000)
.build<TransactionId, Boolean>()
override fun connect(matcher: Selector.Matcher): Flux<TransactionId> {
return Flux.merge(
sources.map { it.connect(matcher) } // todo check
).filter(track::add)
sources.map { it.connect(matcher) }
).filter {
val res = track.getIfPresent(it)
if (res == null) {
track.put(it, true)
return@filter true
}
return@filter false
}
}
}

10
src/main/kotlin/io/emeraldpay/dshackle/upstream/forkchoice/NoChoiceWithPriorityForkChoice.kt
View File

@@ -1,8 +1,8 @@
package io.emeraldpay.dshackle.upstream.forkchoice
import com.google.common.cache.CacheBuilder
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.RingSet
import org.slf4j.LoggerFactory
import java.util.concurrent.atomic.AtomicReference
@@ -11,7 +11,9 @@ class NoChoiceWithPriorityForkChoice(
private val upstreamId: String
) : ForkChoice {
private val head = AtomicReference<BlockContainer>(null)
private val seenBlocks = RingSet<BlockId>(100)
private val seenBlocks = CacheBuilder.newBuilder()
.maximumSize(100)
.build<BlockId, Boolean>()
companion object {
private val log = LoggerFactory.getLogger(NoChoiceWithPriorityForkChoice::class.java)
@@ -21,7 +23,7 @@ class NoChoiceWithPriorityForkChoice(
}
override fun filter(block: BlockContainer): Boolean {
return !seenBlocks.contains(block.hash)
return seenBlocks.getIfPresent(block.hash) == null
}
override fun choose(block: BlockContainer): ForkChoice.ChoiceResult {
@@ -31,7 +33,7 @@ class NoChoiceWithPriorityForkChoice(
log.debug("Already seen block ${block.height} from $upstreamId")
curr
} else {
seenBlocks.add(block.hash)
seenBlocks.put(block.hash, true)
block.copyWithRating(nodeRating)
}
}

10
src/main/kotlin/io/emeraldpay/dshackle/upstream/forkchoice/PriorityForkChoice.kt
View File

@@ -1,14 +1,16 @@
package io.emeraldpay.dshackle.upstream.forkchoice
import com.google.common.cache.CacheBuilder
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.RingSet
import org.slf4j.LoggerFactory
import java.util.concurrent.atomic.AtomicReference
class PriorityForkChoice : ForkChoice {
private val head = AtomicReference<BlockContainer>(null)
private val seenBlocks = RingSet<BlockId>(10)
private val seenBlocks = CacheBuilder.newBuilder()
.maximumSize(10)
.build<BlockId, Boolean>()
companion object {
private val log = LoggerFactory.getLogger(PriorityForkChoice::class.java)
@@ -20,7 +22,7 @@ class PriorityForkChoice : ForkChoice {
override fun filter(block: BlockContainer): Boolean {
val curr = head.get()
return (curr == null || curr.nodeRating <= block.nodeRating) && !seenBlocks.contains(block.hash)
return (curr == null || curr.nodeRating <= block.nodeRating) && seenBlocks.getIfPresent(block.hash) == null
}
override fun choose(block: BlockContainer): ForkChoice.ChoiceResult {
@@ -33,7 +35,7 @@ class PriorityForkChoice : ForkChoice {
curr
} else {
log.debug("Preparing to accept block ${block.height}")
seenBlocks.add(block.hash)
seenBlocks.put(block.hash, true)
block
}
}