solution: Redis caching

README.adoc

@@ -43,14 +43,13 @@ image::call-schema.png[alt="Call Schema",width=100%,align="center"]
 
 == Roadmap
 
-- [ ] Redis caching
+- [ ] JSON RPC emulation, in addition to gRPC protocol
+- [ ] *Support Bitcoin RPC*
+- [ ] Access to ERC-20 tokens on asset level
+- [ ] Subscription to bitcoind notification over gRPC (instead of ZeroMQ)
 - [ ] Prometheus monitoring
 - [ ] BIP-32 Pubkey
-- [ ] *Support Bitcoin RPC*
-- [ ] Subscription to bitcoind notification over GRPC (instead of ZeroMQ)
-- [ ] JSON RPC emulation, in addition to GRPC protocol
 - [ ] Lightweight sidecar node connector
-- [ ] Access to ERC-20 tokens on asset level
 - [ ] External logging
 - [ ] Configurable upstream roles
 

docs/09-caching.adoc

@@ -1,9 +1,67 @@
 == Caching
 
-=== In memory cache
+Dshackle can be configured to cache blockchain data. It can be a _hot_ in-memory cache, and optional _cold_ Redis-based
+cache.
+
+Dshackle has enough information to effectively cache data and evict outdated values. If some data has been removed from
+the blockchain, for example when block was replaced with another block at the same height, then the old values are
+immediately evicted from the caches.
+
+=== In-memory cache
 
 Dshackle keeps latest blocks in memory (by default 64 blocks)
 
 === Redis cache
 
-TBD
+Dshackle can optionally cache blocks and transactions in Redis cache. The values are cached up to 1 hour, but
+fresh blocks and transactions are cached for shorter period.
+
+It makes sense to reuse the same Redis cache between multiple instances of the Dshackle.
+
+.Basic config (dshackle.yaml)
+[source, yaml]
+----
+cache:
+  redis:
+    enabled: true
+----
+
+.Full config (dshackle.yaml)
+[source, yaml]
+----
+cache:
+  redis:
+    enabled: true
+    host: 127.0.0.1
+    port: 6379
+    db: 0
+    password: passw0rd!
+----
+
+
+.Options
+|===
+| Name | Default Value | Description
+
+| enabled
+| false
+| Set to `true` if Dshackle should use Redis for caching
+
+| host
+| 127.0.0.1
+| Redis host
+
+| port
+| 6379
+| Redis port
+
+| db
+| 0
+| Redis database
+
+| password
+| --
+| Password if Redis requires authentication. The value can be read from Environment variable, to do that
+  specify it as `${REDIS_PASSWORD}`, where REDIS_PASSWORD is the name of the variable
+
+|===

gradle.properties

@@ -15,7 +15,7 @@ springVersion=5.1.4.RELEASE
 reactorVersion=3.2.9.RELEASE
 
 # Our Libs
-etherjarVersion=0.9.0-SNAPSHOT
+etherjarVersion=0.10.0-SNAPSHOT
 
 # Testing
 spockVersion=1.2-groovy-2.5

src/main/kotlin/io/emeraldpay/dshackle/Config.kt

@@ -19,25 +19,22 @@ import com.fasterxml.jackson.core.Version
 import com.fasterxml.jackson.databind.DeserializationFeature
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.databind.module.SimpleModule
+import io.lettuce.core.AbstractRedisClient
+import io.lettuce.core.cluster.RedisClusterClient
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.beans.factory.annotation.Qualifier
-import org.springframework.context.ApplicationContext
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
-import org.springframework.context.annotation.Import
 import org.springframework.core.env.Environment
 import org.springframework.scheduling.annotation.EnableAsync
 import org.springframework.scheduling.annotation.EnableScheduling
-import org.springframework.scheduling.annotation.Scheduled
 import reactor.core.scheduler.Scheduler
 import reactor.core.scheduler.Schedulers
 import java.io.File
-import java.lang.IllegalStateException
 import java.text.SimpleDateFormat
 import java.util.*
 import java.util.concurrent.Executors
-import kotlin.system.exitProcess
 
 @Configuration
 @EnableScheduling
@@ -82,4 +79,9 @@ open class Config(
     open fun fileResolver(): FileResolver {
         return FileResolver(configDir())
     }
+
+    @Bean
+    open fun redisClient(): AbstractRedisClient {
+        return RedisClusterClient.create("redis://password@localhost:6379/0");
+    }
 }

src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksRedisCache.kt

@@ -24,8 +24,9 @@ class BlocksRedisCache(
 
     companion object {
         private val log = LoggerFactory.getLogger(BlocksRedisCache::class.java)
-        // max caching time is 24 hours
+        private const val MAX_CACHE_TIME_MINUTES = 60L
-        private const val MAX_CACHE_TIME_HOURS = 24L
+        // doesn't make sense to cached in redis short living objects
+        private const val MIN_CACHE_TIME_SECONDS = 10
     }
 
     override fun read(key: BlockHash): Mono<BlockJson<TransactionRefJson>> {
@@ -37,23 +38,36 @@ class BlocksRedisCache(
                 }
     }
 
+    fun evict(id: BlockHash): Mono<Void> {
+        return Mono.just(id)
+                .flatMap {
+                    redis.del(key(it))
+                }
+                .then()
+    }
+
     /**
      * Add to cache.
      * Note that it returns Mono<Void> which must be subscribed to actually save
      */
-    open fun add(block: BlockJson<TransactionRefJson>): Mono<Void> {
+    fun add(block: BlockJson<TransactionRefJson>): Mono<Void> {
         if (block.timestamp == null || block.hash == null) {
             return Mono.empty()
         }
         return Mono.just(block)
                 .flatMap { block ->
+
                     val data = objectMapper.writeValueAsString(block)
                     //default caching time is age of the block, i.e. block create hour ago
                     //keep for hour, but block create 10 seconds ago cache for 10 seconds, as it
                     //still can be replaced in the blockchain
                     val age = Instant.now().epochSecond - block.timestamp.epochSecond
-                    val ttl = min(age, TimeUnit.HOURS.toSeconds(MAX_CACHE_TIME_HOURS))
+                    val ttl = min(age, TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES))
+                    if (ttl > MIN_CACHE_TIME_SECONDS) {
                         redis.setex(key(block.hash), ttl, data)
+                    } else {
+                        Mono.empty()
+                    }
                 }
                 .doOnError {
                     log.warn("Failed to save to Redis: ${it.message}")
@@ -68,7 +82,7 @@ class BlocksRedisCache(
     /**
      * Key in Redis
      */
-    open fun key(hash: BlockHash): String {
+    fun key(hash: BlockHash): String {
         return "block:${chain.id}:${hash.toHex()}"
     }
 }

src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksWithTxCache.kt

@@ -2,8 +2,10 @@ package io.emeraldpay.dshackle.cache
 
 import io.emeraldpay.dshackle.reader.Reader
 import io.infinitape.etherjar.domain.BlockHash
+import io.infinitape.etherjar.domain.TransactionId
 import io.infinitape.etherjar.rpc.json.BlockJson
 import io.infinitape.etherjar.rpc.json.TransactionJson
+import io.infinitape.etherjar.rpc.json.TransactionRefJson
 import org.slf4j.LoggerFactory
 import org.springframework.beans.BeanUtils
 import reactor.core.publisher.Flux
@@ -17,8 +19,8 @@ import reactor.core.publisher.Mono
  * If any of the expected block transactions is not available it returns empty
  */
 class BlocksWithTxCache(
-        private val blocks: BlocksMemCache,
+        private val blocks: Reader<BlockHash, BlockJson<TransactionRefJson>>,
-        private val txes: TxMemCache
+        private val txes: Reader<TransactionId, TransactionJson>
 ): Reader<BlockHash, BlockJson<TransactionJson>> {
 
     companion object {

src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt

@@ -1,5 +1,6 @@
 package io.emeraldpay.dshackle.cache
 
+import io.emeraldpay.dshackle.reader.CompoundReader
 import io.emeraldpay.dshackle.reader.Reader
 import io.infinitape.etherjar.domain.BlockHash
 import io.infinitape.etherjar.domain.TransactionId
@@ -7,11 +8,16 @@ import io.infinitape.etherjar.rpc.json.BlockJson
 import io.infinitape.etherjar.rpc.json.TransactionJson
 import io.infinitape.etherjar.rpc.json.TransactionRefJson
 import org.slf4j.LoggerFactory
+import reactor.core.publisher.Flux
+import reactor.core.publisher.Mono
+import reactor.core.publisher.TopicProcessor
 
 open class Caches(
-        private val blocksByHash: BlocksMemCache,
+        private val memBlocksByHash: BlocksMemCache,
         private val blocksByHeight: HeightCache,
-        private val txsByHash: TxMemCache
+        private val memTxsByHash: TxMemCache,
+        private val redisBlocksByHash: BlocksRedisCache?,
+        private val redisTxsByHash: TxRedisCache?
 ) {
 
     companion object {
@@ -28,6 +34,22 @@ open class Caches(
         }
     }
 
+    private val blocksByHash: Reader<BlockHash, BlockJson<TransactionRefJson>>
+    private val txsByHash: Reader<TransactionId, TransactionJson>
+
+    init {
+        blocksByHash = if (redisBlocksByHash == null) {
+            memBlocksByHash
+        } else {
+            CompoundReader(memBlocksByHash, redisBlocksByHash)
+        }
+        txsByHash = if (redisTxsByHash == null) {
+            memTxsByHash
+        } else {
+            CompoundReader(memTxsByHash, redisTxsByHash)
+        }
+    }
+
     /**
      * Cache data that was just requested
      */
@@ -40,32 +62,51 @@ open class Caches(
     }
 
     fun cache(tag: Tag, tx: TransactionJson) {
-        txsByHash.add(tx)
+        //do not cache transactions that are not in a block yet
+        if (tx.blockHash == null) {
+            return
+        }
+        memTxsByHash.add(tx)
+        memBlocksByHash.get(tx.blockHash)?.let { block ->
+            redisTxsByHash?.add(tx, block)
+        }
     }
 
     fun cache(tag: Tag, block: BlockJson<TransactionRefJson>) {
+        val job = ArrayList<Mono<Void>>()
         if (tag == Tag.LATEST) {
-            blocksByHash.add(block)
+            //for LATEST data cache in memory, it will be short living so better to avoid Redis
+            memBlocksByHash.add(block)
             val replaced = blocksByHeight.add(block)
             //evict cached transactions if an existing block was updated
             replaced?.let { replacedBlockHash ->
                 var evicted = false
-                blocksByHash.get(replacedBlockHash)?.let { block ->
+                redisBlocksByHash?.evict(replacedBlockHash)
-                    txsByHash.evict(block)
+                memBlocksByHash.get(replacedBlockHash)?.let { block ->
+                    memTxsByHash.evict(block)
+                    redisTxsByHash?.evict(block)
                     evicted = true
                 }
                 if (!evicted) {
-                    txsByHash.evict(replacedBlockHash)
+                    memTxsByHash.evict(replacedBlockHash)
                 }
             }
         } else if (tag == Tag.REQUESTED) {
-            // if block with transactions was requests cache only transactions
+            //shouldn't cache block json with transactions, separate txes and blocks with refs
-            block.transactions.forEach { tx ->
+            val blockOnly = block.withoutTransactionDetails()
-                if (tx is TransactionJson) {
+            memBlocksByHash.add(blockOnly)
-                    cache(Tag.REQUESTED, tx)
+            redisBlocksByHash?.add(blockOnly)?.let(job::add)
+
+            // now cache only transactions
+            val transactions = block.transactions.filterIsInstance<TransactionJson>()
+            if (transactions.isNotEmpty()) {
+                transactions.forEach { cache(Tag.REQUESTED, it) }
+                if (redisTxsByHash != null) {
+                    job.add(Flux.fromIterable(transactions).flatMap { redisTxsByHash.add(it, block) }.then())
                 }
             }
         }
+        Flux.fromIterable(job).flatMap { it }.subscribe() //TODO move out to a caller
     }
 
     fun getBlocksByHash(): Reader<BlockHash, BlockJson<TransactionRefJson>> {
@@ -107,12 +148,19 @@ open class Caches(
         private var blocksByHash: BlocksMemCache? = null
         private var blocksByHeight: HeightCache? = null
         private var txsByHash: TxMemCache? = null
+        private var redisBlocksByHash: BlocksRedisCache? = null
+        private var redisTxsByHash: TxRedisCache? = null
 
         fun setBlockByHash(cache: BlocksMemCache): Builder {
             blocksByHash = cache
             return this
         }
 
+        fun setBlockByHash(cache: BlocksRedisCache): Builder {
+            redisBlocksByHash = cache
+            return this
+        }
+
         fun setBlockByHeight(cache: HeightCache): Builder {
             blocksByHeight = cache
             return this
@@ -123,6 +171,11 @@ open class Caches(
             return this
         }
 
+        fun setTxByHash(cache: TxRedisCache): Builder {
+            redisTxsByHash = cache
+            return this
+        }
+
         fun build(): Caches {
             if (blocksByHash == null) {
                 blocksByHash = BlocksMemCache()
@@ -133,7 +186,7 @@ open class Caches(
             if (txsByHash == null) {
                 txsByHash = TxMemCache()
             }
-            return Caches(blocksByHash!!, blocksByHeight!!, txsByHash!!)
+            return Caches(blocksByHash!!, blocksByHeight!!, txsByHash!!, redisBlocksByHash, redisTxsByHash)
         }
     }
 }

src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt

@@ -0,0 +1,86 @@
+package io.emeraldpay.dshackle.cache
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import io.emeraldpay.dshackle.config.EnvVariables
+import io.emeraldpay.grpc.Chain
+import io.lettuce.core.RedisClient
+import io.lettuce.core.RedisURI
+import io.lettuce.core.api.StatefulRedisConnection
+import org.slf4j.LoggerFactory
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.beans.factory.annotation.Value
+import org.springframework.core.env.Environment
+import org.springframework.stereotype.Repository
+import java.util.*
+import java.util.concurrent.ConcurrentHashMap
+import javax.annotation.PostConstruct
+import kotlin.collections.HashMap
+
+
+@Repository
+class CachesFactory(
+        @Autowired private val objectMapper: ObjectMapper,
+        @Autowired private val env: Environment
+) {
+
+    companion object {
+        private val log = LoggerFactory.getLogger(CachesFactory::class.java)
+        private const val CONFIG_PREFIX = "cache.redis"
+    }
+
+    private var redis: StatefulRedisConnection<String, String>? = null
+    private val all = EnumMap<Chain, Caches>(io.emeraldpay.grpc.Chain::class.java)
+
+    @PostConstruct
+    fun init() {
+        if (!env.getProperty("${CONFIG_PREFIX}.enabled", Boolean::class.java, false)) {
+            return
+        }
+        val address = env.getProperty("${CONFIG_PREFIX}.host", "127.0.0.1")
+        val port = env.getProperty("${CONFIG_PREFIX}.port", Int::class.java, 6379)
+
+        var uri = RedisURI.builder()
+                .withHost(address)
+                .withPort(port)
+
+        env.getProperty("${CONFIG_PREFIX}.db", Int::class.java)?.let { value ->
+            uri = uri.withDatabase(value)
+        }
+
+        //log URI _before_ adding a password, to avoid leaking it to the log
+        log.info("Use Redis cache at: ${uri.build().toURI()}")
+
+        env.getProperty("${CONFIG_PREFIX}.password")?.let { value ->
+            uri = uri.withPassword(value)
+        }
+
+        val client = RedisClient.create(uri.build())
+        val ping = client.connect().sync().ping()
+        if (ping != "PONG") {
+            throw IllegalStateException("Redis connection is not configured. Response: $ping")
+        }
+        redis = client.connect()
+    }
+
+    private fun initCache(chain: Chain): Caches {
+        val caches = Caches.newBuilder()
+        redis?.let { redis ->
+            caches.setBlockByHash(BlocksRedisCache(redis.reactive(), chain, objectMapper))
+            caches.setTxByHash(TxRedisCache(redis.reactive(), chain, objectMapper))
+        }
+        return caches.build()
+    }
+
+    fun getCaches(chain: Chain): Caches {
+        val existing = all[chain]
+        if (existing == null) {
+            synchronized(all) {
+                if (!all.containsKey(chain)) {
+                    all[chain] = initCache(chain)
+                }
+            }
+            return getCaches(chain)
+        }
+        return existing
+    }
+}

src/main/kotlin/io/emeraldpay/dshackle/cache/TxRedisCache.kt

@@ -39,7 +39,7 @@ class TxRedisCache(
                 }
     }
 
-    open fun evict(block: BlockJson<TransactionRefJson>): Mono<Void> {
+    fun evict(block: BlockJson<TransactionRefJson>): Mono<Void> {
         return Mono.just(block)
                 .map { block ->
                     block.transactions.map {
@@ -50,8 +50,15 @@ class TxRedisCache(
                 }.then()
     }
 
+    fun evict(id: TransactionId): Mono<Void> {
+        return Mono.just(id)
+                .flatMap {
+                    redis.del(key(it))
+                }
+                .then()
+    }
 
-    open fun add(tx: TransactionJson, block: BlockJson<TransactionRefJson>): Mono<Void> {
+    fun add(tx: TransactionJson, block: BlockJson<TransactionRefJson>): Mono<Void> {
         if (tx.blockHash == null || block.hash == null || tx.blockHash != block.hash || block.timestamp == null) {
             return Mono.empty()
         }
@@ -78,7 +85,7 @@ class TxRedisCache(
     /**
      * Key in Redis
      */
-    open fun key(hash: TransactionId): String {
+    fun key(hash: TransactionId): String {
         return "tx:${chain.id}:${hash.toHex()}"
     }
 }

src/main/kotlin/io/emeraldpay/dshackle/config/EnvVariables.kt

@@ -0,0 +1,19 @@
+package io.emeraldpay.dshackle.config
+
+/**
+ * Update configuration value from environment variables. Format: ${ENV_VAR_NAME}
+ */
+class EnvVariables {
+
+    companion object {
+        private val envRegex = Regex("\\$\\{(\\w+?)}")
+    }
+
+    fun postProcess(value: String): String {
+        return envRegex.replace(value) { m ->
+            m.groups[1]?.let { g ->
+                System.getProperty(g.value) ?: System.getenv(g.value) ?: ""
+            } ?: ""
+        }
+    }
+}

src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt

@@ -32,7 +32,7 @@ import java.time.Duration
 class UpstreamsConfigReader {
 
     private val log = LoggerFactory.getLogger(UpstreamsConfigReader::class.java)
-    private val envRegex = Regex("\\$\\{(\\w+?)}")
+    private val envVariables = EnvVariables()
 
     fun read(input: InputStream): UpstreamsConfig {
         val yaml = Yaml()
@@ -268,13 +268,13 @@ class UpstreamsConfigReader {
     private fun getListOfString(mappingNode: MappingNode?, key: String): List<String>? {
         return getList<ScalarNode>(mappingNode, key)?.value
                 ?.map { it.value }
-                ?.map(this::postProcess)
+                ?.map(envVariables::postProcess)
     }
 
     private fun getValueAsString(mappingNode: MappingNode?, key: String): String? {
         return getValue(mappingNode, key)?.let {
              return@let it.value
-        }?.let(this::postProcess)
+        }?.let(envVariables::postProcess)
     }
 
     private fun getValueAsInt(mappingNode: MappingNode?, key: String): Int? {
@@ -305,11 +305,4 @@ class UpstreamsConfigReader {
         }
     }
 
-    fun postProcess(value: String): String {
-        return envRegex.replace(value) { m ->
-            m.groups[1]?.let { g ->
-                System.getProperty(g.value) ?: System.getenv(g.value) ?: ""
-            } ?: ""
-        }
-    }
 }

src/main/kotlin/io/emeraldpay/dshackle/reader/CompoundReader.kt

@@ -15,24 +15,22 @@
  */
 package io.emeraldpay.dshackle.reader
 
+import reactor.core.publisher.Flux
 import reactor.core.publisher.Mono
 
+/**
+ * Composition of multiple readers. Reader returns first value returned by any of the source readers.
+ */
 class CompoundReader<K, D>(
-        private val readers: Collection<Reader<K, D>>
+        private vararg val readers: Reader<K, D>
 ): Reader<K, D> {
 
     override fun read(key: K): Mono<D> {
         if (readers.isEmpty()) {
             return Mono.empty()
         }
-        var result = readers.first().read(key)
+        return Flux.fromIterable(readers.asIterable())
-        if (readers.size == 1) {
+                .flatMap { it.read(key) }.next()
-            return result
-        }
-        readers.stream().skip(1).forEach {
-            result = result.switchIfEmpty(it.read(key))
-        }
-        return result
     }
 
 }

src/main/kotlin/io/emeraldpay/dshackle/upstream/CachingEthereumApi.kt

@@ -38,6 +38,9 @@ open class CachingEthereumApi(
     companion object {
         private val log = LoggerFactory.getLogger(CachingEthereumApi::class.java)
 
+        /**
+         * Create caching API with empty memory-only cache
+         */
         @JvmStatic
         fun empty(): CachingEthereumApi {
             return CachingEthereumApi(ObjectMapper(), Caches.default(), EmptyEthereumHead())

src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentUpstreams.kt

@@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.upstream
 import com.fasterxml.jackson.databind.ObjectMapper
 import io.emeraldpay.dshackle.cache.Caches
 import io.emeraldpay.dshackle.cache.CachesEnabled
+import io.emeraldpay.dshackle.cache.CachesFactory
 import io.emeraldpay.dshackle.startup.UpstreamChange
 import io.emeraldpay.dshackle.upstream.calls.CallMethods
 import io.emeraldpay.dshackle.upstream.calls.QuorumBasedMethods
@@ -35,7 +36,8 @@ import kotlin.concurrent.withLock
 
 @Repository
 class CurrentUpstreams(
-        @Autowired private val objectMapper: ObjectMapper
+        @Autowired private val objectMapper: ObjectMapper,
+        @Autowired private val cachesFactory: CachesFactory
 ): Upstreams {
 
     private val log = LoggerFactory.getLogger(CurrentUpstreams::class.java)
@@ -55,7 +57,7 @@ class CurrentUpstreams(
                 log.info("Upstream ${change.upstream.getId()} with chain $chain has been removed")
             } else {
                 if (current == null) {
-                    val created = ChainUpstreams(chain, ArrayList<Upstream>(), Caches.default(), objectMapper)
+                    val created = ChainUpstreams(chain, ArrayList<Upstream>(), cachesFactory.getCaches(chain), objectMapper)
                     if (up is CachesEnabled) {
                         up.setCaches(created.caches)
                     }

src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DirectEthereumApi.kt

@@ -101,7 +101,12 @@ open class DirectEthereumApi(
         return rpcClient.execute(callMapping(method, params))
                 .timeout(timeout, Mono.error(RpcException(-32603, "Upstream timeout")))
                 .doOnNext { value ->
+                    try {
                         caches?.cacheRequested(value)
+                    } catch (e: Throwable) {
+                        //ignore all caching errors, client shouldn't have problems because of them
+                        log.warn("Uncaught caching exception", e)
+                    }
                 }
     }
 

src/test/groovy/io/emeraldpay/dshackle/cache/BlocksRedisCacheSpec.groovy

@@ -52,4 +52,56 @@ class BlocksRedisCacheSpec extends Specification {
         act == block
     }
 
+    def "Evict existing block"() {
+        setup:
+        def cache = new BlocksRedisCache(
+                redis.reactive(), Chain.ETHEREUM, TestingCommons.objectMapper()
+        )
+        def block = new BlockJson<TransactionRefJson>()
+        block.number = 100
+        block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS)
+        block.hash = BlockHash.from(hash2)
+        block.transactions = []
+        block.uncles = []
+
+        when:
+        cache.add(block).subscribe()
+        def act = cache.read(BlockHash.from(hash2)).block()
+        then:
+        act == block
+
+        when:
+        cache.evict(block.hash).subscribe()
+        act = cache.read(BlockHash.from(hash2)).block()
+
+        then:
+        act == null
+    }
+
+    def "Evict non-existing block"() {
+        setup:
+        def cache = new BlocksRedisCache(
+                redis.reactive(), Chain.ETHEREUM, TestingCommons.objectMapper()
+        )
+        def block = new BlockJson<TransactionRefJson>()
+        block.number = 100
+        block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS)
+        block.hash = BlockHash.from(hash2)
+        block.transactions = []
+        block.uncles = []
+
+        when:
+        cache.add(block).subscribe()
+        def act = cache.read(BlockHash.from(hash2)).block()
+        then:
+        act == block
+
+        when:
+        cache.evict(BlockHash.from(hash3)).subscribe()
+        act = cache.read(BlockHash.from(hash2)).block()
+
+        then:
+        act == block
+    }
+
 }

src/test/groovy/io/emeraldpay/dshackle/cache/TxRedisCacheSpec.groovy

@@ -54,6 +54,35 @@ class TxRedisCacheSpec extends Specification {
         act == tx
     }
 
+    def "Evict single tx"() {
+        setup:
+        def block = new BlockJson<TransactionRefJson>()
+        block.number = 100
+        block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS)
+        block.hash = BlockHash.from(hash2)
+        block.transactions = []
+        block.uncles = []
+
+        def tx = new TransactionJson()
+        tx.hash = TransactionId.from(hash3)
+        tx.blockHash = block.hash
+        tx.blockNumber = block.number
+        tx.value = Wei.ofEthers(1.234)
+        tx.nonce = 0
+
+        when:
+        cache.add(tx, block).subscribe()
+        def act = cache.read(tx.hash).block()
+        then:
+        act == tx
+
+        when:
+        cache.evict(tx.hash).subscribe()
+        act = cache.read(tx.hash).block()
+        then:
+        act == null
+    }
+
     def "Evict all by block data"() {
         when:
         def block1 = new BlockJson<TransactionRefJson>()

src/test/groovy/io/emeraldpay/dshackle/config/EnvVariablesSpec.groovy

@@ -0,0 +1,29 @@
+package io.emeraldpay.dshackle.config
+
+import spock.lang.Specification
+
+class EnvVariablesSpec extends Specification {
+
+    EnvVariables reader = new EnvVariables()
+
+    def "Post process for usual strings"() {
+        expect:
+        s == reader.postProcess(s)
+        where:
+        s << ["", "a", "13143", "/etc/client1.myservice.com.key", "true", "1a68f20154fc258fe4149c199ad8f281"]
+    }
+
+    def "Post process replaces from env"() {
+        setup:
+        System.setProperty("id", "1")
+        System.setProperty("HOME", "/home/user")
+        System.setProperty("PASSWORD", "1a68f20154fc258fe4149c199ad8f281")
+        expect:
+        replaced == reader.postProcess(orig)
+        where:
+        orig                | replaced
+        "p_\${id}"          | "p_1"
+        "home: \${HOME}"    | "home: /home/user"
+        "\${PASSWORD}"      | "1a68f20154fc258fe4149c199ad8f281"
+    }
+}

src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy

@@ -134,27 +134,6 @@ class UpstreamsConfigReaderSpec extends Specification {
         }
     }
 
-    def "Post process for usual strings"() {
-        expect:
-        s == reader.postProcess(s)
-        where:
-        s << ["", "a", "13143", "/etc/client1.myservice.com.key", "true", "1a68f20154fc258fe4149c199ad8f281"]
-    }
-
-    def "Post process replaces from env"() {
-        setup:
-        System.setProperty("id", "1")
-        System.setProperty("HOME", "/home/user")
-        System.setProperty("PASSWORD", "1a68f20154fc258fe4149c199ad8f281")
-        expect:
-        replaced == reader.postProcess(orig)
-        where:
-        orig                | replaced
-        "p_\${id}"          | "p_1"
-        "home: \${HOME}"    | "home: /home/user"
-        "\${PASSWORD}"      | "1a68f20154fc258fe4149c199ad8f281"
-    }
-
     def "Parse config without defaults"() {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("upstreams-no-defaults.yaml")

src/test/groovy/io/emeraldpay/dshackle/reader/CompoundReaderSpec.groovy

@@ -0,0 +1,95 @@
+package io.emeraldpay.dshackle.reader
+
+import reactor.core.publisher.Mono
+import reactor.test.StepVerifier
+import spock.lang.Specification
+
+import java.time.Duration
+
+class CompoundReaderSpec extends Specification {
+
+    def reader1 = new Reader<String, String>() {
+        @Override
+        Mono<String> read(String key) {
+            return Mono.just("test-1").delaySubscription(Duration.ofMillis(100))
+        }
+    }
+    def reader2 = new Reader<String, String>() {
+        @Override
+        Mono<String> read(String key) {
+            return Mono.just("test-2").delaySubscription(Duration.ofMillis(200))
+        }
+    }
+    def reader3 = new Reader<String, String>() {
+        @Override
+        Mono<String> read(String key) {
+            return Mono.just("test-3").delaySubscription(Duration.ofMillis(300))
+        }
+    }
+
+    def reader1Empty = new Reader<String, String>() {
+        @Override
+        Mono<String> read(String key) {
+            return Mono.<String>empty().delaySubscription(Duration.ofMillis(100))
+        }
+    }
+
+    def "Return empty when no readers"() {
+        setup:
+        def reader = new CompoundReader<String, String>()
+        when:
+        def act = reader.read("test")
+        then:
+        StepVerifier.create(act)
+            .expectComplete()
+            .verify(Duration.ofSeconds(1))
+    }
+
+    def "Return first"() {
+        setup:
+        def reader = new CompoundReader<String, String>(reader1, reader2, reader3)
+        when:
+        def act = reader.read("test")
+        then:
+        StepVerifier.create(act)
+                .expectNext("test-1")
+                .expectComplete()
+                .verify(Duration.ofSeconds(1))
+    }
+
+    def "Return second"() {
+        setup:
+        def reader = new CompoundReader<String, String>(reader3, reader2)
+        when:
+        def act = reader.read("test")
+        then:
+        StepVerifier.create(act)
+                .expectNext("test-2")
+                .expectComplete()
+                .verify(Duration.ofSeconds(1))
+    }
+
+    def "Return third"() {
+        setup:
+        def reader = new CompoundReader<String, String>(reader3, reader2, reader1)
+        when:
+        def act = reader.read("test")
+        then:
+        StepVerifier.create(act)
+                .expectNext("test-1")
+                .expectComplete()
+                .verify(Duration.ofSeconds(1))
+    }
+
+    def "Ignore empty"() {
+        setup:
+        def reader = new CompoundReader<String, String>(reader3, reader1Empty, reader2, reader1Empty)
+        when:
+        def act = reader.read("test")
+        then:
+        StepVerifier.create(act)
+                .expectNext("test-2")
+                .expectComplete()
+                .verify(Duration.ofSeconds(1))
+    }
+}

src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy

@@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.databind.module.SimpleModule
 import io.emeraldpay.dshackle.cache.Caches
+import io.emeraldpay.dshackle.cache.CachesFactory
 import io.emeraldpay.dshackle.upstream.AggregatedUpstream
 import io.emeraldpay.dshackle.upstream.ChainUpstreams
 import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods
@@ -28,6 +29,7 @@ import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream
 import io.emeraldpay.grpc.Chain
 import io.infinitape.etherjar.rpc.JacksonRpcConverter
 import io.infinitape.etherjar.rpc.ReactorRpcClient
+import org.springframework.core.env.StandardEnvironment
 
 import java.text.SimpleDateFormat
 
@@ -73,4 +75,8 @@ class TestingCommons {
     static AggregatedUpstream aggregatedUpstream(EthereumUpstream up) {
         return new ChainUpstreams(Chain.ETHEREUM, [up], Caches.default(), objectMapper())
     }
+
+    static CachesFactory emptyCaches() {
+        return new CachesFactory(objectMapper(), new StandardEnvironment())
+    }
 }

src/test/groovy/io/emeraldpay/dshackle/upstream/CurrentUpstreamsSpec.groovy

@@ -11,7 +11,7 @@ class CurrentUpstreamsSpec extends Specification {
 
     def "add upstream"() {
         setup:
-        def current = new CurrentUpstreams(TestingCommons.objectMapper())
+        def current = new CurrentUpstreams(TestingCommons.objectMapper(), TestingCommons.emptyCaches())
         def up = new EthereumUpstreamMock("test", Chain.ETHEREUM, TestingCommons.api(Stub(ReactorRpcClient)))
         when:
         current.update(new UpstreamChange(Chain.ETHEREUM, up, UpstreamChange.ChangeType.ADDED))
@@ -22,7 +22,7 @@ class CurrentUpstreamsSpec extends Specification {
 
     def "add multiple upstreams"() {
         setup:
-        def current = new CurrentUpstreams(TestingCommons.objectMapper())
+        def current = new CurrentUpstreams(TestingCommons.objectMapper(), TestingCommons.emptyCaches())
         def up1 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api(Stub(ReactorRpcClient)))
         def up2 = new EthereumUpstreamMock("test2", Chain.ETHEREUM_CLASSIC, TestingCommons.api(Stub(ReactorRpcClient)))
         def up3 = new EthereumUpstreamMock("test3", Chain.ETHEREUM, TestingCommons.api(Stub(ReactorRpcClient)))
@@ -38,7 +38,7 @@ class CurrentUpstreamsSpec extends Specification {
 
     def "remove upstream"() {
         setup:
-        def current = new CurrentUpstreams(TestingCommons.objectMapper())
+        def current = new CurrentUpstreams(TestingCommons.objectMapper(), TestingCommons.emptyCaches())
         def up1 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api(Stub(ReactorRpcClient)))
         def up2 = new EthereumUpstreamMock("test2", Chain.ETHEREUM_CLASSIC, TestingCommons.api(Stub(ReactorRpcClient)))
         def up3 = new EthereumUpstreamMock("test3", Chain.ETHEREUM, TestingCommons.api(Stub(ReactorRpcClient)))
@@ -56,7 +56,7 @@ class CurrentUpstreamsSpec extends Specification {
 
     def "available after adding"() {
         setup:
-        def current = new CurrentUpstreams(TestingCommons.objectMapper())
+        def current = new CurrentUpstreams(TestingCommons.objectMapper(), TestingCommons.emptyCaches())
         def up1 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api(Stub(ReactorRpcClient)))
 
         when:

src/test/resources/upstreams-bitcoin.yaml

@@ -0,0 +1,15 @@
+version: v1
+
+defaultOptions:
+  - chains:
+      - bitcoin
+    options:
+      min-peers: 3
+
+upstreams:
+  - id: local
+    chain: bitcoin
+    connection:
+      bitcoin:
+        rpc:
+          url: "http://localhost:8545"