problem: reading yaml to both plain properties and structured objects is a mess

solution: use just objects for configs

src/main/kotlin/io/emeraldpay/dshackle/Config.kt

@@ -19,13 +19,14 @@ import com.fasterxml.jackson.core.Version
 import com.fasterxml.jackson.databind.DeserializationFeature
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.databind.module.SimpleModule
-import io.lettuce.core.AbstractRedisClient
-import io.lettuce.core.cluster.RedisClusterClient
+import io.emeraldpay.dshackle.config.CacheConfig
+import io.emeraldpay.dshackle.config.MainConfig
+import io.emeraldpay.dshackle.config.MainConfigReader
+import io.emeraldpay.dshackle.config.UpstreamsConfig
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.beans.factory.annotation.Qualifier
-import org.springframework.context.annotation.Bean
-import org.springframework.context.annotation.Configuration
+import org.springframework.context.annotation.*
 import org.springframework.core.env.Environment
 import org.springframework.scheduling.annotation.EnableAsync
 import org.springframework.scheduling.annotation.EnableScheduling
@@ -43,11 +44,37 @@ open class Config(
         @Autowired private val env: Environment
 ) {
 
-    private val log = LoggerFactory.getLogger(Config::class.java)
+    companion object {
+        private val log = LoggerFactory.getLogger(Config::class.java)
+
+        private const val DEFAULT_CONFIG = "/etc/dshackle/dshackle.yaml"
+        private const val LOCAL_CONFIG = "./dshackle.yaml"
+    }
+
+    private var configFilePath: File? = null
+
+    init {
+        configFilePath = getConfigPath()
+    }
+
+    fun getConfigPath(): File {
+        env.getProperty("configPath")?.let {
+            return File(it).normalize()
+        }
+        var target = File(DEFAULT_CONFIG)
+        if (!FileResolver.isAccessible(target)) {
+            target = File(LOCAL_CONFIG)
+            if (!FileResolver.isAccessible(target)) {
+                throw IllegalStateException("Configuration is not found neither at ${DEFAULT_CONFIG} nor ${LOCAL_CONFIG}")
+            }
+        }
+        target = target.normalize()
+        return target
+    }
 
     @Bean
     open fun objectMapper(): ObjectMapper {
-        val module = SimpleModule("EmeraldDShackle", Version(1, 0, 0, null, null, null))
+        val module = SimpleModule("EmeraldDshackle", Version(1, 0, 0, null, null, null))
 
         val objectMapper = ObjectMapper()
         objectMapper.registerModule(module)
@@ -65,23 +92,28 @@ open class Config(
     }
 
     @Bean
-    @Qualifier("configDir")
-    open fun configDir(): File {
-        val config = env.getProperty("configPath") ?: throw IllegalStateException("Config path is not set")
-        if (config.trim().isEmpty()) {
-            throw IllegalStateException("Config path is empty")
-        }
-        log.info("Use configuration from: $config")
-        return File(config).parentFile
+    open fun mainConfig(@Autowired fileResolver: FileResolver): MainConfig {
+        val f = configFilePath ?: throw IllegalStateException("Config path is not set")
+        log.info("Using config: ${f.absolutePath}")
+        val reader = MainConfigReader(fileResolver)
+        return reader.read(f.inputStream())
+                ?: throw IllegalStateException("Config is not available at ${f.absolutePath}")
     }
 
     @Bean
     open fun fileResolver(): FileResolver {
-        return FileResolver(configDir())
+        val f = configFilePath ?: throw IllegalStateException("Config path is not set")
+        return FileResolver(f.absoluteFile.parentFile)
     }
 
     @Bean
-    open fun redisClient(): AbstractRedisClient {
-        return RedisClusterClient.create("redis://password@localhost:6379/0");
+    open fun upstreamsConfig(@Autowired mainConfig: MainConfig): UpstreamsConfig? {
+        return mainConfig.upstreams
     }
+
+    @Bean
+    open fun cacheConfig(@Autowired mainConfig: MainConfig): CacheConfig {
+        return mainConfig.cache ?: CacheConfig()
+    }
+
 }

src/main/kotlin/io/emeraldpay/dshackle/DshackleEnvironment.kt

@@ -1,81 +0,0 @@
-/**
- * Copyright (c) 2019 ETCDEV GmbH
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *        http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.emeraldpay.dshackle
-
-import io.emeraldpay.dshackle.config.ProxyConfig
-import io.emeraldpay.dshackle.config.ProxyConfigReader
-import org.slf4j.LoggerFactory
-import org.springframework.beans.factory.config.YamlPropertiesFactoryBean
-import org.springframework.core.env.*
-import org.springframework.core.io.FileSystemResource
-import org.springframework.core.io.Resource
-import org.springframework.core.io.support.ResourcePropertySource
-import java.io.File
-import java.util.*
-
-const val DEFAULT_CONFIG = "/etc/dshackle/dshackle.yaml"
-const val LOCAL_CONFIG = "./dshackle.yaml"
-
-open class DshackleEnvironment: StandardEnvironment() {
-
-    companion object {
-        private val log = LoggerFactory.getLogger(DshackleEnvironment::class.java)
-    }
-
-    override fun customizePropertySources(propertySources: MutablePropertySources) {
-        super.customizePropertySources(propertySources)
-        propertySources.addLast(mainConfig())
-        propertySources.addLast(ResourcePropertySource("version.properties"))
-    }
-
-    open fun getResource(): File? {
-        var target = File(DEFAULT_CONFIG)
-        if (!isAcceptedConfig(target)) {
-            target = File(LOCAL_CONFIG)
-            if (!isAcceptedConfig(target)) {
-                log.error("Configuration is not found neither at $DEFAULT_CONFIG nor $LOCAL_CONFIG")
-                return null
-            }
-        }
-        target = target.normalize()
-        return target
-    }
-
-    open fun mainConfig(): PropertySource<*> {
-        val target = getResource() ?: return PropertySource.named("mainConfig")
-        val resource = FileSystemResource(target)
-        val loadedProperties = this.loadYaml(resource)
-        loadedProperties["configPath"] = target.absolutePath
-        loadedProperties[ProxyConfig.CONFIG_ID] = extractYamlProxy(resource)
-        return PropertiesPropertySource("mainConfig", loadedProperties)
-    }
-
-    protected fun loadYaml(resource: Resource): Properties {
-        val factory = YamlPropertiesFactoryBean()
-        factory.setResources(resource)
-        factory.afterPropertiesSet()
-        return factory.getObject()!!
-    }
-
-    protected fun extractYamlProxy(resource: Resource): ProxyConfig? {
-        val reader = ProxyConfigReader()
-        return reader.read(resource.inputStream)
-    }
-
-    protected fun isAcceptedConfig(target: File): Boolean {
-        return target.exists() && target.isFile
-    }
-}

src/main/kotlin/io/emeraldpay/dshackle/FileResolver.kt

@@ -21,6 +21,12 @@ class FileResolver(
         private val baseDir: File
 ) {
 
+    companion object {
+        fun isAccessible(file: File): Boolean {
+            return file.exists() && file.isFile && file.canRead()
+        }
+    }
+
     fun resolve(path: String): File {
         val direct = File(path)
         if (direct.isAbsolute) {

src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt

@@ -15,25 +15,27 @@
  */
 package io.emeraldpay.dshackle
 
+import io.emeraldpay.dshackle.config.MainConfig
 import io.grpc.*
 import io.grpc.netty.GrpcSslContexts
 import io.grpc.netty.NettyServerBuilder
 import io.netty.handler.ssl.ClientAuth
+import io.netty.handler.ssl.SslContext
 import org.apache.commons.lang3.StringUtils
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.core.env.Environment
 import org.springframework.core.io.ResourceLoader
 import org.springframework.stereotype.Service
+import java.net.InetSocketAddress
 import javax.annotation.PostConstruct
 import javax.annotation.PreDestroy
 
 @Service
 open class GrpcServer(
         @Autowired val rpcs: List<io.grpc.BindableService>,
-        @Autowired val resourceLoader: ResourceLoader,
-        @Autowired val env: Environment,
-        @Autowired val fileResolver: FileResolver
+        @Autowired val mainConfig: MainConfig,
+        @Autowired val tlsSetup: TlsSetup
 ) {
 
     private val log = LoggerFactory.getLogger(GrpcServer::class.java)
@@ -42,54 +44,17 @@ open class GrpcServer(
 
     @PostConstruct
     fun start() {
-        log.info("Starting GRPC Server...")
+        log.info("Starting gRPC Server...")
         log.debug("Running with DEBUG LOGGING")
-        val port = env.getProperty("port", "2449").toInt()
-        log.info("Listening Native on 0.0.0.0:$port")
-        val serverBuilder = NettyServerBuilder.forPort(port)
-        rpcs.forEach {
-            serverBuilder.addService(it)
+        log.info("Listening Native gRPC on ${mainConfig.host}:${mainConfig.port}")
+        val serverBuilder = NettyServerBuilder
+                .forAddress(InetSocketAddress(mainConfig.host, mainConfig.port))
+        tlsSetup.setupServer("Native gRPC", mainConfig.tls, true)?.let {
+            serverBuilder.sslContext(it)
         }
 
-        val mustBeSecure = env.getProperty("tls.enabled", "") == "true"
-        val tlsDisabled = env.getProperty("tls.enabled", "") == "false"
-        var hasServerCertificate = true
-        if (!tlsDisabled) {
-            if (StringUtils.isEmpty(env.getProperty("tls.server.certificate"))) {
-                if (mustBeSecure) {
-                    log.warn("tls.server.certificate property is not set (path to server TLS certificate)")
-                    System.exit(1)
-                }
-                hasServerCertificate = false
-            }
-            if (StringUtils.isEmpty(env.getProperty("tls.server.key"))) {
-                if (mustBeSecure) {
-                    log.warn("tls.server.key property is not set (path to server TLS certificate key)")
-                    System.exit(1)
-                }
-                hasServerCertificate = false
-            }
-        }
-        if (mustBeSecure || (!tlsDisabled && hasServerCertificate)) {
-            log.info("Using TLS")
-            val sslContextBuilder = GrpcSslContexts.forServer(
-                    fileResolver.resolve(env.getProperty("tls.server.certificate")!!),
-                    fileResolver.resolve(env.getProperty("tls.server.key")!!)
-            )
-            if (StringUtils.isNotEmpty(env.getProperty("tls.client.ca"))) {
-                log.info("Using TLS for client authentication")
-                sslContextBuilder.trustManager(
-                        fileResolver.resolve(env.getProperty("tls.client.ca")!!)
-                )
-                if (env.getProperty("tls.client.require", "true") == "true") {
-                    sslContextBuilder.clientAuth(ClientAuth.REQUIRE)
-                }
-            } else {
-                log.warn("Trust all clients")
-            }
-            serverBuilder.sslContext(sslContextBuilder.build())
-        } else {
-            log.warn("Using insecure transport")
+        rpcs.forEach {
+            serverBuilder.addService(it)
         }
 
         val server = serverBuilder.build()

src/main/kotlin/io/emeraldpay/dshackle/ProxyStarter.kt

@@ -15,6 +15,7 @@
  */
 package io.emeraldpay.dshackle
 
+import io.emeraldpay.dshackle.config.MainConfig
 import io.emeraldpay.dshackle.config.ProxyConfig
 import io.emeraldpay.dshackle.proxy.ProxyServer
 import io.emeraldpay.dshackle.proxy.ReadRpcJson
@@ -31,7 +32,7 @@ import javax.annotation.PostConstruct
  */
 @Service
 class ProxyStarter(
-        @Autowired private val env: Environment,
+        @Autowired private val mainConfig: MainConfig,
         @Autowired private val readRpcJson: ReadRpcJson,
         @Autowired private val writeRpcJson: WriteRpcJson,
         @Autowired private val nativeCall: NativeCall,
@@ -44,7 +45,7 @@ class ProxyStarter(
 
     @PostConstruct
     fun start() {
-        val config = env.getProperty(ProxyConfig.CONFIG_ID, ProxyConfig::class.java)
+        val config = mainConfig.proxy
         if (config == null) {
             log.debug("Proxy server is not configured")
             return

src/main/kotlin/io/emeraldpay/dshackle/Starter.kt

@@ -21,6 +21,7 @@ import org.springframework.boot.SpringApplication
 import org.springframework.boot.autoconfigure.SpringBootApplication
 import org.springframework.context.annotation.Import
 import org.springframework.core.io.ClassPathResource
+import org.springframework.core.io.support.ResourcePropertySource
 
 @SpringBootApplication(scanBasePackages = [ "io.emeraldpay.dshackle" ])
 @Import(Config::class)
@@ -30,7 +31,7 @@ private val log = LoggerFactory.getLogger(Starter::class.java)
 
 fun main(args: Array<String>) {
     val app = SpringApplication(Starter::class.java)
-    app.setEnvironment(DshackleEnvironment())
+    app.setDefaultProperties(ResourcePropertySource("version.properties").source)
     app.setBanner(ResourceBanner(ClassPathResource("banner.txt")))
     app.run(*args)
 }

src/main/kotlin/io/emeraldpay/dshackle/TlsSetup.kt

@@ -34,7 +34,11 @@ class TlsSetup(
         private val log = LoggerFactory.getLogger(TlsSetup::class.java)
     }
 
-    fun setupServer(category: String, config: AuthConfig.ServerTlsAuth): SslContext? {
+    fun setupServer(category: String, config: AuthConfig.ServerTlsAuth?, grpc: Boolean): SslContext? {
+        if (config == null) {
+            log.warn("Using insecure transport for $category")
+            return null
+        }
         val mustBeSecure = config.enabled != null && config.enabled!!
         val tlsDisabled = config.enabled != null && !config.enabled!!
         var hasServerCertificate = true
@@ -56,10 +60,17 @@ class TlsSetup(
         }
         if (mustBeSecure || (!tlsDisabled && hasServerCertificate)) {
             log.info("Using TLS for $category")
-            val sslContextBuilder = SslContextBuilder.forServer(
-                    fileResolver.resolve(config.certificate!!),
-                    fileResolver.resolve(config.key!!)
-            )
+            val sslContextBuilder = if (grpc) {
+                GrpcSslContexts.forServer(
+                        fileResolver.resolve(config.certificate!!),
+                        fileResolver.resolve(config.key!!)
+                )
+            } else {
+                SslContextBuilder.forServer(
+                        fileResolver.resolve(config.certificate!!),
+                        fileResolver.resolve(config.key!!)
+                )
+            }
             if (StringUtils.isNotEmpty(config.clientCa)) {
                 log.info("Using TLS for client authentication for $category")
                 sslContextBuilder.trustManager(

src/main/kotlin/io/emeraldpay/dshackle/YamlPropertySourceFactory.kt

@@ -1,43 +0,0 @@
-/**
- * Copyright (c) 2019 ETCDEV GmbH
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *        http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.emeraldpay.dshackle
-
-import org.apache.commons.lang3.StringUtils
-import org.springframework.beans.factory.config.YamlPropertiesFactoryBean
-import org.springframework.core.env.PropertiesPropertySource
-import org.springframework.core.env.PropertySource
-import org.springframework.core.io.Resource
-import org.springframework.core.io.support.EncodedResource
-import org.springframework.core.io.support.PropertySourceFactory
-import java.util.*
-
-class YamlPropertySourceFactory: PropertySourceFactory {
-
-    override fun createPropertySource(name: String?, resource: EncodedResource): PropertySource<*> {
-        val loadedProperties = this.loadYamlIntoProperties(resource.resource)
-
-        return PropertiesPropertySource(if (StringUtils.isNotBlank(name)) name else resource.resource.filename, loadedProperties)
-
-    }
-
-    private fun loadYamlIntoProperties(resource: Resource): Properties {
-        val factory = YamlPropertiesFactoryBean()
-        factory.setResources(resource)
-        factory.afterPropertiesSet()
-
-        return factory.getObject()
-    }
-}

src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt

@@ -1,6 +1,7 @@
 package io.emeraldpay.dshackle.cache
 
 import com.fasterxml.jackson.databind.ObjectMapper
+import io.emeraldpay.dshackle.config.CacheConfig
 import io.emeraldpay.dshackle.config.EnvVariables
 import io.emeraldpay.grpc.Chain
 import io.lettuce.core.RedisClient
@@ -20,7 +21,7 @@ import kotlin.collections.HashMap
 @Repository
 class CachesFactory(
         @Autowired private val objectMapper: ObjectMapper,
-        @Autowired private val env: Environment
+        @Autowired private val cacheConfig: CacheConfig
 ) {
 
     companion object {
@@ -33,24 +34,20 @@ class CachesFactory(
 
     @PostConstruct
     fun init() {
-        if (!env.getProperty("${CONFIG_PREFIX}.enabled", Boolean::class.java, false)) {
-            return
-        }
-        val address = env.getProperty("${CONFIG_PREFIX}.host", "127.0.0.1")
-        val port = env.getProperty("${CONFIG_PREFIX}.port", Int::class.java, 6379)
+        val redisConfig = cacheConfig.redis ?: return
 
         var uri = RedisURI.builder()
-                .withHost(address)
-                .withPort(port)
+                .withHost(redisConfig.host)
+                .withPort(redisConfig.port)
 
-        env.getProperty("${CONFIG_PREFIX}.db", Int::class.java)?.let { value ->
+        redisConfig.db?.let { value ->
             uri = uri.withDatabase(value)
         }
 
         //log URI _before_ adding a password, to avoid leaking it to the log
         log.info("Use Redis cache at: ${uri.build().toURI()}")
 
-        env.getProperty("${CONFIG_PREFIX}.password")?.let { value ->
+        redisConfig.password?.let { value ->
             uri = uri.withPassword(value)
         }
 

src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfigReader.kt

@@ -65,6 +65,9 @@ class AuthConfigReader : YamlConfigReader() {
             getValueAsBool(node, "enabled")?.let {
                 auth.enabled = it
             }
+            if (auth.enabled != null && !auth.enabled!!) {
+                return null
+            }
             getMapping(node, "server")?.let { node ->
                 auth.certificate = getValueAsString(node, "certificate")
                 auth.key = getValueAsString(node, "key")

src/main/kotlin/io/emeraldpay/dshackle/config/CacheConfig.kt

@@ -0,0 +1,13 @@
+package io.emeraldpay.dshackle.config
+
+class CacheConfig {
+
+    var redis: Redis? = null;
+
+    class Redis(
+            var host: String = "127.0.0.1",
+            var port: Int = 6379,
+            var db: Int? = 0,
+            var password: String? = null
+    )
+}

src/main/kotlin/io/emeraldpay/dshackle/config/CacheConfigReader.kt

@@ -0,0 +1,43 @@
+package io.emeraldpay.dshackle.config
+
+import org.slf4j.LoggerFactory
+import org.yaml.snakeyaml.nodes.MappingNode
+import java.io.InputStream
+
+class CacheConfigReader : YamlConfigReader(), ConfigReader<CacheConfig> {
+
+    companion object {
+        private val log = LoggerFactory.getLogger(CacheConfigReader::class.java)
+    }
+
+    fun read(input: InputStream): CacheConfig? {
+        val configNode = readNode(input)
+        return read(configNode)
+    }
+
+    override fun read(input: MappingNode?): CacheConfig? {
+        return getMapping(input, "cache")?.let { node ->
+            val config = CacheConfig()
+            getMapping(node, "redis")?.let { node ->
+                val redis = CacheConfig.Redis()
+                getValueAsString(node, "host")?.let {
+                    redis.host = it
+                }
+                getValueAsInt(node, "port")?.let {
+                    redis.port = it
+                }
+                getValueAsInt(node, "db")?.let {
+                    redis.db = it
+                }
+                getValueAsString(node, "password")?.let {
+                    redis.password = it
+                }
+                config.redis = redis
+            }
+            if (config.redis == null) {
+                return null
+            }
+            config
+        }
+    }
+}

src/main/kotlin/io/emeraldpay/dshackle/config/ConfigReader.kt

@@ -0,0 +1,9 @@
+package io.emeraldpay.dshackle.config
+
+import org.yaml.snakeyaml.nodes.MappingNode
+
+interface ConfigReader<T> {
+
+    fun read(input: MappingNode?): T?
+
+}

src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt

@@ -0,0 +1,11 @@
+package io.emeraldpay.dshackle.config
+
+class MainConfig {
+    var host = "127.0.0.1"
+    var port = 2449
+    var tls: AuthConfig.ServerTlsAuth? = null
+    var cache: CacheConfig? = null
+    var proxy: ProxyConfig? = null
+    var upstreams: UpstreamsConfig? = null
+
+}

src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt

@@ -0,0 +1,50 @@
+package io.emeraldpay.dshackle.config
+
+import io.emeraldpay.dshackle.FileResolver
+import org.slf4j.LoggerFactory
+import org.yaml.snakeyaml.nodes.MappingNode
+import java.io.InputStream
+
+class MainConfigReader(
+        fileResolver: FileResolver
+) : YamlConfigReader(), ConfigReader<MainConfig> {
+
+    companion object {
+        private val log = LoggerFactory.getLogger(MainConfigReader::class.java)
+    }
+
+    private val authConfigReader = AuthConfigReader()
+    private val proxyConfigReader = ProxyConfigReader()
+    private val upstreamsConfigReader = UpstreamsConfigReader(fileResolver)
+    private val cacheConfigReader = CacheConfigReader()
+
+    fun read(input: InputStream): MainConfig? {
+        val configNode = readNode(input)
+        return read(configNode)
+    }
+
+    override fun read(input: MappingNode?): MainConfig? {
+        val config = MainConfig()
+        getValueAsString(input, "host")?.let {
+            config.host = it
+        }
+        getValueAsInt(input, "port")?.let {
+            config.port = it
+        }
+
+        authConfigReader.readServerTls(input)?.let {
+            config.tls = it
+        }
+        proxyConfigReader.read(input)?.let {
+            config.proxy = it
+        }
+        upstreamsConfigReader.read(input)?.let {
+            config.upstreams = it
+        }
+        cacheConfigReader.read(input)?.let {
+            config.cache = it
+        }
+        return config
+    }
+
+}

src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt

@@ -26,7 +26,7 @@ import java.io.InputStreamReader
 /**
  * Read YAML config, part related to Proxy configuration
  */
-class ProxyConfigReader : YamlConfigReader() {
+class ProxyConfigReader : YamlConfigReader(), ConfigReader<ProxyConfig> {
 
     companion object {
         private val log = LoggerFactory.getLogger(ProxyConfigReader::class.java)
@@ -37,10 +37,14 @@ class ProxyConfigReader : YamlConfigReader() {
 
     fun read(input: InputStream): ProxyConfig? {
         val configNode = readNode(input)
-        return read(getMapping(configNode, "proxy"))
+        return read(configNode)
     }
 
-    fun read(input: MappingNode?): ProxyConfig? {
+    override fun read(input: MappingNode?): ProxyConfig? {
+        return readInternal(getMapping(input, "proxy"))
+    }
+
+    fun readInternal(input: MappingNode?): ProxyConfig? {
         if (input == null) {
             return null
         }

src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt

@@ -22,7 +22,6 @@ import kotlin.collections.ArrayList
 import kotlin.collections.HashMap
 
 class UpstreamsConfig {
-    var version: String? = null
     var defaultOptions: MutableList<DefaultOptions> = ArrayList<DefaultOptions>()
     var upstreams: MutableList<Upstream<*>> = ArrayList<Upstream<*>>()
 

src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt

@@ -15,29 +15,38 @@
  */
 package io.emeraldpay.dshackle.config
 
+import io.emeraldpay.dshackle.FileResolver
 import org.apache.commons.lang3.StringUtils
 import org.slf4j.LoggerFactory
-import org.yaml.snakeyaml.Yaml
 import org.yaml.snakeyaml.nodes.MappingNode
 import org.yaml.snakeyaml.nodes.ScalarNode
 import reactor.util.function.Tuples
 import java.io.InputStream
-import java.io.InputStreamReader
 import java.net.URI
 import java.time.Duration
 
-class UpstreamsConfigReader : YamlConfigReader() {
+class UpstreamsConfigReader(
+        private val fileResolver: FileResolver
+) : YamlConfigReader(), ConfigReader<UpstreamsConfig> {
 
     private val log = LoggerFactory.getLogger(UpstreamsConfigReader::class.java)
     private val authConfigReader = AuthConfigReader()
 
-    fun read(input: InputStream): UpstreamsConfig {
+    fun read(input: InputStream): UpstreamsConfig? {
         val configNode = readNode(input)
+        return readInternal(configNode)
+    }
 
+    override fun read(input: MappingNode?): UpstreamsConfig? {
+        return getMapping(input, "upstreams")?.let {
+            readInternal(it)
+        }
+    }
+
+    fun readInternal(input: MappingNode?): UpstreamsConfig? {
         val config = UpstreamsConfig()
-        config.version = getValueAsString(configNode, "version")
 
-        getList<MappingNode>(configNode, "defaultOptions")?.value?.forEach { opts ->
+        getList<MappingNode>(input, "defaults")?.value?.forEach { opts ->
             val defaultOptions = UpstreamsConfig.DefaultOptions()
             config.defaultOptions.add(defaultOptions)
             defaultOptions.chains = getListOfString(opts, "chains")
@@ -47,7 +56,32 @@ class UpstreamsConfigReader : YamlConfigReader() {
         }
 
         config.upstreams = ArrayList<UpstreamsConfig.Upstream<*>>()
-        getList<MappingNode>(configNode, "upstreams")?.value?.forEachIndexed { pos, upNode ->
+
+        getValueAsString(input, "include")?.let { path ->
+            fileResolver.resolve(path).let { file ->
+                if (file.exists() && file.isFile && file.canRead()) {
+                    read(file.inputStream())?.let {
+                        it.upstreams.forEach { upstream -> config.upstreams.add(upstream) }
+                    }
+                } else {
+                    log.warn("Failed to read config from $path")
+                }
+            }
+        }
+
+        getListOfString(input, "include")?.forEach { path ->
+            fileResolver.resolve(path).let { file ->
+                if (file.exists() && file.isFile && file.canRead()) {
+                    read(file.inputStream())?.let {
+                        it.upstreams.forEach { upstream -> config.upstreams.add(upstream) }
+                    }
+                } else {
+                    log.warn("Failed to read config from $path")
+                }
+            }
+        }
+
+        getList<MappingNode>(input, "upstreams")?.value?.forEachIndexed { pos, upNode ->
             val connNode = getMapping(upNode, "connection")
             if (hasAny(connNode, "ethereum")) {
                 val connConfigNode = getMapping(connNode, "ethereum")!!
@@ -121,10 +155,10 @@ class UpstreamsConfigReader : YamlConfigReader() {
 
     internal fun readUpstreamGrpc(upNode: MappingNode, upstream: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>) {
         if (hasAny(upNode, "labels")) {
-            log.warn("Labels are not applied to gRPC upstream")
+            log.warn("Labels should be not applied to gRPC upstream")
         }
         if (hasAny(upNode, "chain")) {
-            log.warn("Chain is not applied to gRPC upstream")
+            log.warn("Chain should be not applied to gRPC upstream")
         }
     }
 

src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt

@@ -24,7 +24,7 @@ import org.yaml.snakeyaml.nodes.ScalarNode
 import java.io.InputStream
 import java.io.InputStreamReader
 
-open class YamlConfigReader {
+abstract class YamlConfigReader {
     private val envVariables = EnvVariables()
 
     fun readNode(input: String): MappingNode {

src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyServer.kt

@@ -59,10 +59,8 @@ class ProxyServer(
                 .host(config.host)
                 .port(config.port)
 
-        config.tls?.let { tls ->
-            tlsSetup.setupServer("proxy", tls)?.let { sslContext ->
-                serverBuilder = serverBuilder.secure { secure -> secure.sslContext(sslContext) }
-            }
+        tlsSetup.setupServer("proxy", config.tls, false)?.let { sslContext ->
+            serverBuilder = serverBuilder.secure { secure -> secure.sslContext(sslContext) }
         }
 
         val server: DisposableServer = serverBuilder

src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt

@@ -40,10 +40,10 @@ import kotlin.system.exitProcess
 
 @Repository
 open class ConfiguredUpstreams(
-        @Autowired val env: Environment,
         @Autowired private val objectMapper: ObjectMapper,
         @Autowired private val currentUpstreams: CurrentUpstreams,
-        @Autowired private val fileResolver: FileResolver
+        @Autowired private val fileResolver: FileResolver,
+        @Autowired private val config: UpstreamsConfig
 ) {
 
     private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java)
@@ -59,7 +59,6 @@ open class ConfiguredUpstreams(
 
     @PostConstruct
     fun start() {
-        val config = readConfig()
         val defaultOptions = buildDefaultOptions(config)
         config.upstreams.forEach { up ->
 
@@ -79,23 +78,6 @@ open class ConfiguredUpstreams(
         }
     }
 
-    private fun readConfig(): UpstreamsConfig {
-        val path = env.getProperty("upstreams.config")
-        if (StringUtils.isEmpty(path)) {
-            log.error("Path to upstreams is not set (upstreams.config)")
-            exitProcess(1)
-        }
-        val upstreamConfig = fileResolver.resolve(path!!).normalize()
-        val ok = upstreamConfig.exists() && upstreamConfig.isFile
-        if (!ok) {
-            log.error("Unable to setup upstreams from ${upstreamConfig.path}")
-            exitProcess(1)
-        }
-        log.info("Read upstream configuration from ${upstreamConfig.path}")
-        val reader = UpstreamsConfigReader()
-        return reader.read(upstreamConfig.inputStream())
-    }
-
     private fun buildDefaultOptions(config: UpstreamsConfig): HashMap<Chain, UpstreamsConfig.Options> {
         val defaultOptions = HashMap<Chain, UpstreamsConfig.Options>()
         config.defaultOptions.forEach { defaultsConfig ->

src/test/groovy/io/emeraldpay/dshackle/TlsSetupSpec.groovy

@@ -31,7 +31,7 @@ class TlsSetupSpec extends Specification {
                 enabled: false
         )
         when:
-        def act = tlsSetup.setupServer("test", config)
+        def act = tlsSetup.setupServer("test", config, false)
         then:
         act == null
     }
@@ -44,7 +44,7 @@ class TlsSetupSpec extends Specification {
                 key: "127.0.0.1.p8.key"
         )
         when:
-        def act = tlsSetup.setupServer("test", config)
+        def act = tlsSetup.setupServer("test", config, false)
         then:
         act != null
         act.server
@@ -67,7 +67,7 @@ class TlsSetupSpec extends Specification {
                 clientCa: "ca.myhost.dev.crt"
         )
         when:
-        def act = tlsSetup.setupServer("test", config)
+        def act = tlsSetup.setupServer("test", config, false)
         then:
         act != null
         act.server
@@ -87,7 +87,7 @@ class TlsSetupSpec extends Specification {
                 key: "127.0.0.1.p8.key",
         )
         when:
-        tlsSetup.setupServer("test", config)
+        tlsSetup.setupServer("test", config, false)
         then:
         def t = thrown(IllegalArgumentException)
         t.message == "Certificate not set"
@@ -100,7 +100,7 @@ class TlsSetupSpec extends Specification {
                 certificate: "127.0.0.1.crt"
         )
         when:
-        tlsSetup.setupServer("test", config)
+        tlsSetup.setupServer("test", config, false)
         then:
         def t = thrown(IllegalArgumentException)
         t.message == "Certificate Key not set"
@@ -114,7 +114,7 @@ class TlsSetupSpec extends Specification {
                 key: "127.0.0.1.p8.key",
         )
         when:
-        tlsSetup.setupServer("test", config)
+        tlsSetup.setupServer("test", config, false)
         then:
         def t = thrown(IllegalArgumentException)
     }
@@ -127,7 +127,7 @@ class TlsSetupSpec extends Specification {
                 key: "none.p8.key",
         )
         when:
-        tlsSetup.setupServer("test", config)
+        tlsSetup.setupServer("test", config, false)
         then:
         def t = thrown(IllegalArgumentException)
     }
@@ -140,7 +140,7 @@ class TlsSetupSpec extends Specification {
                 key: "127.0.0.1.key",
         )
         when:
-        tlsSetup.setupServer("test", config)
+        tlsSetup.setupServer("test", config, false)
         then:
         def t = thrown(IllegalArgumentException)
     }
@@ -154,7 +154,7 @@ class TlsSetupSpec extends Specification {
                 clientRequire: true
         )
         when:
-        tlsSetup.setupServer("test", config)
+        tlsSetup.setupServer("test", config, false)
         then:
         def t = thrown(IllegalArgumentException)
     }
@@ -169,7 +169,7 @@ class TlsSetupSpec extends Specification {
                 clientCa: "none.crt"
         )
         when:
-        tlsSetup.setupServer("test", config)
+        tlsSetup.setupServer("test", config, false)
         then:
         def t = thrown(IllegalArgumentException)
     }
@@ -184,7 +184,7 @@ class TlsSetupSpec extends Specification {
                 clientCa: "ca.myhost.dev.key"
         )
         when:
-        tlsSetup.setupServer("test", config)
+        tlsSetup.setupServer("test", config, false)
         then:
         def t = thrown(IllegalArgumentException)
     }

src/test/groovy/io/emeraldpay/dshackle/config/CacheConfigReaderSpec.groovy

@@ -0,0 +1,24 @@
+package io.emeraldpay.dshackle.config
+
+import spock.lang.Specification
+
+class CacheConfigReaderSpec extends Specification {
+
+    CacheConfigReader reader = new CacheConfigReader()
+
+    def "Read full"() {
+        setup:
+        def config = this.class.getClassLoader().getResourceAsStream("cache-redis-full.yaml")
+        when:
+        def act = reader.read(config)
+
+        then:
+        act.redis != null
+        with(act.redis) {
+            host == "redis-master"
+            port == 1234
+            db == 5
+            password == "HelloWorld!1"
+        }
+    }
+}

src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy

@@ -0,0 +1,73 @@
+package io.emeraldpay.dshackle.config
+
+import io.emeraldpay.dshackle.FileResolver
+import io.emeraldpay.dshackle.test.TestingCommons
+import io.emeraldpay.grpc.Chain
+import spock.lang.Specification
+
+class MainConfigReaderSpec extends Specification {
+
+    MainConfigReader reader = new MainConfigReader(TestingCommons.fileResolver())
+
+    def "Read full config"() {
+        setup:
+        def config = this.class.getClassLoader().getResourceAsStream("dshackle-full.yaml")
+        when:
+        def act = reader.read(config)
+
+        then:
+        act != null
+        act.host == "192.168.1.101"
+        act.port == 2448
+        act.tls != null
+        with(act.tls) {
+            certificate == "/path/127.0.0.1.crt"
+            key == "/path/127.0.0.1.p8.key"
+            !clientRequire
+            clientCa == "/path/ca.dshackle.test.crt"
+        }
+        act.cache != null
+        with(act.cache) {
+            redis != null
+            redis.host == "redis-master"
+        }
+        act.proxy != null
+        with(act.proxy) {
+            port == 8082
+            tls != null
+            routes != null
+            routes.size() == 3
+            with(routes[0]) {
+                id == "eth"
+                blockchain == Chain.ETHEREUM
+            }
+            with(routes[1]) {
+                id == "etc"
+                blockchain == Chain.ETHEREUM_CLASSIC
+            }
+            with(routes[2]) {
+                id == "kovan"
+                blockchain == Chain.TESTNET_KOVAN
+            }
+        }
+        act.upstreams != null
+        with(act.upstreams) {
+            defaultOptions != null
+            defaultOptions.size() == 1
+            with(defaultOptions[0]) {
+                chains == ["ethereum"]
+                options.minPeers == 3
+            }
+            upstreams.size() == 3
+            with(upstreams[0]) {
+                id == "remote"
+            }
+            with(upstreams[1]) {
+                id == "local"
+            }
+            with(upstreams[2]) {
+                id == "infura"
+            }
+        }
+    }
+}

src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy

@@ -23,7 +23,7 @@ import spock.lang.Specification
 
 class UpstreamsConfigReaderSpec extends Specification {
 
-    UpstreamsConfigReader reader = new UpstreamsConfigReader()
+    UpstreamsConfigReader reader = new UpstreamsConfigReader(TestingCommons.fileResolver())
 
     def "Parse standard config"() {
         setup:
@@ -32,7 +32,6 @@ class UpstreamsConfigReaderSpec extends Specification {
         def act = reader.read(config)
         then:
         act != null
-        act.version == "v1"
         with(act.defaultOptions) {
             size() == 1
             with(get(0)) {
@@ -81,7 +80,6 @@ class UpstreamsConfigReaderSpec extends Specification {
         def act = reader.read(config)
         then:
         act != null
-        act.version == "v1"
         act.upstreams.size() == 1
         with(act.upstreams.get(0)) {
             id == "remote"
@@ -141,7 +139,6 @@ class UpstreamsConfigReaderSpec extends Specification {
         def act = reader.read(config)
         then:
         act != null
-        act.version == "v1"
         with(act.defaultOptions) {
             size() == 0
         }

src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy

@@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.module.SimpleModule
 import io.emeraldpay.dshackle.FileResolver
 import io.emeraldpay.dshackle.cache.Caches
 import io.emeraldpay.dshackle.cache.CachesFactory
+import io.emeraldpay.dshackle.config.CacheConfig
 import io.emeraldpay.dshackle.upstream.AggregatedUpstream
 import io.emeraldpay.dshackle.upstream.ChainUpstreams
 import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods
@@ -78,7 +79,7 @@ class TestingCommons {
     }
 
     static CachesFactory emptyCaches() {
-        return new CachesFactory(objectMapper(), new StandardEnvironment())
+        return new CachesFactory(objectMapper(), new CacheConfig())
     }
 
     static FileResolver fileResolver() {

src/test/resources/cache-redis-full.yaml

@@ -0,0 +1,7 @@
+cache:
+  redis:
+    enabled: true
+    host: redis-master
+    port: 1234
+    db: 5
+    password: HelloWorld!1

src/test/resources/dshackle-full.yaml

@@ -0,0 +1,66 @@
+version: v1
+host: 192.168.1.101
+port: 2448
+
+tls:
+  enabled: true
+  server:
+    certificate: "/path/127.0.0.1.crt"
+    key: "/path/127.0.0.1.p8.key"
+  client:
+    require: false
+    ca: "/path/ca.dshackle.test.crt"
+
+cache:
+  redis:
+    enabled: true
+    host: redis-master
+
+proxy:
+  port: 8082
+  tls:
+    enabled: true
+    server:
+      certificate: "/path/second/127.0.0.1.crt"
+      key: "/path/second/127.0.0.1.p8.key"
+    client:
+      require: false
+      ca: "/path/second/ca.dshackle.test.crt"
+  routes:
+    - id: eth
+      blockchain: ethereum
+    - id: etc
+      blockchain: ethereum_classic
+    - id: kovan
+      blockchain: kovan
+
+upstreams:
+  defaults:
+    - chains:
+        - ethereum
+      options:
+        min-peers: 3
+  include:
+    - "upstreams-extra.yaml"
+  upstreams:
+    - id: local
+      chain: ethereum
+      connection:
+        ethereum:
+          rpc:
+            url: "http://localhost:8545"
+          ws:
+            url: "ws://localhost:8546"
+            origin: "http://localhost"
+            basic-auth:
+              username: 9c199ad8f281f20154fc258fe41a6814
+              password: 258fe4149c199ad8f2811a68f20154fc
+    - id: infura
+      chain: ethereum
+      connection:
+        ethereum:
+          rpc:
+            url: "https://mainnet.infura.io/v3/fa28c968191849c1aff541ad1d8511f2"
+            basic-auth:
+              username: 4fc258fe41a68149c199ad8f281f2015
+              password: 1a68f20154fc258fe4149c199ad8f281

src/test/resources/upstreams-basic.yaml

@@ -1,6 +1,6 @@
 version: v1
 
-defaultOptions:
+defaults:
   - chains:
       - ethereum
     options:

src/test/resources/upstreams-extra.yaml

@@ -0,0 +1,9 @@
+upstreams:
+  - id: remote
+    connection:
+      grpc:
+        host: "10.2.0.15"
+        tls:
+          ca: /etc/ca.myservice.com.crt
+          certificate: /etc/client1.myservice.com.crt
+          key: /etc/client1.myservice.com.key