From 7fd27d581a357312df8da5c682230750d2e849fb Mon Sep 17 00:00:00 2001 From: Igor Artamonov Date: Sun, 8 Sep 2019 22:30:46 -0400 Subject: [PATCH] solution: paths relative to config, default is at /etc/dshackle --- README.adoc | 4 +- docs/02-quick-start.adoc | 2 +- docs/03-server-config.adoc | 12 ++-- .../kotlin/io/emeraldpay/dshackle/Config.kt | 29 +++++++- .../dshackle/DshackleEnvironment.kt | 67 +++++++++++++++++++ .../io/emeraldpay/dshackle/FileResolver.kt | 32 +++++++++ .../io/emeraldpay/dshackle/GrpcServer.kt | 10 +-- .../kotlin/io/emeraldpay/dshackle/Starter.kt | 8 +-- .../dshackle/upstream/ConfiguredUpstreams.kt | 23 +++---- .../dshackle/upstream/grpc/GrpcUpstreams.kt | 11 +-- .../dshackle/FileResolverSpec.groovy | 42 ++++++++++++ 11 files changed, 207 insertions(+), 33 deletions(-) create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/DshackleEnvironment.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/FileResolver.kt create mode 100644 src/test/groovy/io/emeraldpay/dshackle/FileResolverSpec.groovy diff --git a/README.adoc b/README.adoc index acfd0918..516ce751 100644 --- a/README.adoc +++ b/README.adoc @@ -72,7 +72,7 @@ upstreams: This configures: -- setups 2 upstreams, one for Ethereum Mainnet and another for Kovan Testnet (both upstreams are configured for Infura) +- setups 2 upstreams, one for Ethereum Mainnet and another for Kovan Testnet (both upstreams are configured to use Infura endpoint) - for Ethereum Mainnet it connects using JSON RPC and Websockets connections, for Kovan just JSON RPC is used - Infura authentication config is omitted for this demo - `${INFURA_USER}` will be provided through environment variable @@ -90,7 +90,7 @@ export INFURA_USER=... .Run Dshackle [source,bash] ---- -docker run -p 2449:2449 -v $(pwd):/config -w /config -e "INFURA_USER=$INFURA_USER" emeraldpay/dshackle +docker run -p 2449:2449 -v $(pwd):/etc/dshackle -e "INFURA_USER=$INFURA_USER" emeraldpay/dshackle ---- Now it listen on port 2449 at the localhost and can be connected from any gRPC compatible client. diff --git a/docs/02-quick-start.adoc b/docs/02-quick-start.adoc index 2e1a1cd9..3bda3eeb 100644 --- a/docs/02-quick-start.adoc +++ b/docs/02-quick-start.adoc @@ -71,7 +71,7 @@ export INFURA_USER=... .Run Dshackle [source,bash] ---- -docker run -p 2449:2449 -v $(pwd):/config -w /config -e "INFURA_USER=$INFURA_USER" emeraldpay/dshackle +docker run -p 2449:2449 -v $(pwd):/etc/dshackle -e "INFURA_USER=$INFURA_USER" emeraldpay/dshackle ---- .Connect and listen for new blocks on Ethereum Mainnet diff --git a/docs/03-server-config.adoc b/docs/03-server-config.adoc index 627d9997..d7c0b541 100644 --- a/docs/03-server-config.adoc +++ b/docs/03-server-config.adoc @@ -1,6 +1,10 @@ == Server Configuration +Dshackle server tries to load its configuration from `/etc/dshackle/dshackle.yaml`, if it can't find a file at that path +it tries to load file `dshackle.yaml` from current working directory. If none of them found server fails to run with an error. + [source,yaml] +.Example dshackle.yaml configuration: ---- version: v1 port: 2449 @@ -16,10 +20,10 @@ upstreams: config: "upstreams.yaml" ---- -Configures following: +It configures following: - server is listening on `0.0.0.0:2449` - TLS is enabled -- server certificate in located at `server.crt` with key for it at `server.p8.key` -- server requires a client authentication by TLS client certificate signed by `ca.crt` certificate -- upstreams configuration is configured in file `upstreams.yaml` +- server certificate is located at `server.crt` with the key for it at `server.p8.key` +- the server requires a client authentication by TLS client certificate signed by `ca.crt` certificate +- upstreams configuration is configured in the file `upstreams.yaml` diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt index 44a3d635..fe90a9b8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt @@ -19,22 +19,34 @@ import com.fasterxml.jackson.core.Version import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.module.SimpleModule +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.context.ApplicationContext import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Import +import org.springframework.core.env.Environment import org.springframework.scheduling.annotation.EnableAsync import org.springframework.scheduling.annotation.EnableScheduling import org.springframework.scheduling.annotation.Scheduled import reactor.core.scheduler.Scheduler import reactor.core.scheduler.Schedulers +import java.io.File +import java.lang.IllegalStateException import java.text.SimpleDateFormat import java.util.* import java.util.concurrent.Executors +import kotlin.system.exitProcess @Configuration @EnableScheduling @EnableAsync -open class Config { +open class Config( + @Autowired private val env: Environment +) { + + private val log = LoggerFactory.getLogger(Config::class.java) @Bean open fun objectMapper(): ObjectMapper { @@ -55,4 +67,19 @@ open class Config { return Schedulers.fromExecutorService(Executors.newFixedThreadPool(16)) } + @Bean + @Qualifier("configDir") + open fun configDir(): File { + val config = env.getProperty("configPath") ?: throw IllegalStateException("Config path is not set") + if (config.trim().isEmpty()) { + throw IllegalStateException("Config path is empty") + } + log.info("Use configuration from: $config") + return File(config).parentFile + } + + @Bean + open fun fileResolver(): FileResolver { + return FileResolver(configDir()) + } } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/DshackleEnvironment.kt b/src/main/kotlin/io/emeraldpay/dshackle/DshackleEnvironment.kt new file mode 100644 index 00000000..c8d6b200 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/DshackleEnvironment.kt @@ -0,0 +1,67 @@ +/** + * Copyright (c) 2019 ETCDEV GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle + +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.config.YamlPropertiesFactoryBean +import org.springframework.core.env.* +import org.springframework.core.io.FileSystemResource +import org.springframework.core.io.Resource +import java.io.File +import java.util.* + +const val DEFAULT_CONFIG = "/etc/dshackle/dshackle.yaml" +const val LOCAL_CONFIG = "./dshackle.yaml" + +open class DshackleEnvironment: StandardEnvironment() { + + companion object { + private val log = LoggerFactory.getLogger(DshackleEnvironment::class.java) + } + + + override fun customizePropertySources(propertySources: MutablePropertySources) { + super.customizePropertySources(propertySources) + propertySources.addLast(mainConfig()) + } + + open fun mainConfig(): PropertySource<*> { + var target = File(DEFAULT_CONFIG) + if (!isAcceptedConfig(target)) { + target = File(LOCAL_CONFIG) + if (!isAcceptedConfig(target)) { + log.error("Configuration is not found neither at $DEFAULT_CONFIG nor $LOCAL_CONFIG") + return PropertySource.named("mainConfig") + } + } + target = target.normalize() + log.info("Load configuration from: ${target.absolutePath}") + val loadedProperties = this.loadYaml(FileSystemResource(target)) + loadedProperties["configPath"] = target.absolutePath + return PropertiesPropertySource("mainConfig", loadedProperties) + } + + protected fun loadYaml(resource: Resource): Properties { + val factory = YamlPropertiesFactoryBean() + factory.setResources(resource) + factory.afterPropertiesSet() + return factory.getObject()!! + } + + protected fun isAcceptedConfig(target: File): Boolean { + return target.exists() && target.isFile + } +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/FileResolver.kt b/src/main/kotlin/io/emeraldpay/dshackle/FileResolver.kt new file mode 100644 index 00000000..3d47fe8a --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/FileResolver.kt @@ -0,0 +1,32 @@ +/** + * Copyright (c) 2019 ETCDEV GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle + +import java.io.File + +class FileResolver( + private val baseDir: File +) { + + fun resolve(path: String): File { + val direct = File(path) + if (direct.isAbsolute) { + return direct + } + return File(baseDir, path) + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt b/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt index f5c0346f..abea9539 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt @@ -25,7 +25,6 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.core.env.Environment import org.springframework.core.io.ResourceLoader import org.springframework.stereotype.Service -import java.io.File import javax.annotation.PostConstruct import javax.annotation.PreDestroy @@ -33,7 +32,8 @@ import javax.annotation.PreDestroy open class GrpcServer( @Autowired val rpcs: List, @Autowired val resourceLoader: ResourceLoader, - @Autowired val env: Environment + @Autowired val env: Environment, + @Autowired val fileResolver: FileResolver ) { private val log = LoggerFactory.getLogger(GrpcServer::class.java) @@ -73,13 +73,13 @@ open class GrpcServer( if (mustBeSecure || (!tlsDisabled && hasServerCertificate)) { log.info("Using TLS") val sslContextBuilder = GrpcSslContexts.forServer( - File(env.getProperty("tls.server.certificate")!!), - File(env.getProperty("tls.server.key")!!) + fileResolver.resolve(env.getProperty("tls.server.certificate")!!), + fileResolver.resolve(env.getProperty("tls.server.key")!!) ) if (StringUtils.isNotEmpty(env.getProperty("tls.client.ca"))) { log.info("Using TLS for client authentication") sslContextBuilder.trustManager( - File(env.getProperty("tls.client.ca")!!) + fileResolver.resolve(env.getProperty("tls.client.ca")!!) ) if (env.getProperty("tls.client.require", "true") == "true") { sslContextBuilder.clientAuth(ClientAuth.REQUIRE) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt b/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt index 96fdfbc6..525d6eaf 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt @@ -15,19 +15,19 @@ */ package io.emeraldpay.dshackle -import org.springframework.beans.factory.config.YamlPropertiesFactoryBean +import org.slf4j.LoggerFactory import org.springframework.boot.SpringApplication import org.springframework.boot.autoconfigure.SpringBootApplication -import org.springframework.boot.env.YamlPropertySourceLoader import org.springframework.context.annotation.Import -import org.springframework.context.annotation.PropertySource @SpringBootApplication(scanBasePackages = [ "io.emeraldpay.dshackle" ]) -@PropertySource("file:./dshackle.yaml", ignoreResourceNotFound = true, factory = YamlPropertySourceFactory::class) @Import(Config::class) open class Starter +private val log = LoggerFactory.getLogger(Starter::class.java) + fun main(args: Array) { val app = SpringApplication(Starter::class.java) + app.setEnvironment(DshackleEnvironment()) app.run(*args) } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ConfiguredUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ConfiguredUpstreams.kt index d0874dc5..5a65307e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ConfiguredUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ConfiguredUpstreams.kt @@ -16,6 +16,7 @@ package io.emeraldpay.dshackle.upstream import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.dshackle.FileResolver import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfigReader import io.emeraldpay.dshackle.upstream.ethereum.DirectEthereumApi @@ -29,22 +30,19 @@ import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.core.env.Environment -import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Repository -import reactor.core.publisher.Flux -import reactor.core.publisher.TopicProcessor -import java.io.File import java.net.URI import java.util.* -import java.util.concurrent.ConcurrentHashMap import javax.annotation.PostConstruct import kotlin.collections.HashMap +import kotlin.system.exitProcess @Repository open class ConfiguredUpstreams( @Autowired val env: Environment, @Autowired private val objectMapper: ObjectMapper, - @Autowired private val currentUpstreams: CurrentUpstreams + @Autowired private val currentUpstreams: CurrentUpstreams, + @Autowired private val fileResolver: FileResolver ) { private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java) @@ -83,13 +81,13 @@ open class ConfiguredUpstreams( val path = env.getProperty("upstreams.config") if (StringUtils.isEmpty(path)) { log.error("Path to upstreams is not set (upstreams.config)") - System.exit(1) + exitProcess(1) } - val upstreamConfig = File(path!!) + val upstreamConfig = fileResolver.resolve(path!!).normalize() val ok = upstreamConfig.exists() && upstreamConfig.isFile if (!ok) { log.error("Unable to setup upstreams from ${upstreamConfig.path}") - System.exit(1) + exitProcess(1) } log.info("Read upstream configuration from ${upstreamConfig.path}") val reader = UpstreamsConfigReader() @@ -139,7 +137,7 @@ open class ConfiguredUpstreams( } conn.rpc?.tls?.let { tls -> tls.ca?.let { ca -> - File(ca).inputStream().use { cert -> rpcTransport.setTrustedCertificate(cert) } + fileResolver.resolve(ca).inputStream().use { cert -> rpcTransport.setTrustedCertificate(cert) } } } val rpcClient = DefaultRpcClient(rpcTransport) @@ -181,9 +179,10 @@ open class ConfiguredUpstreams( val ds = GrpcUpstreams( config.id!!, endpoint.host!!, - endpoint.port ?: 443, + endpoint.port ?: 2449, objectMapper, - endpoint.auth + endpoint.auth, + fileResolver ) log.info("Using ALL CHAINS (gRPC) upstream, at ${endpoint.host}:${endpoint.port}") ds.start() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt index 01ad94da..87a1b600 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt @@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.upstream.grpc import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.ReactorBlockchainGrpc +import io.emeraldpay.dshackle.FileResolver import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.upstream.UpstreamChange import io.emeraldpay.grpc.Chain @@ -29,7 +30,6 @@ import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory import reactor.core.Disposable import reactor.core.publisher.Flux -import java.io.File import java.time.Duration import java.util.* import java.util.concurrent.Executors @@ -42,7 +42,8 @@ class GrpcUpstreams( private val host: String, private val port: Int, private val objectMapper: ObjectMapper, - private val auth: UpstreamsConfig.TlsAuth? = null + private val auth: UpstreamsConfig.TlsAuth? = null, + private val fileResolver: FileResolver ) { private val log = LoggerFactory.getLogger(GrpcUpstreams::class.java) @@ -127,9 +128,11 @@ class GrpcUpstreams( internal fun withTls(auth: UpstreamsConfig.TlsAuth): SslContext { val sslContext = SslContextBuilder.forClient() .clientAuth(ClientAuth.REQUIRE) - sslContext.trustManager(File(auth.ca!!).inputStream()) + sslContext.trustManager(fileResolver.resolve(auth.ca!!).inputStream()) if (StringUtils.isNotEmpty(auth.key) && StringUtils.isNoneEmpty(auth.certificate)) { - sslContext.keyManager(File(auth.certificate!!).inputStream(), File(auth.key!!).inputStream()) + sslContext.keyManager( + fileResolver.resolve(auth.certificate!!).inputStream(), + fileResolver.resolve(auth.key!!).inputStream()) } else { log.warn("Connect to remote using only CA certificate") } diff --git a/src/test/groovy/io/emeraldpay/dshackle/FileResolverSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/FileResolverSpec.groovy new file mode 100644 index 00000000..4c0595cd --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/FileResolverSpec.groovy @@ -0,0 +1,42 @@ +/** + * Copyright (c) 2019 ETCDEV GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle + +import spock.lang.Specification + +class FileResolverSpec extends Specification { + + def "Leaves absolute path"() { + setup: + def resolver = new FileResolver(new File("/tmp")) + expect: + resolver.resolve(path).absolutePath == path + where: + path << ["/test.txt", "/etc/dshackle/dshackle.yaml"] + } + + def "Start from base dir for related path"() { + setup: + def resolver = new FileResolver(new File("/tmp")) + expect: + resolver.resolve(path).canonicalPath == new File(fullPath).canonicalPath + where: + path | fullPath + "test.txt" | "/tmp/test.txt" + "./test.txt" | "/tmp/test.txt" + "./tls/ca.crt" | "/tmp/tls/ca.crt" + } +}