Chains config refactoring (#311)

Generate code from chains config

foundation/src/main/kotlin/io/emeraldpay/dshackle/foundation/ChainOptions.kt

@@ -0,0 +1,90 @@
+package io.emeraldpay.dshackle.foundation
+
+import java.time.Duration
+
+class ChainOptions {
+    data class Options(
+        val disableUpstreamValidation: Boolean,
+        val disableValidation: Boolean,
+        val validationInterval: Int,
+        val timeout: Duration,
+        val providesBalance: Boolean?,
+        val validatePeers: Boolean,
+        val minPeers: Int,
+        val validateSyncing: Boolean,
+        val validateCallLimit: Boolean,
+        val validateChain: Boolean,
+    )
+
+    open class DefaultOptions : PartialOptions() {
+        var chains: List<String>? = null
+        var options: PartialOptions? = null
+    }
+
+    open class PartialOptions {
+        companion object {
+            @JvmStatic
+            fun getDefaults(): PartialOptions {
+                val options = PartialOptions()
+                options.minPeers = 1
+                return options
+            }
+        }
+
+        var disableValidation: Boolean? = null
+        var disableUpstreamValidation: Boolean? = null
+        var validationInterval: Int? = null
+            set(value) {
+                require(value == null || value > 0) {
+                    "validation-interval must be a positive number: $value"
+                }
+                field = value
+            }
+        var timeout: Duration? = null
+        var providesBalance: Boolean? = null
+        var validatePeers: Boolean? = null
+        var validateCalllimit: Boolean? = null
+        var minPeers: Int? = null
+            set(value) {
+                require(value == null || value >= 0) {
+                    "min-peers must be a positive number: $value"
+                }
+                field = value
+            }
+        var validateSyncing: Boolean? = null
+        var validateChain: Boolean? = null
+
+        fun merge(overwrites: PartialOptions?): PartialOptions {
+            if (overwrites == null) {
+                return this
+            }
+            val copy = PartialOptions()
+            copy.validatePeers = overwrites.validatePeers ?: this.validatePeers
+            copy.minPeers = overwrites.minPeers ?: this.minPeers
+            copy.disableValidation = overwrites.disableValidation ?: this.disableValidation
+            copy.validationInterval = overwrites.validationInterval ?: this.validationInterval
+            copy.providesBalance = overwrites.providesBalance ?: this.providesBalance
+            copy.validateSyncing = overwrites.validateSyncing ?: this.validateSyncing
+            copy.validateCalllimit = overwrites.validateCalllimit ?: this.validateCalllimit
+            copy.timeout = overwrites.timeout ?: this.timeout
+            copy.validateChain = overwrites.validateChain ?: this.validateChain
+            copy.disableUpstreamValidation =
+                overwrites.disableUpstreamValidation ?: this.disableUpstreamValidation
+            return copy
+        }
+
+        fun buildOptions(): Options =
+            Options(
+                this.disableUpstreamValidation ?: false,
+                this.disableValidation ?: false,
+                this.validationInterval ?: 30,
+                this.timeout ?: Duration.ofSeconds(60),
+                this.providesBalance,
+                this.validatePeers ?: true,
+                this.minPeers ?: 1,
+                this.validateSyncing ?: true,
+                this.validateCalllimit ?: true,
+                this.validateChain ?: true,
+            )
+    }
+}

foundation/src/main/kotlin/io/emeraldpay/dshackle/foundation/ChainOptionsReader.kt

@@ -0,0 +1,48 @@
+package io.emeraldpay.dshackle.foundation
+
+import org.yaml.snakeyaml.nodes.MappingNode
+import java.time.Duration
+
+class ChainOptionsReader : YamlConfigReader<ChainOptions.PartialOptions>() {
+    override fun read(upNode: MappingNode?): ChainOptions.PartialOptions? {
+        return if (hasAny(upNode, "options")) {
+            return getMapping(upNode, "options")?.let { values ->
+                readOptions(values)
+            }
+        } else {
+            null
+        }
+    }
+
+    fun readOptions(values: MappingNode): ChainOptions.PartialOptions {
+        val options = ChainOptions.PartialOptions()
+        getValueAsBool(values, "validate-peers")?.let {
+            options.validatePeers = it
+        }
+        getValueAsBool(values, "validate-syncing")?.let {
+            options.validateSyncing = it
+        }
+        getValueAsBool(values, "validate-call-limit")?.let {
+            options.validateCalllimit = it
+        }
+        getValueAsBool(values, "validate-chain")?.let {
+            options.validateChain = it
+        }
+        getValueAsInt(values, "min-peers")?.let {
+            options.minPeers = it
+        }
+        getValueAsInt(values, "timeout")?.let {
+            options.timeout = Duration.ofSeconds(it.toLong())
+        }
+        getValueAsBool(values, "disable-validation")?.let {
+            options.disableValidation = it
+        }
+        getValueAsInt(values, "validation-interval")?.let {
+            options.validationInterval = it
+        }
+        getValueAsBool(values, "balance")?.let {
+            options.providesBalance = it
+        }
+        return options
+    }
+}

foundation/src/main/kotlin/io/emeraldpay/dshackle/foundation/ConfigReader.kt

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) 2020 EmeraldPay, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.emeraldpay.dshackle.foundation
+
+import org.yaml.snakeyaml.nodes.MappingNode
+
+interface ConfigReader<T> {
+
+    fun read(input: MappingNode?): T?
+}

foundation/src/main/kotlin/io/emeraldpay/dshackle/foundation/EnvVariables.kt

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) 2020 EmeraldPay, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.emeraldpay.dshackle.foundation
+
+/**
+ * Update configuration value from environment variables. Format: ${ENV_VAR_NAME}
+ */
+class EnvVariables {
+
+    companion object {
+        private val envRegex = Regex("\\$\\{(\\w+?)}")
+    }
+
+    fun postProcess(value: String): String {
+        return envRegex.replace(value) { m ->
+            m.groups[1]?.let { g ->
+                System.getProperty(g.value) ?: System.getenv(g.value) ?: ""
+            } ?: ""
+        }
+    }
+}

foundation/src/main/kotlin/io/emeraldpay/dshackle/foundation/YamlConfigReader.kt

@@ -0,0 +1,187 @@
+/**
+ * Copyright (c) 2020 EmeraldPay, Inc
+ * Copyright (c) 2020 ETCDEV GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.emeraldpay.dshackle.foundation
+
+import org.yaml.snakeyaml.Yaml
+import org.yaml.snakeyaml.nodes.CollectionNode
+import org.yaml.snakeyaml.nodes.MappingNode
+import org.yaml.snakeyaml.nodes.Node
+import org.yaml.snakeyaml.nodes.NodeTuple
+import org.yaml.snakeyaml.nodes.ScalarNode
+import java.io.InputStream
+import java.io.InputStreamReader
+import java.util.Locale
+import kotlin.time.Duration
+import kotlin.time.toJavaDuration
+
+abstract class YamlConfigReader<T> : ConfigReader<T> {
+    private val envVariables = EnvVariables()
+
+    val filename = "dshackle.yaml"
+
+    fun read(input: InputStream): T? {
+        val configNode = readNode(input)
+        return read(configNode)
+    }
+    fun readNode(input: String): MappingNode {
+        return readNode(input.byteInputStream())
+    }
+
+    fun readNode(input: InputStream): MappingNode {
+        val yaml = Yaml()
+        return asMappingNode(yaml.compose(InputStreamReader(input)))
+    }
+
+    fun mergeMappingNode(a: MappingNode?, b: MappingNode?): MappingNode? = when {
+        a == null -> b
+        b == null -> a
+        else -> {
+            val mergedTuples = a.value.toMutableList()
+
+            mergedTuples.addAll(
+                b.value.filter { tupleB ->
+                    mergedTuples.none { it.keyNode.valueAsString() == tupleB.keyNode.valueAsString() } && (tupleB.valueNode is MappingNode || tupleB.valueNode is ScalarNode)
+                },
+            )
+
+            a.value.forEach { tupleA ->
+                b.value.find { it.keyNode.valueAsString() == tupleA.keyNode.valueAsString() }?.let { tupleB ->
+                    if (tupleA.valueNode is MappingNode && tupleB.valueNode is MappingNode) {
+                        mergedTuples[mergedTuples.indexOf(tupleA)] = NodeTuple(tupleA.keyNode, mergeMappingNode(tupleA.valueNode as MappingNode, tupleB.valueNode as MappingNode))
+                    } else if (tupleA.valueNode is ScalarNode && tupleB.valueNode is ScalarNode) {
+                        mergedTuples[mergedTuples.indexOf(tupleA)] = NodeTuple(tupleA.keyNode, tupleB.valueNode)
+                    }
+                }
+            }
+
+            MappingNode(a.tag, mergedTuples, a.flowStyle)
+        }
+    }
+
+    protected fun hasAny(mappingNode: MappingNode?, key: String): Boolean =
+        mappingNode?.let { node ->
+            node.value
+                .stream()
+                .anyMatch { it.keyNode.valueAsString() == key }
+        } ?: false
+
+    @Suppress("UNCHECKED_CAST")
+    private fun <T> getValue(mappingNode: MappingNode?, key: String, type: Class<T>): T? {
+        if (mappingNode == null) {
+            return null
+        }
+        return mappingNode.value
+            .stream()
+            .filter { type.isAssignableFrom(it.valueNode.javaClass) }
+            .filter { it.keyNode.valueAsString() == key }
+            .map { n -> n.valueNode as T }
+            .findFirst()
+            .orElse(null)
+    }
+
+    fun Node.valueAsString(): String? = if (this is ScalarNode) this.value else null
+
+    protected fun getMapping(mappingNode: MappingNode?, key: String): MappingNode? {
+        return getValue(mappingNode, key, MappingNode::class.java)
+    }
+
+    private fun getValue(mappingNode: MappingNode?, key: String): ScalarNode? {
+        return getValue(mappingNode, key, ScalarNode::class.java)
+    }
+
+    @Suppress("UNCHECKED_CAST")
+    protected fun <T> getList(mappingNode: MappingNode?, key: String): CollectionNode<T>? {
+        val value = getValue(mappingNode, key, CollectionNode::class.java) ?: return null
+        return value as CollectionNode<T>
+    }
+
+    protected fun getListOfString(mappingNode: MappingNode?, key: String): List<String>? {
+        return getList<ScalarNode>(mappingNode, key)?.value
+            ?.map { it.value }
+            ?.map(envVariables::postProcess)
+    }
+
+    protected fun getValueAsString(mappingNode: MappingNode?, key: String): String? {
+        return getValue(mappingNode, key)?.let {
+            return@let it.value
+        }?.let(envVariables::postProcess)
+    }
+
+    protected fun getValueAsDuration(mappingNode: MappingNode?, key: String): java.time.Duration? {
+        return getValue(mappingNode, key)?.let {
+            return@let if (it.isPlain) {
+                Duration.parse(it.value).toJavaDuration()
+            } else {
+                null
+            }
+        }
+    }
+
+    protected fun getValueAsInt(mappingNode: MappingNode?, key: String): Int? {
+        return getValue(mappingNode, key)?.let {
+            return@let if (it.isPlain) {
+                it.value.toIntOrNull()
+            } else {
+                null
+            }
+        }
+    }
+
+    protected fun getValueAsLong(mappingNode: MappingNode?, key: String): Long? {
+        return getValue(mappingNode, key)?.let {
+            return@let if (it.isPlain) {
+                it.value.toLongOrNull()
+            } else {
+                null
+            }
+        }
+    }
+
+    protected fun getValueAsBool(mappingNode: MappingNode?, key: String): Boolean? {
+        return getValue(mappingNode, key)?.let {
+            return@let if (it.isPlain) {
+                it.value.lowercase(Locale.getDefault()) == "true"
+            } else {
+                null
+            }
+        }
+    }
+
+    protected fun asMappingNode(node: Node): MappingNode {
+        return if (MappingNode::class.java.isAssignableFrom(node.javaClass)) {
+            node as MappingNode
+        } else {
+            throw IllegalArgumentException("Not a map")
+        }
+    }
+
+    fun getValueAsBytes(mappingNode: MappingNode?, key: String): Int? {
+        return getValueAsString(mappingNode, key)?.let(envVariables::postProcess)?.let {
+            val m = Regex("^(\\d+)(m|mb|k|kb|b)?$").find(it.lowercase().trim())
+                ?: throw IllegalArgumentException("Not a data size: $it. Example of correct values: '1024', '1kb', '5mb'")
+            val multiplier = m.groups[2]?.let {
+                when (it.value) {
+                    "k", "kb" -> 1024
+                    "m", "mb" -> 1024 * 1024
+                    else -> 1
+                }
+            } ?: 1
+            val base = m.groups[1]!!.value.toInt()
+            base * multiplier
+        }
+    }
+}

foundation/src/main/kotlin/org/drpc/chainsconfig/ChainsConfig.kt

@@ -0,0 +1,55 @@
+package io.emeraldpay.dshackle.config
+
+import io.emeraldpay.dshackle.foundation.ChainOptions
+import java.time.Duration
+
+data class ChainsConfig(private val chains: List<ChainConfig>) : Iterable<ChainsConfig.ChainConfig> {
+    private val chainMap: Map<String, ChainConfig> = chains.fold(emptyMap()) { acc, item ->
+        acc.plus(item.shortNames.map { Pair(it, item) })
+    }
+
+    override fun iterator(): Iterator<ChainConfig> {
+        return chains.iterator()
+    }
+    companion object {
+        @JvmStatic
+        fun default(): ChainsConfig = ChainsConfig(emptyList())
+    }
+
+    data class ChainConfig(
+        val expectedBlockTime: Duration,
+        val syncingLagSize: Int,
+        val laggingLagSize: Int,
+        val options: ChainOptions.PartialOptions,
+        val chainId: String,
+        val netVersion: Long,
+        val grpcId: Int,
+        val code: String,
+        val shortNames: List<String>,
+        val callLimitContract: String?,
+        val id: String,
+        val blockchain: String,
+    ) {
+        companion object {
+            @JvmStatic
+            fun default() = ChainConfig(
+                Duration.ofSeconds(12),
+                6,
+                1,
+                ChainOptions.PartialOptions(),
+                "0x0",
+                0,
+                0,
+                "UNKNOWN",
+                emptyList(),
+                null,
+                "undefined",
+                "undefined",
+            )
+        }
+    }
+
+    fun resolve(chain: String): ChainConfig {
+        return chainMap[chain] ?: ChainConfig.default()
+    }
+}

foundation/src/main/kotlin/org/drpc/chainsconfig/ChainsConfigReader.kt

@@ -0,0 +1,114 @@
+package io.emeraldpay.dshackle.config
+
+import io.emeraldpay.dshackle.foundation.ChainOptions
+import io.emeraldpay.dshackle.foundation.ChainOptionsReader
+import io.emeraldpay.dshackle.foundation.YamlConfigReader
+import org.yaml.snakeyaml.DumperOptions
+import org.yaml.snakeyaml.nodes.MappingNode
+import org.yaml.snakeyaml.nodes.NodeTuple
+import org.yaml.snakeyaml.nodes.ScalarNode
+import org.yaml.snakeyaml.nodes.Tag
+
+class ChainsConfigReader(
+    private val chainsOptionsReader: ChainOptionsReader,
+) : YamlConfigReader<ChainsConfig>() {
+
+    private val defaultConfig = this::class.java.getResourceAsStream("/chains.yaml")!!
+
+    private fun readChains(input: MappingNode?): Map<Int, Pair<String, MappingNode>> {
+        return getMapping(input, "chain-settings")?.let { config ->
+            val default = getMapping(config, "default")
+            getList<MappingNode>(config, "protocols")?.let { protocols ->
+                protocols.value.fold(emptyMap()) { acc, protocol ->
+                    val blockchain = getValueAsString(protocol, "id")
+                        ?: throw IllegalArgumentException("Blockchain id is not defined")
+                    val settings = mergeMappingNode(default, getMapping(protocol, "settings"))
+                    acc.plus(
+                        getList<MappingNode>(protocol, "chains")?.let { chains ->
+                            chains.value.map { chain ->
+                                val chainSettings = mergeMappingNode(settings, getMapping(chain, "settings"))
+                                val updatedChain = mergeMappingNode(
+                                    chain,
+                                    MappingNode(
+                                        chain.tag,
+                                        listOf(
+                                            NodeTuple(ScalarNode(Tag.STR, "settings", null, null, DumperOptions.ScalarStyle.LITERAL), chainSettings),
+                                            NodeTuple(
+                                                ScalarNode(Tag.STR, "blockchain", null, null, DumperOptions.ScalarStyle.LITERAL),
+                                                ScalarNode(Tag.STR, blockchain, null, null, DumperOptions.ScalarStyle.LITERAL),
+                                            ),
+                                        ),
+                                        chain.flowStyle,
+                                    ),
+                                )
+                                val grpcId = getValueAsInt(updatedChain, "grpcId")
+                                    ?: throw IllegalArgumentException("grpcId for chain is not defined")
+                                Pair(grpcId, Pair(blockchain, updatedChain!!))
+                            }
+                        } ?: listOf(),
+                    )
+                }
+            }
+        } ?: emptyMap()
+    }
+
+    private fun parseChain(blockchain: String, node: MappingNode): ChainsConfig.ChainConfig {
+        val id = getValueAsString(node, "id")
+            ?: throw IllegalArgumentException("undefined id for $blockchain")
+        val settings = getMapping(node, "settings") ?: throw IllegalArgumentException("undefined settings for $blockchain")
+        val lags = getMapping(settings, "lags")?.let { lagConfig ->
+            Pair(
+                getValueAsInt(lagConfig, "syncing")
+                    ?: throw IllegalArgumentException("undefined syncing for $blockchain"),
+                getValueAsInt(lagConfig, "lagging")
+                    ?: throw IllegalArgumentException("undefined lagging for $blockchain"),
+            )
+        } ?: throw IllegalArgumentException("undefined lags for $blockchain")
+        val expectedBlockTime = getValueAsDuration(settings, "expected-block-time")
+            ?: throw IllegalArgumentException("undefined expected block time")
+        val validateContract = getValueAsString(node, "call-validate-contract")
+        val options = chainsOptionsReader.read(settings) ?: ChainOptions.PartialOptions()
+        val chainId = getValueAsString(node, "chain-id")
+            ?: throw IllegalArgumentException("undefined chain id for $blockchain")
+        val code = getValueAsString(node, "code")
+            ?: throw IllegalArgumentException("undefined code for $blockchain")
+        val grpcId = getValueAsInt(node, "grpcId")
+            ?: throw IllegalArgumentException("undefined code for $blockchain")
+        val netVersion = getValueAsLong(node, "net-version") ?: chainId.drop(2).toLong(radix = 16)
+        val shortNames = getListOfString(node, "short-names")
+            ?: throw IllegalArgumentException("undefined shortnames for $blockchain")
+        return ChainsConfig.ChainConfig(
+            expectedBlockTime = expectedBlockTime,
+            syncingLagSize = lags.first,
+            laggingLagSize = lags.second,
+            options = options,
+            callLimitContract = validateContract,
+            chainId = chainId,
+            code = code,
+            grpcId = grpcId,
+            netVersion = netVersion,
+            shortNames = shortNames,
+            id = id,
+            blockchain = blockchain,
+        )
+    }
+
+    override fun read(input: MappingNode?): ChainsConfig {
+        val default = readChains(readNode(defaultConfig))
+        val current = readChains(input)
+        val chains = default.keys.toSet().plus(current.keys).map {
+            val defChain = default[it]
+            val curChain = current[it]
+            when {
+                curChain == null && defChain != null -> parseChain(defChain.first, defChain.second)
+                curChain != null && defChain == null -> parseChain(curChain.first, curChain.second)
+                curChain != null && defChain != null -> {
+                    val merged = mergeMappingNode(defChain.second, curChain.second)
+                    parseChain(defChain.first, merged!!)
+                }
+                else -> ChainsConfig.ChainConfig.default()
+            }
+        }
+        return ChainsConfig(chains)
+    }
+}