StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

config merge

Browse Source
This commit is contained in:
a10zn8
2023-02-10 21:33:58 +04:00
parent ad0d0b2b57
commit 2809e87d33
10 changed files with 169 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
docs/04-upstream-config.adoc
View File

@@ -32,8 +32,6 @@ If you request balance for an address that is not indexed then it returns 0 bala
- To track all transactions you need to setup index for transactions, which is disabled by default.
Run it with `-reindex` option, or set `txindex=1` in the config.
===
=== Example Configuration
.upstreams.yaml
@@ -317,3 +315,33 @@ For JSON RPC and Websockets a Basic Authentication can be used:
- `username` - username
- `password` - password
=== Chains specific configuration
We can use chain settings to specify chain specific behavior, for example rules for dshackle to work with upstream statuses
.chains.yaml
[source,yaml]
----
chain-settings:
default:
lags:
syncing: 6
lagging: 1
chains:
- id: eth
lags:
syncing: 6
lagging: 1
- id: polygon
lags:
syncing: 20
lagging: 10
----
Options
[cols="2,5a"]
|===
| Option | Description
| `lags.syncing` | the size of the lag after which the upstream is determined to be syncing
| `lags.lagging` | the size of the lag after which the upstream is determined to be lagging
|===

30
src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt
View File

@@ -3,7 +3,7 @@ package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Chain
import java.lang.IllegalStateException
class ChainsConfig(private val chains: Map<Chain, RawChainConfig>?, val currentDefault: RawChainConfig?) {
data class ChainsConfig(private val chains: Map<Chain, RawChainConfig>, val currentDefault: RawChainConfig?) {
companion object {
@JvmStatic
fun default(): ChainsConfig = ChainsConfig(emptyMap(), RawChainConfig.default())
@@ -25,7 +25,7 @@ class ChainsConfig(private val chains: Map<Chain, RawChainConfig>?, val currentD
fun resolve(chain: Chain): ChainConfig {
val default = currentDefault ?: panic()
val raw = chains?.get(chain) ?: default
val raw = chains[chain] ?: default
return ChainConfig(
laggingLagSize = raw.laggingLagSize ?: default.laggingLagSize ?: panic(),
@@ -33,5 +33,31 @@ class ChainsConfig(private val chains: Map<Chain, RawChainConfig>?, val currentD
)
}
fun patch(patch: ChainsConfig) = ChainsConfig(
merge(this.chains, patch.chains),
merge(this.currentDefault!!, patch.currentDefault)
)
private fun merge(
current: RawChainConfig,
patch: RawChainConfig?
) = RawChainConfig(
syncingLagSize = patch?.syncingLagSize ?: current.syncingLagSize,
laggingLagSize = patch?.laggingLagSize ?: current.laggingLagSize
)
private fun merge(
current: Map<Chain, RawChainConfig>,
patch: Map<Chain, RawChainConfig>
): Map<Chain, RawChainConfig> {
val currentMut = current.toMutableMap()
for (k in patch) {
currentMut.merge(k.key, k.value) { v1, v2 -> merge(v1, v2) }
}
return currentMut.toMap()
}
fun panic(): Nothing = throw IllegalStateException("Chains settings state is illegal - default config is null")
}

18
src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt
View File

@@ -3,10 +3,24 @@ package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Global
import org.yaml.snakeyaml.nodes.CollectionNode
import org.yaml.snakeyaml.nodes.MappingNode
import java.io.InputStream
class ChainsConfigReader : YamlConfigReader<ChainsConfig>() {
private val defaultConfig = this::class.java.getResourceAsStream("/chains.yaml")!!
override fun read(input: MappingNode?): ChainsConfig {
val default = readInternal(defaultConfig)
val current = readInternal(input)
return default.patch(current)
}
fun readInternal(input: InputStream): ChainsConfig {
val configNode = readNode(input)
return readInternal(configNode)
}
fun readInternal(input: MappingNode?): ChainsConfig {
return getMapping(input, "chain-settings")?.let {
val chains = getList<MappingNode>(it, "chains")?.let {
@@ -37,8 +51,8 @@ class ChainsConfigReader : YamlConfigReader<ChainsConfig>() {
private fun readChains(node: CollectionNode<MappingNode>): List<Pair<String, ChainsConfig.RawChainConfig>> {
return node.value.mapNotNull {
val key = getValueAsString(it, "name")
?: throw InvalidConfigYamlException(filename, it.startMark, "chain name required")
val key = getValueAsString(it, "id")
?: throw InvalidConfigYamlException(filename, it.startMark, "chain id required")
val value = readChain(it)
if (value != null) {
return@mapNotNull key to value

7
src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt
View File

@@ -16,17 +16,12 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.FileResolver
import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.nodes.MappingNode
class MainConfigReader(
fileResolver: FileResolver
) : YamlConfigReader<MainConfig>() {
companion object {
private val log = LoggerFactory.getLogger(MainConfigReader::class.java)
}
private val authConfigReader = AuthConfigReader()
private val proxyConfigReader = ProxyConfigReader()
private val upstreamsConfigReader = UpstreamsConfigReader(fileResolver)
@@ -39,7 +34,7 @@ class MainConfigReader(
private val compressionConfigReader = CompressionConfigReader()
private val chainsConfigReader = ChainsConfigReader()
override fun read(input: MappingNode?): MainConfig? {
override fun read(input: MappingNode?): MainConfig {
val config = MainConfig()
getValueAsString(input, "host")?.let {
config.host = it

9
src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt
View File

@@ -20,6 +20,7 @@ import io.emeraldpay.dshackle.FileResolver
import org.apache.commons.lang3.StringUtils
import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.nodes.MappingNode
import java.io.InputStream
import java.net.URI
import java.time.Duration
import java.util.Locale
@@ -38,6 +39,10 @@ class UpstreamsConfigReader(
}
}
fun readInternal(input: InputStream): UpstreamsConfig? {
val configNode = readNode(input)
return readInternal(configNode)
}
fun readInternal(input: MappingNode?): UpstreamsConfig {
val config = UpstreamsConfig()
@@ -55,7 +60,7 @@ class UpstreamsConfigReader(
getValueAsString(input, "include")?.let { path ->
fileResolver.resolve(path).let { file ->
if (file.exists() && file.isFile && file.canRead()) {
read(file.inputStream())?.let {
readInternal(file.inputStream())?.let {
it.upstreams.forEach { upstream -> config.upstreams.add(upstream) }
}
} else {
@@ -67,7 +72,7 @@ class UpstreamsConfigReader(
getListOfString(input, "include")?.forEach { path ->
fileResolver.resolve(path).let { file ->
if (file.exists() && file.isFile && file.canRead()) {
read(file.inputStream())?.let {
readInternal(file.inputStream())?.let {
it.upstreams.forEach { upstream -> config.upstreams.add(upstream) }
}
} else {

2
src/main/resources/chains.yaml
View File

@@ -10,7 +10,7 @@ chain-settings:
lags:
syncing: 6
lagging: 1
- name: polygon
- id: polygon
lags:
syncing: 20
lagging: 10

23
src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy
View File

@@ -28,13 +28,24 @@ class ChainsConfigReaderSpec extends Specification {
def stream = this.class.getClassLoader().getResourceAsStream("configs/chains-basic.yaml")
when:
def config = reader.read(stream)
def act = config.resolve(Chain.ETHEREUM)
def act2 = config.resolve(Chain.POLYGON)
def eth = config.resolve(Chain.ETHEREUM)
def pol = config.resolve(Chain.POLYGON)
def opt = config.resolve(Chain.OPTIMISM)
def sep = config.resolve(Chain.TESTNET_SEPOLIA)
then:
act.laggingLagSize == 5
act.syncingLagSize == 10
eth.laggingLagSize == 1
eth.syncingLagSize == 6
act2.laggingLagSize == 1
act2.syncingLagSize == 6
pol.laggingLagSize == 10
pol.syncingLagSize == 20
opt.laggingLagSize == 3
opt.syncingLagSize == 6
sep.laggingLagSize == 1
sep.syncingLagSize == 10
eth.laggingLagSize == 1
eth.syncingLagSize == 6
}
}

38
src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy
View File

@@ -27,7 +27,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-basic.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
with(act.defaultOptions) {
@@ -75,7 +75,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-ws-only.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
act.upstreams.size() == 1
@@ -100,7 +100,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-ws-full.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
act.upstreams.size() == 1
@@ -127,7 +127,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-bitcoin.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
with(act.defaultOptions) {
@@ -155,7 +155,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("upstreams-ethereum-pos.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
act.upstreams.size() == 1
@@ -175,7 +175,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-bitcoin-esplora.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
with(act.defaultOptions) {
@@ -204,7 +204,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-ds.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
act.upstreams.size() == 1
@@ -227,7 +227,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-labels.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
act.upstreams.size() == 2
@@ -248,7 +248,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-options.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
act.upstreams.size() == 2
@@ -264,7 +264,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-no-defaults.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
with(act.defaultOptions) {
@@ -287,7 +287,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-methods.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
with(act.upstreams.get(0)) {
@@ -307,7 +307,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-methods-quorum.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
with(act.upstreams.get(0)) {
@@ -330,7 +330,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-no-id.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
act.upstreams.size() == 1
@@ -357,7 +357,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-basic.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
act.upstreams.size() == 2
@@ -369,7 +369,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-roles.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
act.upstreams.size() == 2
@@ -381,7 +381,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-roles-2.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
act.upstreams.size() == 3
@@ -394,7 +394,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-roles-invalid.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
act.upstreams.size() == 2
@@ -406,7 +406,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("upstreams-node-id.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
act.upstreams.size() == 2
@@ -424,7 +424,7 @@ class UpstreamsConfigReaderSpec extends Specification {
setup:
def config = this.class.getClassLoader().getResourceAsStream("upstreams-method-groups.yaml")
when:
def act = reader.read(config)
def act = reader.readInternal(config)
then:
act != null
act.upstreams.size() == 1

44
src/test/kotlin/io/emeraldpay/dshackle/config/ChainsConfigTest.kt Normal file
View File

@@ -0,0 +1,44 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Chain
import org.junit.jupiter.api.Assertions.*
import org.junit.jupiter.api.Test
internal class ChainsConfigTest {
@Test
fun patch() {
val orig = ChainsConfig(
mapOf(
Chain.BITCOIN to ChainsConfig.RawChainConfig(0, 0),
Chain.ETHEREUM to ChainsConfig.RawChainConfig(1, 2),
Chain.POLYGON to ChainsConfig.RawChainConfig(3, 4)
),
ChainsConfig.RawChainConfig(1, 2)
)
val patch = ChainsConfig(
mapOf(
Chain.BITCOIN to ChainsConfig.RawChainConfig(null, 10000),
Chain.POLYGON to ChainsConfig.RawChainConfig(10, 11),
Chain.ARBITRUM to ChainsConfig.RawChainConfig(999, 999)
),
ChainsConfig.RawChainConfig(100, null)
)
val res = orig.patch(patch)
assertEquals(
ChainsConfig(
mapOf(
Chain.BITCOIN to ChainsConfig.RawChainConfig(0, 10000),
Chain.ETHEREUM to ChainsConfig.RawChainConfig(1, 2),
Chain.POLYGON to ChainsConfig.RawChainConfig(10, 11),
Chain.ARBITRUM to ChainsConfig.RawChainConfig(999, 999)
),
ChainsConfig.RawChainConfig(100, 2)
),
res
)
}
}

6
src/test/resources/configs/chains-basic.yaml
View File

@@ -10,3 +10,9 @@ chain-settings:
lags:
syncing: 6
lagging: 1
- id: optimism
lags:
lagging: 3
- id: sepolia
lags:
syncing: 10