Chains config refactoring (#311)

Generate code from chains config
This commit is contained in:
Vyacheslav
2023-09-29 13:36:34 +03:00
committed by GitHub
parent a2c5743fb2
commit ef6af898e0
80 changed files with 1527 additions and 1321 deletions

View File

@@ -18,11 +18,10 @@ jobs:
with:
distribution: 'zulu'
java-version: 20
- name: Upload to Docker
- name: Setup gradle
uses: gradle/gradle-build-action@v2
with:
arguments: jib -Pdocker=drpcorg
- name: Upload to Docker
run: make jib
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

View File

@@ -30,14 +30,12 @@ jobs:
- name: Install System Libs
run: sudo apt-get install -y openssl libapr1
- name: Check
- name: Setup gradle
uses: gradle/gradle-build-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
arguments: check
env:
CI: true
- name: Check
run: make test
- name: Upload Coverage Report
uses: codecov/codecov-action@v1

18
Makefile Normal file
View File

@@ -0,0 +1,18 @@
all: build-foundation build-main
build-foundation:
cd foundation && ../gradlew build publishToMavenLocal
build-main:
./gradlew build
test: build-foundation
./gradlew check
jib: build-foundation
./gradlew jib -Pdocker=drpcorg
clean:
./gradlew clean;
cd foundation && ../gradlew clean

View File

@@ -18,6 +18,7 @@ plugins {
id 'idea'
id 'application'
id 'jacoco'
id 'chainsconfig.codegen'
alias(libs.plugins.kotlin)
alias(libs.plugins.jib)
@@ -119,6 +120,7 @@ dependencies {
implementation(variantOf(libs.netty.tcnative.boringssl) { classifier("osx-aarch_64") })
implementation(variantOf(libs.netty.tcnative.boringssl) { classifier("linux-x86_64") })
implementation(variantOf(libs.netty.tcnative.boringssl) { classifier("osx-x86_64") })
implementation 'dshackle:foundation:1.0.0'
}
compileKotlin {
@@ -224,7 +226,9 @@ protobuf {
sourceSets {
main {
resources.srcDirs += project.buildDir.absolutePath + "/generated/version"
kotlin {
srcDir project.buildDir.absolutePath + "/generated/kotlin"
}
proto {
srcDir 'emerald-grpc/proto'
}
@@ -318,3 +322,11 @@ static def getVersion() {
return result.replaceAll(/^v/, '')
}
ktlint {
filter {
exclude { element -> element.file.path.contains("generated/") }
}
}
compileKotlin.dependsOn chainscodegen

14
buildSrc/build.gradle.kts Normal file
View File

@@ -0,0 +1,14 @@
plugins {
`kotlin-dsl`
}
repositories {
mavenLocal()
mavenCentral()
}
dependencies {
implementation("org.yaml:snakeyaml:1.24")
implementation("dshackle:foundation:1.0.0")
implementation("com.squareup:kotlinpoet:1.14.2")
}

View File

@@ -0,0 +1,118 @@
import com.squareup.kotlinpoet.*
import com.squareup.kotlinpoet.ParameterizedTypeName.Companion.parameterizedBy
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.ChainsConfigReader
import io.emeraldpay.dshackle.foundation.ChainOptionsReader
open class CodeGen(private val config: ChainsConfig) {
companion object {
fun generateFromChains(path: File) {
val chainConfigReader = ChainsConfigReader(ChainOptionsReader())
val config = chainConfigReader.read(null)
CodeGen(config).generateChainsFile().writeTo(path)
}
}
private fun addEnumProperties(builder: TypeSpec.Builder): TypeSpec.Builder {
builder.addEnumConstant(
"UNSPECIFIED",
TypeSpec.anonymousClassBuilder()
.addSuperclassConstructorParameter("%L, %S, %S, %S, %L, %L", 0, "UNSPECIFIED", "Unknown", "0x0", 0, "emptyList()")
.build(),
)
for (chain in config) {
builder.addEnumConstant(
chain.blockchain.uppercase().replace('-', '_') + "__" + chain.id.uppercase().replace('-', '_'),
TypeSpec.anonymousClassBuilder()
.addSuperclassConstructorParameter(
"%L, %S, %S, %S, %L, %L",
chain.grpcId,
chain.code,
chain.blockchain.replaceFirstChar { it.uppercase() } + " " + chain.id.replaceFirstChar { it.uppercase() },
chain.chainId,
chain.netVersion,
"listOf(" + chain.shortNames.map { "\"${it}\"" }.joinToString() + ")",
)
.build(),
)
}
return builder
}
fun generateChainsFile(): FileSpec {
val byIdFun = FunSpec.builder("byId")
.addParameter("id", Int::class)
.returns(ClassName("", "Chain"))
.beginControlFlow("for (chain in values())")
.beginControlFlow("if (chain.id == id)")
.addStatement("return chain")
.endControlFlow()
.endControlFlow()
.addStatement("return UNSPECIFIED")
.build()
val chainType = addEnumProperties(
TypeSpec.enumBuilder("Chain")
.addType(TypeSpec.companionObjectBuilder().addFunction(byIdFun).build())
.primaryConstructor(
FunSpec.constructorBuilder()
.addParameter("id", Int::class)
.addParameter("chainCode", String::class)
.addParameter("chainName", String::class)
.addParameter("chainId", String::class)
.addParameter("netVersion", Long::class)
.addParameter("shortNames", List::class.asClassName().parameterizedBy(String::class.asClassName()))
.build(),
)
.addProperty(
PropertySpec.builder("id", Int::class)
.initializer("id")
.build(),
)
.addProperty(
PropertySpec.builder("chainCode", String::class)
.initializer("chainCode")
.build(),
)
.addProperty(
PropertySpec.builder("netVersion", Long::class)
.initializer("netVersion")
.build(),
)
.addProperty(
PropertySpec.builder("chainId", String::class)
.initializer("chainId")
.build(),
)
.addProperty(
PropertySpec.builder("chainName", String::class)
.initializer("chainName")
.build(),
)
.addProperty(
PropertySpec.builder("shortNames", List::class.asClassName().parameterizedBy(String::class.asClassName()))
.initializer("shortNames")
.build(),
),
).build()
return FileSpec.builder("io.emeraldpay.dshackle", "Chain")
.addType(chainType)
.build()
}
}
open class ChainsCodeGenTask : DefaultTask() {
init {
group = "custom"
description = "Generate chains config"
}
@TaskAction
fun chainscodegenClass() {
val output = project.layout.buildDirectory.dir("generated/kotlin").get().asFile
output.mkdirs()
CodeGen.generateFromChains(output)
}
}
tasks.register<ChainsCodeGenTask>("chainscodegen")

32
foundation/build.gradle Normal file
View File

@@ -0,0 +1,32 @@
plugins {
id 'org.jetbrains.kotlin.jvm' version '1.9.10'
id 'maven-publish'
}
repositories {
mavenLocal()
mavenCentral()
}
group = 'dshackle'
dependencies {
implementation 'org.yaml:snakeyaml:1.24'
testImplementation 'org.junit.jupiter:junit-jupiter:5.9.1'
}
test {
useJUnitPlatform()
}
version '1.0.0'
publishing {
publications {
mavenJava(MavenPublication) {
from components.java
}
}
repositories {
mavenLocal()
}
}

View File

View File

@@ -0,0 +1,90 @@
package io.emeraldpay.dshackle.foundation
import java.time.Duration
class ChainOptions {
data class Options(
val disableUpstreamValidation: Boolean,
val disableValidation: Boolean,
val validationInterval: Int,
val timeout: Duration,
val providesBalance: Boolean?,
val validatePeers: Boolean,
val minPeers: Int,
val validateSyncing: Boolean,
val validateCallLimit: Boolean,
val validateChain: Boolean,
)
open class DefaultOptions : PartialOptions() {
var chains: List<String>? = null
var options: PartialOptions? = null
}
open class PartialOptions {
companion object {
@JvmStatic
fun getDefaults(): PartialOptions {
val options = PartialOptions()
options.minPeers = 1
return options
}
}
var disableValidation: Boolean? = null
var disableUpstreamValidation: Boolean? = null
var validationInterval: Int? = null
set(value) {
require(value == null || value > 0) {
"validation-interval must be a positive number: $value"
}
field = value
}
var timeout: Duration? = null
var providesBalance: Boolean? = null
var validatePeers: Boolean? = null
var validateCalllimit: Boolean? = null
var minPeers: Int? = null
set(value) {
require(value == null || value >= 0) {
"min-peers must be a positive number: $value"
}
field = value
}
var validateSyncing: Boolean? = null
var validateChain: Boolean? = null
fun merge(overwrites: PartialOptions?): PartialOptions {
if (overwrites == null) {
return this
}
val copy = PartialOptions()
copy.validatePeers = overwrites.validatePeers ?: this.validatePeers
copy.minPeers = overwrites.minPeers ?: this.minPeers
copy.disableValidation = overwrites.disableValidation ?: this.disableValidation
copy.validationInterval = overwrites.validationInterval ?: this.validationInterval
copy.providesBalance = overwrites.providesBalance ?: this.providesBalance
copy.validateSyncing = overwrites.validateSyncing ?: this.validateSyncing
copy.validateCalllimit = overwrites.validateCalllimit ?: this.validateCalllimit
copy.timeout = overwrites.timeout ?: this.timeout
copy.validateChain = overwrites.validateChain ?: this.validateChain
copy.disableUpstreamValidation =
overwrites.disableUpstreamValidation ?: this.disableUpstreamValidation
return copy
}
fun buildOptions(): Options =
Options(
this.disableUpstreamValidation ?: false,
this.disableValidation ?: false,
this.validationInterval ?: 30,
this.timeout ?: Duration.ofSeconds(60),
this.providesBalance,
this.validatePeers ?: true,
this.minPeers ?: 1,
this.validateSyncing ?: true,
this.validateCalllimit ?: true,
this.validateChain ?: true,
)
}
}

View File

@@ -0,0 +1,48 @@
package io.emeraldpay.dshackle.foundation
import org.yaml.snakeyaml.nodes.MappingNode
import java.time.Duration
class ChainOptionsReader : YamlConfigReader<ChainOptions.PartialOptions>() {
override fun read(upNode: MappingNode?): ChainOptions.PartialOptions? {
return if (hasAny(upNode, "options")) {
return getMapping(upNode, "options")?.let { values ->
readOptions(values)
}
} else {
null
}
}
fun readOptions(values: MappingNode): ChainOptions.PartialOptions {
val options = ChainOptions.PartialOptions()
getValueAsBool(values, "validate-peers")?.let {
options.validatePeers = it
}
getValueAsBool(values, "validate-syncing")?.let {
options.validateSyncing = it
}
getValueAsBool(values, "validate-call-limit")?.let {
options.validateCalllimit = it
}
getValueAsBool(values, "validate-chain")?.let {
options.validateChain = it
}
getValueAsInt(values, "min-peers")?.let {
options.minPeers = it
}
getValueAsInt(values, "timeout")?.let {
options.timeout = Duration.ofSeconds(it.toLong())
}
getValueAsBool(values, "disable-validation")?.let {
options.disableValidation = it
}
getValueAsInt(values, "validation-interval")?.let {
options.validationInterval = it
}
getValueAsBool(values, "balance")?.let {
options.providesBalance = it
}
return options
}
}

View File

@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.config
package io.emeraldpay.dshackle.foundation
import org.yaml.snakeyaml.nodes.MappingNode

View File

@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.config
package io.emeraldpay.dshackle.foundation
/**
* Update configuration value from environment variables. Format: ${ENV_VAR_NAME}

View File

@@ -14,12 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.config
package io.emeraldpay.dshackle.foundation
import org.yaml.snakeyaml.Yaml
import org.yaml.snakeyaml.nodes.CollectionNode
import org.yaml.snakeyaml.nodes.MappingNode
import org.yaml.snakeyaml.nodes.Node
import org.yaml.snakeyaml.nodes.NodeTuple
import org.yaml.snakeyaml.nodes.ScalarNode
import java.io.InputStream
import java.io.InputStreamReader
@@ -45,6 +46,32 @@ abstract class YamlConfigReader<T> : ConfigReader<T> {
return asMappingNode(yaml.compose(InputStreamReader(input)))
}
fun mergeMappingNode(a: MappingNode?, b: MappingNode?): MappingNode? = when {
a == null -> b
b == null -> a
else -> {
val mergedTuples = a.value.toMutableList()
mergedTuples.addAll(
b.value.filter { tupleB ->
mergedTuples.none { it.keyNode.valueAsString() == tupleB.keyNode.valueAsString() } && (tupleB.valueNode is MappingNode || tupleB.valueNode is ScalarNode)
},
)
a.value.forEach { tupleA ->
b.value.find { it.keyNode.valueAsString() == tupleA.keyNode.valueAsString() }?.let { tupleB ->
if (tupleA.valueNode is MappingNode && tupleB.valueNode is MappingNode) {
mergedTuples[mergedTuples.indexOf(tupleA)] = NodeTuple(tupleA.keyNode, mergeMappingNode(tupleA.valueNode as MappingNode, tupleB.valueNode as MappingNode))
} else if (tupleA.valueNode is ScalarNode && tupleB.valueNode is ScalarNode) {
mergedTuples[mergedTuples.indexOf(tupleA)] = NodeTuple(tupleA.keyNode, tupleB.valueNode)
}
}
}
MappingNode(a.tag, mergedTuples, a.flowStyle)
}
}
protected fun hasAny(mappingNode: MappingNode?, key: String): Boolean =
mappingNode?.let { node ->
node.value
@@ -114,6 +141,16 @@ abstract class YamlConfigReader<T> : ConfigReader<T> {
}
}
protected fun getValueAsLong(mappingNode: MappingNode?, key: String): Long? {
return getValue(mappingNode, key)?.let {
return@let if (it.isPlain) {
it.value.toLongOrNull()
} else {
null
}
}
}
protected fun getValueAsBool(mappingNode: MappingNode?, key: String): Boolean? {
return getValue(mappingNode, key)?.let {
return@let if (it.isPlain) {

View File

@@ -0,0 +1,55 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.foundation.ChainOptions
import java.time.Duration
data class ChainsConfig(private val chains: List<ChainConfig>) : Iterable<ChainsConfig.ChainConfig> {
private val chainMap: Map<String, ChainConfig> = chains.fold(emptyMap()) { acc, item ->
acc.plus(item.shortNames.map { Pair(it, item) })
}
override fun iterator(): Iterator<ChainConfig> {
return chains.iterator()
}
companion object {
@JvmStatic
fun default(): ChainsConfig = ChainsConfig(emptyList())
}
data class ChainConfig(
val expectedBlockTime: Duration,
val syncingLagSize: Int,
val laggingLagSize: Int,
val options: ChainOptions.PartialOptions,
val chainId: String,
val netVersion: Long,
val grpcId: Int,
val code: String,
val shortNames: List<String>,
val callLimitContract: String?,
val id: String,
val blockchain: String,
) {
companion object {
@JvmStatic
fun default() = ChainConfig(
Duration.ofSeconds(12),
6,
1,
ChainOptions.PartialOptions(),
"0x0",
0,
0,
"UNKNOWN",
emptyList(),
null,
"undefined",
"undefined",
)
}
}
fun resolve(chain: String): ChainConfig {
return chainMap[chain] ?: ChainConfig.default()
}
}

View File

@@ -0,0 +1,114 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.foundation.ChainOptionsReader
import io.emeraldpay.dshackle.foundation.YamlConfigReader
import org.yaml.snakeyaml.DumperOptions
import org.yaml.snakeyaml.nodes.MappingNode
import org.yaml.snakeyaml.nodes.NodeTuple
import org.yaml.snakeyaml.nodes.ScalarNode
import org.yaml.snakeyaml.nodes.Tag
class ChainsConfigReader(
private val chainsOptionsReader: ChainOptionsReader,
) : YamlConfigReader<ChainsConfig>() {
private val defaultConfig = this::class.java.getResourceAsStream("/chains.yaml")!!
private fun readChains(input: MappingNode?): Map<Int, Pair<String, MappingNode>> {
return getMapping(input, "chain-settings")?.let { config ->
val default = getMapping(config, "default")
getList<MappingNode>(config, "protocols")?.let { protocols ->
protocols.value.fold(emptyMap()) { acc, protocol ->
val blockchain = getValueAsString(protocol, "id")
?: throw IllegalArgumentException("Blockchain id is not defined")
val settings = mergeMappingNode(default, getMapping(protocol, "settings"))
acc.plus(
getList<MappingNode>(protocol, "chains")?.let { chains ->
chains.value.map { chain ->
val chainSettings = mergeMappingNode(settings, getMapping(chain, "settings"))
val updatedChain = mergeMappingNode(
chain,
MappingNode(
chain.tag,
listOf(
NodeTuple(ScalarNode(Tag.STR, "settings", null, null, DumperOptions.ScalarStyle.LITERAL), chainSettings),
NodeTuple(
ScalarNode(Tag.STR, "blockchain", null, null, DumperOptions.ScalarStyle.LITERAL),
ScalarNode(Tag.STR, blockchain, null, null, DumperOptions.ScalarStyle.LITERAL),
),
),
chain.flowStyle,
),
)
val grpcId = getValueAsInt(updatedChain, "grpcId")
?: throw IllegalArgumentException("grpcId for chain is not defined")
Pair(grpcId, Pair(blockchain, updatedChain!!))
}
} ?: listOf(),
)
}
}
} ?: emptyMap()
}
private fun parseChain(blockchain: String, node: MappingNode): ChainsConfig.ChainConfig {
val id = getValueAsString(node, "id")
?: throw IllegalArgumentException("undefined id for $blockchain")
val settings = getMapping(node, "settings") ?: throw IllegalArgumentException("undefined settings for $blockchain")
val lags = getMapping(settings, "lags")?.let { lagConfig ->
Pair(
getValueAsInt(lagConfig, "syncing")
?: throw IllegalArgumentException("undefined syncing for $blockchain"),
getValueAsInt(lagConfig, "lagging")
?: throw IllegalArgumentException("undefined lagging for $blockchain"),
)
} ?: throw IllegalArgumentException("undefined lags for $blockchain")
val expectedBlockTime = getValueAsDuration(settings, "expected-block-time")
?: throw IllegalArgumentException("undefined expected block time")
val validateContract = getValueAsString(node, "call-validate-contract")
val options = chainsOptionsReader.read(settings) ?: ChainOptions.PartialOptions()
val chainId = getValueAsString(node, "chain-id")
?: throw IllegalArgumentException("undefined chain id for $blockchain")
val code = getValueAsString(node, "code")
?: throw IllegalArgumentException("undefined code for $blockchain")
val grpcId = getValueAsInt(node, "grpcId")
?: throw IllegalArgumentException("undefined code for $blockchain")
val netVersion = getValueAsLong(node, "net-version") ?: chainId.drop(2).toLong(radix = 16)
val shortNames = getListOfString(node, "short-names")
?: throw IllegalArgumentException("undefined shortnames for $blockchain")
return ChainsConfig.ChainConfig(
expectedBlockTime = expectedBlockTime,
syncingLagSize = lags.first,
laggingLagSize = lags.second,
options = options,
callLimitContract = validateContract,
chainId = chainId,
code = code,
grpcId = grpcId,
netVersion = netVersion,
shortNames = shortNames,
id = id,
blockchain = blockchain,
)
}
override fun read(input: MappingNode?): ChainsConfig {
val default = readChains(readNode(defaultConfig))
val current = readChains(input)
val chains = default.keys.toSet().plus(current.keys).map {
val defChain = default[it]
val curChain = current[it]
when {
curChain == null && defChain != null -> parseChain(defChain.first, defChain.second)
curChain != null && defChain == null -> parseChain(curChain.first, curChain.second)
curChain != null && defChain != null -> {
val merged = mergeMappingNode(defChain.second, curChain.second)
parseChain(defChain.first, merged!!)
}
else -> ChainsConfig.ChainConfig.default()
}
}
return ChainsConfig(chains)
}
}

View File

@@ -0,0 +1,407 @@
version: v1
chain-settings:
default:
expected-block-time: 12s
lags:
syncing: 6
lagging: 1
protocols:
- id: bitcoin
settings:
expected-block-time: 10m
lags:
syncing: 3
lagging: 1
chains:
- id: mainnet
chain-id: 0x0
short-names: [bitcoin, btc]
code: BTC
grpcId: 1
- id: testnet
chain-id: 0x0
short-names: [bitcoin-testnet]
code: TESTNET_BITCOIN
grpcId: 10003
- id: ethereum
settings:
expected-block-time: 12s
lags:
syncing: 6
lagging: 1
chains:
- id: mainnet
chain-id: 0x1
short-names: [eth, ethereum, homestead]
code: ETH
grpcId: 100
call-validate-contract: 0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96
- id: goerli
chain-id: 0x5
code: GOERLI
grpcId: 10005
short-names: [goerli, goerli-testnet]
call-validate-contract: 0xCD9303A1F6da2a68f465A579a24cc2Ee5AE2192f
- id: ropsten
code: ROPSTEN
grpcId: 10006
chain-id: 0x3
short-names: [ropsten, ropsten-testnet]
- id: sepolia
code: SEPOLIA
grpcId: 10008
chain-id: 0xaa36a7
short-names: [sepolia, sepolia-testnet]
- id: holesky
code: ETHEREUM_HOLESKY
grpcId: 10027
chain-id: 0x4268
short-names: [holesky, ethereum-holesky]
- id: ethereum-classic
chains:
- id: mainnet
short-names: [ethereum-classic, etc]
chain-id: 0x3d
net-version: 1
code: ETC
grpcId: 101
- id: fantom
settings:
expected-block-time: 3s
options:
validate-peers: false
lags:
syncing: 10
lagging: 5
chains:
- id: mainnet
short-names: [fantom]
code: FTM
grpcId: 102
chain-id: 0xfa
- id: testnet
code: FANTOM_TESTNET
grpcId: 10016
short-names: [fantom-testnet]
chain-id: 0xfa2
- id: polygon
settings:
expected-block-time: 2.7s
lags:
syncing: 20
lagging: 10
chains:
- id: mainnet
call-validate-contract: 0x53Daa71B04d589429f6d3DF52db123913B818F23
code: POLYGON
grpcId: 1002
chain-id: 0x89
short-names: [polygon, matic]
- id: mumbai
call-validate-contract: 0x53Daa71B04d589429f6d3DF52db123913B818F23
code: POLYGON_POS_MUMBAI
grpcId: 10013
chain-id: 0x13881
short-names: [polygon-mumbai]
- id: arbitrum
settings:
expected-block-time: 260ms
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
chains:
- id: mainnet
code: ARBITRUM
grpcId: 1004
short-names: [arbitrum, arb]
chain-id: 0xa4b1
- id: goerli
code: ARBITRUM_TESTNET
grpcId: 10009
short-names: [arbitrum-testnet, arbitrum-goerli]
chain-id: 0x66eed
settings:
expected-block-time: 1s
- id: optimism
settings:
expected-block-time: 2s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
chains:
- id: mainnet
code: OPTIMISM
grpcId: 1005
short-names: [optimism]
chain-id: 0xa
- id: goerli
code: OPTIMISM_TESTNET
grpcId: 10010
short-names: [optimism-testnet, optimism-goerli]
chain-id: 0x1A4
- id: bsc
settings:
expected-block-time: 3s
lags:
syncing: 20
lagging: 10
chains:
- id: mainnet
code: BSC
grpcId: 1006
chain-id: 0x38
short-names: [bsc, binance, bnb-smart-chain]
- id: testnet
code: BSC_TESTNET
grpcId: 10026
short-names: [bsc-testnet]
chain-id: 0x61
- id: polygon-zkevm
settings:
expected-block-time: 2.7s
options:
disable-validation: true
lags:
syncing: 40
lagging: 20
chains:
- id: mainnet
code: POLYGON_ZKEVM
grpcId: 1007
short-names: [polygon-zkevm]
chain-id: 0x44d
- id: testnet
code: POLYGON_ZKEVM_TESTNET
grpcId: 10011
short-names: [polygon-zkevm-testnet]
chain-id: 0x5a2
settings:
expected-block-time: 1m
- id: arbitrum-nova
settings:
expected-block-time: 1s
options:
disable-validation: true
lags:
syncing: 40
lagging: 20
chains:
- id: mainnet
code: ARBITRUM_NOVA
grpcId: 1008
short-names: [arbitrum-nova]
chain-id: 0xa4ba
- id: zksync
settings:
expected-block-time: 5s
options:
disable-validation: true
lags:
syncing: 40
lagging: 20
chains:
- id: mainnet
code: ZKSYNC
grpcId: 1009
chain-id: 0x144
short-names: [zksync]
- id: testnet
code: ZKS_TESTNET
grpcId: 10012
chain-id: 0x118
short-names: [zksync-testnet]
- id: base
settings:
expected-block-time: 2s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
chains:
- id: mainnet
code: BASE
grpcId: 1010
short-names: [base]
chain-id: 0x2105
- id: goerli
code: BASE_GOERLI
grpcId: 10014
short-names: [base-goerli]
chain-id: 0x14a33
- id: linea
settings:
expected-block-time: 12s
lags:
syncing: 6
lagging: 1
chains:
- id: mainnet
code: LINEA
grpcId: 1011
short-names: [linea]
chain-id: 0xe708
- id: goerli
code: LINEA_GOERLI
grpcId: 10015
short-names: [linea-goerli]
chain-id: 0xe704
- id: gnosis
settings:
expected-block-time: 6s
options:
validate-peers: false
lags:
syncing: 10
lagging: 5
chains:
- id: mainnet
code: GNOSIS
grpcId: 1012
short-names: [gnosis]
chain-id: 0x64
- id: chiado
code: GNOSIS_CHIADO
grpcId: 10017
short-names: [gnosis-chiado]
chain-id: 0x27d8
- id: avalanche
settings:
expected-block-time: 2s
options:
validate-peers: false
validate-syncing: false
lags:
syncing: 10
lagging: 5
chains:
- id: mainnet
code: AVALANCHE
grpcId: 1013
short-names: [avalanche]
chain-id: 0xa86a
- id: fuji
code: AVALANCHE_FUJI
grpcId: 10018
short-names: [avalanche-fuji]
chain-id: 0xa869
- id: aurora
settings:
expected-block-time: 1s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
chains:
- id: mainnet
code: AURORA
grpcId: 1015
short-names: [aurora]
chain-id: 0x4e454152
- id: testnet
code: AURORA_TESTNET
grpcId: 10021
short-names: [aurora-testnet]
chain-id: 0x4e454153
- id: mantle
settings:
expected-block-time: 500ms
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
chains:
- id: mainnet
code: MANTLE
grpcId: 1017
short-names: [mantle]
chain-id: 0x1388
- id: testnet
code: MANTLE_TESTNET
grpcId: 10023
short-names: [mantle-testnet]
chain-id: 0x1389
- id: klaytn
settings:
expected-block-time: 1s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
chains:
- id: mainnet
code: KLAYTN
grpcId: 1018
short-names: [klaytn]
chain-id: 0x2019
- id: baobab
code: KLAYTN_BAOBAB
grpcId: 10024
short-names: [klaytn-baobab]
chain-id: 0x3e9
- id: celo
settings:
expected-block-time: 5s
lags:
syncing: 10
lagging: 5
chains:
- id: mainnet
code: CELO
grpcId: 1019
short-names: [celo]
chain-id: 0xa4ec
- id: alfajores
code: CELO_ALFAJORES
grpcId: 10028
short-names: [celo-alfajores]
chain-id: 0xaef3
- id: moonbeam
settings:
expected-block-time: 12s
lags:
syncing: 6
lagging: 1
chains:
- id: mainnet
code: MOONBEAM
grpcId: 1020
short-names: [moonbeam]
chain-id: 0x504
- id: moonriver
code: MOONRIVER
grpcId: 1021
short-names: [moonriver]
chain-id: 0x505
- id: moonbase-alpha
code: MOONBEAM_ALPHA
grpcId: 10029
short-names: [moonbase-alpha]
chain-id: 0x507
- id: scroll
settings:
expected-block-time: 3s
options:
validate-peers: false
lags:
syncing: 10
lagging: 5
chains:
- id: alphanet
code: SCROLL_ALPHANET
grpcId: 10022
short-names: [scroll-alphanet]
chain-id: 0x82751
- id: sepolia
code: SCROLL_SEPOLIA
grpcId: 10025
short-names: [scroll-sepolia]
chain-id: 0x8274f

View File

@@ -0,0 +1,52 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.foundation.EnvVariables
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.CsvSource
import org.junit.jupiter.params.provider.ValueSource
class EnvVariablesTest {
lateinit var reader: EnvVariables
@BeforeEach
fun setup() {
reader = EnvVariables()
}
@ParameterizedTest
@ValueSource(strings = ["", "a", "13143", "/etc/client1.myservice.com.key", "true", "1a68f20154fc258fe4149c199ad8f281"])
fun `Post process for usual strings`(s: String) {
assertEquals(s, reader.postProcess(s))
}
@ParameterizedTest
@CsvSource(
"p_\${id}, p_1",
"home: \${HOME}, home: /home/user",
"\${PASSWORD}, 1a68f20154fc258fe4149c199ad8f281",
)
fun `Post process replaces from env`(orig: String, replaced: String) {
System.setProperty("id", "1")
System.setProperty("HOME", "/home/user")
System.setProperty("PASSWORD", "1a68f20154fc258fe4149c199ad8f281")
assertEquals(replaced, reader.postProcess(orig))
}
}

View File

@@ -0,0 +1,80 @@
package io.emeraldpay.dshackle.foundation
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.CsvSource
import org.yaml.snakeyaml.Yaml
import org.yaml.snakeyaml.nodes.MappingNode
import java.io.StringReader
class YamlConfigReaderTest {
@ParameterizedTest
@CsvSource(
"1024, 1024",
"1k, 1024",
"1kb, 1024",
"1K, 1024",
"16kb, 16384",
"1M, 1048576",
"4mb, 4194304",
)
fun `reads bytes values`(input: String, expected: Int) {
val rdr = Impl()
assertEquals(expected, rdr.getValueAsBytes(rdr.asNode("test", input), "test"))
}
@Test
fun `mergeMappingNode should merge nodes correctly`() {
val yaml = Yaml()
val a = yaml.compose(
StringReader(
"""
key1: value1
key2:
subkey1: subvalue1
""",
),
) as MappingNode
val b = yaml.compose(
StringReader(
"""
key1: value3
key2:
subkey1: subvalue1Modified
subkey2: subvalue2
key3: value3
""",
),
) as MappingNode
val result = Impl().mergeMappingNode(a, b)!!
val yml = Impl()
assertEquals(3, result.value.size)
assertEquals("value3", yml.getNodeValue(result, "key1"))
assertEquals("subvalue1Modified", yml.getNodeValue(yml.getNode(result, "key2") as MappingNode, "subkey1"))
assertEquals("subvalue2", yml.getNodeValue(yml.getNode(result, "key2") as MappingNode, "subkey2"))
assertEquals("value3", yml.getNodeValue(result, "key3"))
}
class Impl : YamlConfigReader<Any>() {
override fun read(input: MappingNode?): Any? {
return null
}
fun getNode(node: MappingNode, key: String): Any? {
return Impl().getMapping(node, key)
}
fun getNodeValue(node: MappingNode, key: String): String? {
return Impl().getValueAsString(node, key)
}
fun asNode(key: String, value: String): MappingNode {
return Yaml().compose(StringReader("$key: $value")) as MappingNode
}
}
}

View File

@@ -0,0 +1,44 @@
package org.drpc.chainsconfig
import io.emeraldpay.dshackle.config.ChainsConfigReader
import io.emeraldpay.dshackle.foundation.ChainOptionsReader
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
internal class ChainsConfigReaderTest {
@Test
fun `read standard config without custom`() {
val reader = ChainsConfigReader(ChainOptionsReader())
val config = reader.read(null)
val arb = config.resolve("arbitrum-goerli")
assertEquals(arb.chainId, "0x66eed")
assertEquals(arb.expectedBlockTime.seconds, 1L)
assertEquals(arb.options.validatePeers, false)
assertEquals(arb.id, "goerli")
val ethc = config.resolve("ethereum-classic")
assertEquals(ethc.chainId, "0x3d")
assertEquals(ethc.netVersion, 1)
assertEquals(ethc.code, "ETC")
assertEquals(ethc.grpcId, 101)
assertEquals(ethc.expectedBlockTime.seconds, 12)
assertEquals(ethc.laggingLagSize, 1)
assertEquals(ethc.syncingLagSize, 6)
assertEquals(ethc.id, "mainnet")
}
@Test
fun `read standard config with custom one`() {
val reader = ChainsConfigReader(ChainOptionsReader())
val chains = reader.read(this.javaClass.classLoader.getResourceAsStream("configs/chains-basic.yaml")!!)!!
val config = chains.resolve("fantom")
assertEquals(config.expectedBlockTime.seconds, 10)
assertEquals(config.chainId, "0xfb")
assertEquals(config.syncingLagSize, 11)
assertEquals(config.laggingLagSize, 4)
assertEquals(config.code, "FTA")
assertEquals(config.grpcId, 102)
assertEquals(config.netVersion, 251)
}
}

View File

@@ -0,0 +1,18 @@
version: v1
chain-settings:
protocols:
- id: fantom
settings:
expected-block-time: 10s
options:
validate-peers: true
lags:
syncing: 11
lagging: 4
chains:
- id: mainnet
short-names: [fantom]
code: FTA
grpcId: 102
chain-id: 0xfb

View File

@@ -1 +1 @@
enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS")
enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS")

View File

@@ -6,10 +6,10 @@ enum class BlockchainType {
companion object {
val pow = setOf(
Chain.ETHEREUM_CLASSIC__MAINNET, Chain.RSK__MAINNET, Chain.ETHEREUM__KOVAN,
Chain.ETHEREUM__MORDEN, Chain.ETHEREUM__RINKEBY
Chain.ETHEREUM_CLASSIC__MAINNET,
)
val bitcoin = setOf(Chain.BITCOIN__MAINNET, Chain.BITCOIN__TESTNET)
@JvmStatic
fun from(chain: Chain): BlockchainType {
return if (pow.contains(chain)) {

View File

@@ -1,126 +0,0 @@
/*
* Copyright (c) 2016-2019 ETCDEV GmbH, All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle
enum class Chain(val id: Int, val chainCode: String, val chainName: String) {
UNSPECIFIED(0, "UNSPECIFIED", "Unknown"),
BITCOIN__MAINNET(1, "BTC", "Bitcoin"), // GRIN(2, "GRIN", "Grin"),
// Networks with tokens
ETHEREUM__MAINNET(100, "ETH", "Ethereum"),
ETHEREUM_CLASSIC__MAINNET(101, "ETC", "Ethereum Classic"),
FANTOM__MAINNET(
102,
"FTM",
"Fantom"
), // LIGHTNING(1001, "BTC_LN", "Bitcoin Lightning"),
POLYGON_POS__MAINNET(1002, "POLYGON", "Polygon Matic"),
RSK__MAINNET(1003, "RSK", "Bitcoin RSK"),
ARBITRUM__MAINNET(
1004,
"ARBITRUM",
"Arbitrum"
),
OPTIMISM__MAINNET(1005, "OPTIMISM", "Optimism"),
BSC__MAINNET(1006, "BSC", "Binance Smart Chain"),
POLYGON_ZKEVM__MAINNET(
1007,
"POLYGON_ZKEVM",
"Polygon ZK-EVM"
),
ARBITRUM_NOVA__MAINNET(
1008,
"ARBITRUM_NOVA",
"Arbitrum Nova"
),
ZKSYNC__MAINNET(1009, "ZKSYNC", "ZkSync Era"),
BASE__MAINNET(1010, "BASE", "Base"),
LINEA__MAINNET(1011, "LINEA", "Linea"),
GNOSIS__MAINNET(1012, "GNOSIS", "Gnosis"),
AVALANCHE__MAINNET(1013, "AVALANCHE", "Avalanche"),
// STARKNET__MAINNET(1014, "STARKNET", "Starknet"),
AURORA__MAINNET(1015, "AURORA", "Aurora"),
// SCROLL__MAINNET(1016, "SCROLL", "Scroll"),
MANTLE__MAINNET(1017, "MANTLE", "Mantle"),
KLAYTN__MAINNET(1018, "KLAYTN", "Klaytn"),
CELO__MAINNET(1019, "CELO_MAINNET", "Celo"),
MOONBEAM__MAINNET(1020, "MOONBEAM_MAINNET", "Moonbeam"),
MOONBEAM__MOONRIVER(1021, "MOONBASE_MOONRIVER", "Moonriver"),
// Testnets
ETHEREUM__MORDEN(10001, "MORDEN", "Morden Testnet"),
ETHEREUM__KOVAN(10002, "KOVAN", "Kovan Testnet"),
BITCOIN__TESTNET(
10003,
"TESTNET_BITCOIN",
"Bitcoin Testnet"
), // TESTNET_FLOONET(10004, "FLOONET", "Floonet Testnet"),
ETHEREUM__GOERLI(10005, "GOERLI", "Goerli Testnet"),
ETHEREUM__ROPSTEN(
10006,
"ROPSTEN",
"Ropsten Testnet"
),
ETHEREUM__RINKEBY(10007, "RINKEBY", "Rinkeby Testnet"),
ETHEREUM__SEPOLIA(10008, "SEPOLIA", "Sepolia Testnet"),
ARBITRUM__GOERLI(
10009,
"ARBITRUM_TESTNET",
"Arbitrum Testnet"
),
OPTIMISM__GOERLI(10010, "OPTIMISM_TESTNET", "Optimism Testnet"),
POLYGON_ZKEVM__TESTNET(
10011,
"POLYGON_ZKEVM_TESTNET",
"Polygon ZK-EVM Testnet"
),
ZKSYNC__TESTNET(
10012,
"ZKS_TESTNET",
"ZkSync Testnet"
),
POLYGON_POS__MUMBAI(
10013,
"POLYGON_POS_MUMBAI",
"Polygon POS Mumbai Testnet"
),
BASE__GOERLI(10014, "BASE_GOERLI", "Base Goerli Testnet"),
LINEA__GOERLI(10015, "LINEA_GOERLI", "Linea Goerli Testnet"),
FANTOM__TESTNET(10016, "FANTOM_TESTNET", "Fantom Testnet"),
GNOSIS__CHIADO(10017, "GNOSIS_CHIADO", "Gnosis Chiado Testnet"),
AVALANCHE__FUJI(10018, "AVALANCHE_FUJI", "Avalanche Fuji Testnet"),
// STARKNET__GOERLI(10019, "STARKNET_GOERLI", "Starknet Goerli"),
// STARKNET__GOERLI2(10020, "STARKNET_GOERLI2", "Starknet Goerli 2"),
AURORA__TESTNET(10021, "AURORA_TESTNET", "Aurora Testnet"),
SCROLL__ALPHANET(10022, "SCROLL_ALPHANET", "Scroll Alphanet"),
MANTLE__TESTNET(10023, "MANTLE_TESTNET", "Mantle Testnet"),
KLAYTN__BAOBAB(10024, "KLAYTN_BAOBAB", "Klaytn Baobab"),
SCROLL__SEPOLIA(10025, "SCROLL_SEPOLIA", "Scroll Sepolia"),
BSC__TESTNET(10026, "BSC_TESTNET", "Binance Smart Chain Testnet"),
ETHEREUM__HOLESKY(10027, "ETHEREUM_HOLESKY", "Ethereum Holesky"),
CELO__ALFAJORES(10028, "CELO_ALFAJORES", "Celo Alfajores"),
MOONBEAM__ALPHA(10029, "MOONBEAM_ALPHA", "Moonbase Alpha");
companion object {
fun byId(id: Int): Chain {
for (chain in values()) {
if (chain.id == id) {
return chain
}
}
return UNSPECIFIED
}
}
}

View File

@@ -49,7 +49,7 @@ import kotlin.system.exitProcess
@EnableAsync
open class Config(
private val env: Environment,
private val ctx: ApplicationContext
private val ctx: ApplicationContext,
) {
companion object {

View File

@@ -43,78 +43,19 @@ class Global {
var metricsExtended = false
val chainNames = mapOf(
"ethereum" to Chain.ETHEREUM__MAINNET,
"ethereum-classic" to Chain.ETHEREUM_CLASSIC__MAINNET,
"eth" to Chain.ETHEREUM__MAINNET,
"polygon" to Chain.POLYGON_POS__MAINNET,
"matic" to Chain.POLYGON_POS__MAINNET,
"arbitrum" to Chain.ARBITRUM__MAINNET,
"arb" to Chain.ARBITRUM__MAINNET,
"optimism" to Chain.OPTIMISM__MAINNET,
"binance" to Chain.BSC__MAINNET,
"bsc" to Chain.BSC__MAINNET,
"bnb-smart-chain" to Chain.BSC__MAINNET,
"bsc-testnet" to Chain.BSC__TESTNET,
"etc" to Chain.ETHEREUM_CLASSIC__MAINNET,
"morden" to Chain.ETHEREUM__MORDEN,
"kovan" to Chain.ETHEREUM__KOVAN,
"kovan-testnet" to Chain.ETHEREUM__KOVAN,
"goerli" to Chain.ETHEREUM__GOERLI,
"goerli-testnet" to Chain.ETHEREUM__GOERLI,
"rinkeby" to Chain.ETHEREUM__RINKEBY,
"rinkeby-testnet" to Chain.ETHEREUM__RINKEBY,
"ropsten" to Chain.ETHEREUM__ROPSTEN,
"ropsten-testnet" to Chain.ETHEREUM__ROPSTEN,
"bitcoin" to Chain.BITCOIN__MAINNET,
"bitcoin-testnet" to Chain.BITCOIN__TESTNET,
"sepolia" to Chain.ETHEREUM__SEPOLIA,
"sepolia-testnet" to Chain.ETHEREUM__SEPOLIA,
"ethereum-holesky" to Chain.ETHEREUM__HOLESKY,
"optimism-testnet" to Chain.OPTIMISM__GOERLI,
"arbitrum-testnet" to Chain.ARBITRUM__GOERLI,
"arbitrum-nova" to Chain.ARBITRUM_NOVA__MAINNET,
"polygon-zkevm" to Chain.POLYGON_ZKEVM__MAINNET,
"polygon-zkevm-testnet" to Chain.POLYGON_ZKEVM__TESTNET,
"zksync" to Chain.ZKSYNC__MAINNET,
"zksync-testnet" to Chain.ZKSYNC__TESTNET,
"polygon-mumbai" to Chain.POLYGON_POS__MUMBAI,
"base" to Chain.BASE__MAINNET,
"base-goerli" to Chain.BASE__GOERLI,
"linea" to Chain.LINEA__MAINNET,
"linea-goerli" to Chain.LINEA__GOERLI,
"fantom" to Chain.FANTOM__MAINNET,
"fantom-testnet" to Chain.FANTOM__TESTNET,
"gnosis" to Chain.GNOSIS__MAINNET,
"gnosis-chiado" to Chain.GNOSIS__CHIADO,
"avalanche" to Chain.AVALANCHE__MAINNET,
"avalanche-fuji" to Chain.AVALANCHE__FUJI,
// "starknet" to Chain.CHAIN_STARKNET__MAINNET,
// "starknet-goerli" to Chain.CHAIN_STARKNET__GOERLI,
// "starknet-goerli2" to Chain.CHAIN_STARKNET__GOERLI2,
"aurora" to Chain.AURORA__MAINNET,
"aurora-testnet" to Chain.AURORA__TESTNET,
// "scroll" to Chain.CHAIN_SCROLL__MAINNET,
"scroll-alphanet" to Chain.SCROLL__ALPHANET,
"scroll-sepolia" to Chain.SCROLL__SEPOLIA,
"mantle" to Chain.MANTLE__MAINNET,
"mantle-testnet" to Chain.MANTLE__TESTNET,
"klaytn" to Chain.KLAYTN__MAINNET,
"klaytn-baobab" to Chain.KLAYTN__BAOBAB,
"celo" to Chain.CELO__MAINNET,
"celo-alfajores" to Chain.CELO__ALFAJORES,
"moonriver" to Chain.MOONBEAM__MOONRIVER,
"moonbeam" to Chain.MOONBEAM__MAINNET,
"moonbase-alpha" to Chain.MOONBEAM__ALPHA
)
fun chainById(id: String?): Chain {
if (id == null) {
return Chain.UNSPECIFIED
}
return chainNames[
id.lowercase(Locale.getDefault()).replace("_", "-").trim()
] ?: Chain.UNSPECIFIED
return Chain.values().find { chain ->
chain.shortNames.contains(id.lowercase(Locale.getDefault()).replace("_", "-").trim())
} ?: Chain.UNSPECIFIED
}
fun chainByChainId(id: String): Chain {
return Chain.values().find { chain ->
chain.chainId == id
} ?: Chain.UNSPECIFIED
}
@JvmStatic

View File

@@ -1,5 +1,6 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.foundation.YamlConfigReader
import org.yaml.snakeyaml.nodes.MappingNode
class AccessLogReader : YamlConfigReader<AccessLogConfig>() {

View File

@@ -16,6 +16,7 @@
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.foundation.YamlConfigReader
import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.nodes.MappingNode
import org.yaml.snakeyaml.nodes.ScalarNode

View File

@@ -1,5 +1,6 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.foundation.YamlConfigReader
import org.slf4j.LoggerFactory
import org.springframework.util.ResourceUtils
import org.yaml.snakeyaml.nodes.MappingNode
@@ -55,9 +56,10 @@ class AuthorizationConfigReader : YamlConfigReader<AuthorizationConfig>() {
}
return AuthorizationConfig(
enabled, publicKeyOwner,
enabled,
publicKeyOwner,
authServer ?: AuthorizationConfig.ServerConfig.default(),
authClient ?: AuthorizationConfig.ClientConfig.default()
authClient ?: AuthorizationConfig.ClientConfig.default(),
)
}

View File

@@ -15,6 +15,7 @@
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.foundation.YamlConfigReader
import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.nodes.MappingNode

View File

@@ -1,86 +0,0 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Chain
import java.time.Duration
data class ChainsConfig(private val chains: Map<Chain, RawChainConfig>, val currentDefault: RawChainConfig?) {
companion object {
@JvmStatic
fun default(): ChainsConfig = ChainsConfig(emptyMap(), RawChainConfig.default())
}
data class RawChainConfig(
var expectedBlockTime: Duration? = null,
var syncingLagSize: Int? = null,
var laggingLagSize: Int? = null,
var callLimitContract: String? = null,
var options: UpstreamsConfig.PartialOptions? = null
) {
companion object {
@JvmStatic
fun default() = RawChainConfig(
syncingLagSize = 6,
laggingLagSize = 1
)
}
}
data class ChainConfig(
val expectedBlockTime: Duration,
val syncingLagSize: Int,
val laggingLagSize: Int,
val options: UpstreamsConfig.PartialOptions,
val callLimitContract: String?
) {
companion object {
@JvmStatic
fun default() = ChainConfig(Duration.ofSeconds(12), 6, 1, UpstreamsConfig.PartialOptions(), null)
}
}
fun resolve(chain: Chain): ChainConfig {
val default = currentDefault ?: panic()
val raw = chains[chain] ?: default
val options = default.options?.merge(raw.options) ?: raw.options ?: UpstreamsConfig.PartialOptions()
return ChainConfig(
laggingLagSize = raw.laggingLagSize ?: default.laggingLagSize ?: panic(),
syncingLagSize = raw.syncingLagSize ?: default.syncingLagSize ?: panic(),
options = options,
callLimitContract = raw.callLimitContract,
expectedBlockTime = raw.expectedBlockTime ?: default.expectedBlockTime ?: panic(),
)
}
fun patch(patch: ChainsConfig) = ChainsConfig(
merge(this.chains, patch.chains),
merge(this.currentDefault!!, patch.currentDefault)
)
private fun merge(
current: RawChainConfig,
patch: RawChainConfig?
) = RawChainConfig(
syncingLagSize = patch?.syncingLagSize ?: current.syncingLagSize,
laggingLagSize = patch?.laggingLagSize ?: current.laggingLagSize,
options = patch?.options ?: current.options,
callLimitContract = patch?.callLimitContract ?: current.callLimitContract,
expectedBlockTime = patch?.expectedBlockTime ?: current.expectedBlockTime
)
private fun merge(
current: Map<Chain, RawChainConfig>,
patch: Map<Chain, RawChainConfig>
): Map<Chain, RawChainConfig> {
val currentMut = current.toMutableMap()
for (k in patch) {
currentMut.merge(k.key, k.value) { v1, v2 -> merge(v1, v2) }
}
return currentMut.toMap()
}
fun panic(): Nothing = throw IllegalStateException("Chains settings state is illegal - default config is null")
}

View File

@@ -1,81 +0,0 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Global
import org.yaml.snakeyaml.nodes.CollectionNode
import org.yaml.snakeyaml.nodes.MappingNode
import java.io.InputStream
class ChainsConfigReader(
private val upstreamsConfigReader: UpstreamsConfigReader
) : YamlConfigReader<ChainsConfig>() {
private val defaultConfig = this::class.java.getResourceAsStream("/chains.yaml")!!
override fun read(input: MappingNode?): ChainsConfig {
val default = readInternal(defaultConfig)
val current = readInternal(input)
val res = default.patch(current)
return res
}
fun readInternal(input: InputStream): ChainsConfig {
val configNode = readNode(input)
return readInternal(configNode)
}
fun readInternal(input: MappingNode?): ChainsConfig {
return getMapping(input, "chain-settings")?.let {
val chains = getList<MappingNode>(it, "chains")?.let {
readChains(it)
}
val default = getMapping(it, "default")?.let {
readChain(it)
}
return ChainsConfig(
chains
?.map { Global.chainById(it.first) to it.second }
?.associateBy({ it.first }, { it.second }) ?: emptyMap(),
default
)
} ?: ChainsConfig.default()
}
private fun readChain(node: MappingNode): ChainsConfig.RawChainConfig? {
val rawConfig = ChainsConfig.RawChainConfig()
getMapping(node, "lags")?.let { lagConfig ->
getValueAsInt(lagConfig, "syncing")?.let {
rawConfig.syncingLagSize = it
}
getValueAsInt(lagConfig, "lagging")?.let {
rawConfig.laggingLagSize = it
}
}
getValueAsDuration(node, "expected-block-time")?.let {
rawConfig.expectedBlockTime = it
}
getValueAsString(node, "call-validate-contract")?.let {
rawConfig.callLimitContract = it
}
upstreamsConfigReader.tryReadOptions(node)?.let {
rawConfig.options = it
}
return rawConfig
}
private fun readChains(node: CollectionNode<MappingNode>): List<Pair<String, ChainsConfig.RawChainConfig>> {
return node.value.mapNotNull {
val key = getValueAsString(it, "id")
?: throw InvalidConfigYamlException(filename, it.startMark, "chain id required")
val value = readChain(it)
if (value != null) {
return@mapNotNull key to value
} else {
return@mapNotNull null
}
}
}
}

View File

@@ -1,5 +1,6 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.foundation.YamlConfigReader
import org.yaml.snakeyaml.nodes.MappingNode
class CompressionConfigReader : YamlConfigReader<CompressionConfig>() {

View File

@@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.foundation.YamlConfigReader
import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.nodes.CollectionNode
import org.yaml.snakeyaml.nodes.MappingNode

View File

@@ -16,15 +16,18 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.FileResolver
import io.emeraldpay.dshackle.foundation.ChainOptionsReader
import io.emeraldpay.dshackle.foundation.YamlConfigReader
import org.yaml.snakeyaml.nodes.MappingNode
class MainConfigReader(
fileResolver: FileResolver
fileResolver: FileResolver,
) : YamlConfigReader<MainConfig>() {
private val authConfigReader = AuthConfigReader()
private val proxyConfigReader = ProxyConfigReader()
private val upstreamsConfigReader = UpstreamsConfigReader(fileResolver)
private val optionsReader = ChainOptionsReader()
private val upstreamsConfigReader = UpstreamsConfigReader(fileResolver, optionsReader)
private val cacheConfigReader = CacheConfigReader()
private val tokensConfigReader = TokensConfigReader()
private val monitoringConfigReader = MonitoringConfigReader()
@@ -32,7 +35,7 @@ class MainConfigReader(
private val healthConfigReader = HealthConfigReader()
private val signatureConfigReader = SignatureConfigReader(fileResolver)
private val compressionConfigReader = CompressionConfigReader()
private val chainsConfigReader = ChainsConfigReader(upstreamsConfigReader)
private val chainsConfigReader = ChainsConfigReader(optionsReader)
private val authorizationConfigReader = AuthorizationConfigReader()
override fun read(input: MappingNode?): MainConfig {

View File

@@ -15,6 +15,7 @@
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.foundation.YamlConfigReader
import org.yaml.snakeyaml.nodes.MappingNode
class MonitoringConfigReader : YamlConfigReader<MonitoringConfig>() {

View File

@@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.foundation.YamlConfigReader
import org.apache.commons.lang3.StringUtils
import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.nodes.MappingNode

View File

@@ -1,6 +1,7 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.FileResolver
import io.emeraldpay.dshackle.foundation.YamlConfigReader
import org.yaml.snakeyaml.nodes.MappingNode
class SignatureConfigReader(val fileResolver: FileResolver) : YamlConfigReader<SignatureConfig>() {

View File

@@ -16,6 +16,7 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.foundation.YamlConfigReader
import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.nodes.MappingNode
import java.util.Locale

View File

@@ -16,108 +16,22 @@
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnectorFactory.ConnectorMode
import org.apache.commons.lang3.ObjectUtils.firstNonNull
import java.net.URI
import java.time.Duration
import java.util.Arrays
import java.util.Locale
import java.util.concurrent.ConcurrentHashMap
open class UpstreamsConfig {
var defaultOptions: MutableList<DefaultOptions> = ArrayList<DefaultOptions>()
var defaultOptions: MutableList<ChainOptions.DefaultOptions> = ArrayList()
var upstreams: MutableList<Upstream<*>> = ArrayList<Upstream<*>>()
data class Options(
val disableUpstreamValidation: Boolean,
val disableValidation: Boolean,
val validationInterval: Int,
val timeout: Duration,
val providesBalance: Boolean?,
val validatePeers: Boolean,
val minPeers: Int,
val validateSyncing: Boolean,
val validateCallLimit: Boolean,
val validateChain: Boolean
)
open class PartialOptions {
var disableValidation: Boolean? = null
var disableUpstreamValidation: Boolean? = null
var validationInterval: Int? = null
set(value) {
require(value == null || value > 0) {
"validation-interval must be a positive number: $value"
}
field = value
}
var timeout: Duration? = null
var providesBalance: Boolean? = null
var validatePeers: Boolean? = null
var validateCalllimit: Boolean? = null
var minPeers: Int? = null
set(value) {
require(value == null || value >= 0) {
"min-peers must be a positive number: $value"
}
field = value
}
var validateSyncing: Boolean? = null
var validateChain: Boolean? = null
fun merge(overwrites: PartialOptions?): PartialOptions {
if (overwrites == null) {
return this
}
val copy = PartialOptions()
copy.validatePeers = firstNonNull(overwrites.validatePeers, this.validatePeers)
copy.minPeers = firstNonNull(overwrites.minPeers, this.minPeers)
copy.disableValidation = firstNonNull(overwrites.disableValidation, this.disableValidation)
copy.validationInterval = firstNonNull(overwrites.validationInterval, this.validationInterval)
copy.providesBalance = firstNonNull(overwrites.providesBalance, this.providesBalance)
copy.validateSyncing = firstNonNull(overwrites.validateSyncing, this.validateSyncing)
copy.validateCalllimit = firstNonNull(overwrites.validateCalllimit, this.validateCalllimit)
copy.timeout = firstNonNull(overwrites.timeout, this.timeout)
copy.validateChain = firstNonNull(overwrites.validateChain, this.validateChain)
copy.disableUpstreamValidation = firstNonNull(overwrites.disableUpstreamValidation, this.disableUpstreamValidation)
return copy
}
fun buildOptions(): Options =
Options(
firstNonNull(this.disableUpstreamValidation, false)!!,
firstNonNull(this.disableValidation, false)!!,
firstNonNull(this.validationInterval, 30)!!,
firstNonNull(this.timeout, Defaults.timeout)!!,
this.providesBalance,
firstNonNull(this.validatePeers, true)!!,
firstNonNull(this.minPeers, 1)!!,
firstNonNull(this.validateSyncing, true)!!,
firstNonNull(this.validateCalllimit, true)!!,
firstNonNull(this.validateChain, true)!!
)
companion object {
@JvmStatic
fun getDefaults(): PartialOptions {
val options = PartialOptions()
options.minPeers = 1
return options
}
}
}
class DefaultOptions : PartialOptions() {
var chains: List<String>? = null
var options: PartialOptions? = null
}
class Upstream<T : UpstreamConnection> {
var id: String? = null
var nodeId: Int? = null
var chain: String? = null
var options: PartialOptions? = null
var options: ChainOptions.PartialOptions? = null
var isEnabled = true
var connection: T? = null
val labels = Labels()
@@ -137,7 +51,7 @@ open class UpstreamsConfig {
enum class UpstreamRole {
PRIMARY,
SECONDARY,
FALLBACK
FALLBACK,
}
open class UpstreamConnection
@@ -185,7 +99,7 @@ open class UpstreamsConfig {
data class BitcoinZeroMq(
val host: String = "127.0.0.1",
val port: Int
val port: Int,
)
class HttpEndpoint(val url: URI) {
@@ -220,7 +134,8 @@ open class UpstreamsConfig {
ETHEREUM_JSON_RPC("ethereum"),
BITCOIN_JSON_RPC("bitcoin"),
DSHACKLE("dshackle", "grpc"),
UNKNOWN("unknown");
UNKNOWN("unknown"),
;
private val code: Array<out String>
@@ -256,6 +171,6 @@ open class UpstreamsConfig {
class Method(
val name: String,
val quorum: String? = null,
val static: String? = null
val static: String? = null,
)
}

View File

@@ -17,16 +17,19 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.FileResolver
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.foundation.ChainOptionsReader
import io.emeraldpay.dshackle.foundation.YamlConfigReader
import org.apache.commons.lang3.StringUtils
import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.nodes.MappingNode
import java.io.InputStream
import java.net.URI
import java.time.Duration
import java.util.Locale
class UpstreamsConfigReader(
private val fileResolver: FileResolver
private val fileResolver: FileResolver,
private val optionsReader: ChainOptionsReader,
) : YamlConfigReader<UpstreamsConfig>() {
private val log = LoggerFactory.getLogger(UpstreamsConfigReader::class.java)
@@ -39,28 +42,28 @@ class UpstreamsConfigReader(
}
}
fun readInternal(input: InputStream): UpstreamsConfig? {
fun readInternal(input: InputStream): UpstreamsConfig {
val configNode = readNode(input)
return readInternal(configNode)
}
fun readInternal(input: MappingNode?): UpstreamsConfig {
private fun readInternal(input: MappingNode?): UpstreamsConfig {
val config = UpstreamsConfig()
getList<MappingNode>(input, "defaults")?.value?.forEach { opts ->
val defaultOptions = UpstreamsConfig.DefaultOptions()
val defaultOptions = ChainOptions.DefaultOptions()
config.defaultOptions.add(defaultOptions)
defaultOptions.chains = getListOfString(opts, "chains")
getMapping(opts, "options")?.let { values ->
defaultOptions.options = readOptions(values)
defaultOptions.options = optionsReader.readOptions(values)
}
}
config.upstreams = ArrayList<UpstreamsConfig.Upstream<*>>()
config.upstreams = ArrayList()
getValueAsString(input, "include")?.let { path ->
fileResolver.resolve(path).let { file ->
if (file.exists() && file.isFile && file.canRead()) {
readInternal(file.inputStream())?.let {
readInternal(file.inputStream()).let {
it.upstreams.forEach { upstream -> config.upstreams.add(upstream) }
}
} else {
@@ -72,7 +75,7 @@ class UpstreamsConfigReader(
getListOfString(input, "include")?.forEach { path ->
fileResolver.resolve(path).let { file ->
if (file.exists() && file.isFile && file.canRead()) {
readInternal(file.inputStream())?.let {
readInternal(file.inputStream()).let {
it.upstreams.forEach { upstream -> config.upstreams.add(upstream) }
}
} else {
@@ -222,7 +225,7 @@ class UpstreamsConfigReader(
private fun <T : UpstreamsConfig.UpstreamConnection> readUpstream(
config: UpstreamsConfig,
upNode: MappingNode,
connFactory: () -> T
connFactory: () -> T,
) {
val upstream = UpstreamsConfig.Upstream<T>()
readUpstreamCommon(upNode, upstream)
@@ -256,10 +259,10 @@ class UpstreamsConfigReader(
} ?: true
}
internal fun readUpstreamCommon(upNode: MappingNode, upstream: UpstreamsConfig.Upstream<*>) {
private fun readUpstreamCommon(upNode: MappingNode, upstream: UpstreamsConfig.Upstream<*>) {
upstream.id = getValueAsString(upNode, "id")
upstream.nodeId = getValueAsInt(upNode, "node-id")
upstream.options = tryReadOptions(upNode)
upstream.options = optionsReader.read(upNode)
upstream.methods = tryReadMethods(upNode)
upstream.methodGroups = tryReadMethodGroups(upNode)
getValueAsBool(upNode, "enabled")?.let {
@@ -277,15 +280,15 @@ class UpstreamsConfigReader(
}
}
internal fun readUpstreamGrpc(
upNode: MappingNode
private fun readUpstreamGrpc(
upNode: MappingNode,
) {
if (hasAny(upNode, "chain")) {
log.warn("Chain should be not applied to gRPC upstream")
}
}
internal fun readUpstreamStandard(upNode: MappingNode, upstream: UpstreamsConfig.Upstream<*>) {
private fun readUpstreamStandard(upNode: MappingNode, upstream: UpstreamsConfig.Upstream<*>) {
upstream.chain = getValueAsString(upNode, "chain")
getValueAsString(upNode, "role")?.let {
val name = it.trim().let {
@@ -301,80 +304,38 @@ class UpstreamsConfigReader(
}
}
internal fun tryReadOptions(upNode: MappingNode): UpstreamsConfig.PartialOptions? {
return if (hasAny(upNode, "options")) {
return getMapping(upNode, "options")?.let { values ->
readOptions(values)
}
} else {
null
}
}
internal fun tryReadMethods(upNode: MappingNode): UpstreamsConfig.Methods? {
private fun tryReadMethods(upNode: MappingNode): UpstreamsConfig.Methods? {
return getMapping(upNode, "methods")?.let { mnode ->
val enabled = getList<MappingNode>(mnode, "enabled")?.value?.map { m ->
val enabled = getList<MappingNode>(mnode, "enabled")?.value?.mapNotNull { m ->
getValueAsString(m, "name")?.let { name ->
UpstreamsConfig.Method(
name = name,
quorum = getValueAsString(m, "quorum"),
static = getValueAsString(m, "static")
static = getValueAsString(m, "static"),
)
}
}?.filterNotNull()?.toSet() ?: emptySet()
val disabled = getList<MappingNode>(mnode, "disabled")?.value?.map { m ->
}?.toSet() ?: emptySet()
val disabled = getList<MappingNode>(mnode, "disabled")?.value?.mapNotNull { m ->
getValueAsString(m, "name")?.let { name ->
UpstreamsConfig.Method(
name = name
name = name,
)
}
}?.filterNotNull()?.toSet() ?: emptySet()
}?.toSet() ?: emptySet()
UpstreamsConfig.Methods(
enabled,
disabled
disabled,
)
}
}
internal fun tryReadMethodGroups(upNode: MappingNode): UpstreamsConfig.MethodGroups? {
private fun tryReadMethodGroups(upNode: MappingNode): UpstreamsConfig.MethodGroups? {
return getMapping(upNode, "method-groups")?.let {
UpstreamsConfig.MethodGroups(
enabled = getListOfString(it, "enabled")?.toSet() ?: emptySet(),
disabled = getListOfString(it, "disabled")?.toSet() ?: emptySet()
disabled = getListOfString(it, "disabled")?.toSet() ?: emptySet(),
)
}
}
internal fun readOptions(values: MappingNode): UpstreamsConfig.PartialOptions {
val options = UpstreamsConfig.PartialOptions()
getValueAsBool(values, "validate-peers")?.let {
options.validatePeers = it
}
getValueAsBool(values, "validate-syncing")?.let {
options.validateSyncing = it
}
getValueAsBool(values, "validate-call-limit")?.let {
options.validateCalllimit = it
}
getValueAsBool(values, "validate-chain")?.let {
options.validateChain = it
}
getValueAsInt(values, "min-peers")?.let {
options.minPeers = it
}
getValueAsInt(values, "timeout")?.let {
options.timeout = Duration.ofSeconds(it.toLong())
}
getValueAsBool(values, "disable-validation")?.let {
options.disableValidation = it
}
getValueAsInt(values, "validation-interval")?.let {
options.validationInterval = it
}
getValueAsBool(values, "balance")?.let {
options.providesBalance = it
}
return options
}
}

View File

@@ -27,6 +27,7 @@ import io.emeraldpay.dshackle.config.AuthorizationConfig
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.CompressionConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.CallTargetsHolder
@@ -88,7 +89,7 @@ open class ConfiguredUpstreams(
@Qualifier("headScheduler")
private val headScheduler: Scheduler,
private val authorizationConfig: AuthorizationConfig,
private val grpcAuthContext: GrpcAuthContext
private val grpcAuthContext: GrpcAuthContext,
) : ApplicationRunner {
@Value("\${spring.application.max-metadata-size}")
private var maxMetadataSize: Int = Defaults.maxMetadataSize
@@ -109,7 +110,7 @@ open class ConfiguredUpstreams(
}
log.debug("Start upstream ${up.id}")
if (up.connection is UpstreamsConfig.GrpcConnection) {
val options = up.options ?: UpstreamsConfig.PartialOptions()
val options = up.options ?: ChainOptions.PartialOptions()
buildGrpcUpstream(up.nodeId, up.cast(UpstreamsConfig.GrpcConnection::class.java), options.buildOptions(), compressionConfig.grpc.clientEnabled)
} else {
val chain = Global.chainById(up.chain)
@@ -117,10 +118,10 @@ open class ConfiguredUpstreams(
log.error("Chain is unknown: ${up.chain}")
return@forEach
}
val chainConfig = chainsConfig.resolve(chain)
val chainConfig = chainsConfig.resolve(up.chain!!)
val options = chainConfig.options
.merge(defaultOptions[chain] ?: UpstreamsConfig.PartialOptions.getDefaults())
.merge(up.options ?: UpstreamsConfig.PartialOptions())
.merge(defaultOptions[chain] ?: ChainOptions.PartialOptions.getDefaults())
.merge(up.options ?: ChainOptions.PartialOptions())
.buildOptions()
val upstream = when (BlockchainType.from(chain)) {
BlockchainType.EVM_POS -> {
@@ -129,7 +130,7 @@ open class ConfiguredUpstreams(
up.cast(UpstreamsConfig.EthereumPosConnection::class.java),
chain,
options,
chainConfig
chainConfig,
)
}
BlockchainType.EVM_POW -> {
@@ -138,7 +139,7 @@ open class ConfiguredUpstreams(
up.cast(UpstreamsConfig.EthereumConnection::class.java),
chain,
options,
chainConfig
chainConfig,
)
}
BlockchainType.BITCOIN -> {
@@ -146,7 +147,7 @@ open class ConfiguredUpstreams(
up.cast(UpstreamsConfig.BitcoinConnection::class.java),
chain,
options,
chainConfig
chainConfig,
)
}
}
@@ -158,8 +159,8 @@ open class ConfiguredUpstreams(
}
}
private fun buildDefaultOptions(config: UpstreamsConfig): HashMap<Chain, UpstreamsConfig.PartialOptions> {
val defaultOptions = HashMap<Chain, UpstreamsConfig.PartialOptions>()
private fun buildDefaultOptions(config: UpstreamsConfig): HashMap<Chain, ChainOptions.PartialOptions> {
val defaultOptions = HashMap<Chain, ChainOptions.PartialOptions>()
config.defaultOptions.forEach { defaultsConfig ->
defaultsConfig.chains?.forEach { chainName ->
Global.chainById(chainName).let { chain ->
@@ -183,7 +184,7 @@ open class ConfiguredUpstreams(
enabled = config.methods?.enabled?.map { it.name }?.toSet() ?: emptySet(),
disabled = config.methods?.disabled?.map { it.name }?.toSet() ?: emptySet(),
groupsEnabled = config.methodGroups?.enabled ?: emptySet(),
groupsDisabled = config.methodGroups?.disabled ?: emptySet()
groupsDisabled = config.methodGroups?.disabled ?: emptySet(),
).also {
config.methods?.enabled?.forEach { m ->
if (m.quorum != null) {
@@ -203,8 +204,8 @@ open class ConfiguredUpstreams(
nodeId: Int?,
config: UpstreamsConfig.Upstream<UpstreamsConfig.EthereumPosConnection>,
chain: Chain,
options: UpstreamsConfig.Options,
chainConf: ChainsConfig.ChainConfig
options: ChainOptions.Options,
chainConf: ChainsConfig.ChainConfig,
): Upstream? {
val conn = config.connection!!
val execution = conn.execution
@@ -220,7 +221,7 @@ open class ConfiguredUpstreams(
urls,
NoChoiceWithPriorityForkChoice(conn.upstreamRating, config.id!!),
BlockValidator.ALWAYS_VALID,
chainConf
chainConf,
)
val methods = buildMethods(config, chain)
if (connectorFactory == null) {
@@ -242,7 +243,7 @@ open class ConfiguredUpstreams(
connectorFactory,
chainConf,
true,
eventPublisher
eventPublisher,
)
upstream.start()
if (!upstream.isRunning) {
@@ -255,8 +256,8 @@ open class ConfiguredUpstreams(
private fun buildBitcoinUpstream(
config: UpstreamsConfig.Upstream<UpstreamsConfig.BitcoinConnection>,
chain: Chain,
options: UpstreamsConfig.Options,
chainConf: ChainsConfig.ChainConfig
options: ChainOptions.Options,
chainConf: ChainsConfig.ChainConfig,
): Upstream? {
val conn = config.connection!!
val httpFactory = buildHttpFactory(conn)
@@ -289,7 +290,7 @@ open class ConfiguredUpstreams(
chain, directApi, head,
options, config.role,
QuorumForLabels.QuorumItem(1, config.labels),
methods, esplora, chainConf
methods, esplora, chainConf,
)
upstream.start()
return upstream
@@ -299,8 +300,8 @@ open class ConfiguredUpstreams(
nodeId: Int?,
config: UpstreamsConfig.Upstream<UpstreamsConfig.EthereumConnection>,
chain: Chain,
options: UpstreamsConfig.Options,
chainConf: ChainsConfig.ChainConfig
options: ChainOptions.Options,
chainConf: ChainsConfig.ChainConfig,
): Upstream? {
val conn = config.connection!!
@@ -314,7 +315,7 @@ open class ConfiguredUpstreams(
urls,
MostWorkForkChoice(),
EthereumBlockValidator(),
chainConf
chainConf,
)
if (connectorFactory == null) {
return null
@@ -331,7 +332,7 @@ open class ConfiguredUpstreams(
connectorFactory,
chainConf,
false,
eventPublisher
eventPublisher,
)
upstream.start()
return upstream
@@ -340,13 +341,13 @@ open class ConfiguredUpstreams(
private fun buildGrpcUpstream(
nodeId: Int?,
config: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>,
options: UpstreamsConfig.Options,
compression: Boolean
options: ChainOptions.Options,
compression: Boolean,
) {
if (!this::grpcUpstreamsScheduler.isInitialized) {
grpcUpstreamsScheduler = Schedulers.fromExecutorService(
Executors.newFixedThreadPool(2),
"GrpcUpstreamsStatuses"
"GrpcUpstreamsStatuses",
)
}
val endpoint = config.connection!!
@@ -371,7 +372,7 @@ open class ConfiguredUpstreams(
clientSpansInterceptor,
maxMetadataSize,
headScheduler,
grpcAuthContext
grpcAuthContext,
).apply {
timeout = options.timeout
}
@@ -399,14 +400,15 @@ open class ConfiguredUpstreams(
id: String,
chain: Chain,
conn: UpstreamsConfig.EthereumConnection,
urls: ArrayList<URI>? = null
urls: ArrayList<URI>? = null,
): EthereumWsConnectionPoolFactory? {
return conn.ws?.let { endpoint ->
val wsConnectionFactory = EthereumWsConnectionFactory(
id, chain,
id,
chain,
endpoint.url,
endpoint.origin ?: URI("http://localhost"),
headScheduler
headScheduler,
).apply {
config = endpoint
basicAuth = endpoint.basicAuth
@@ -414,7 +416,7 @@ open class ConfiguredUpstreams(
val wsApi = EthereumWsConnectionPoolFactory(
id,
endpoint.connections,
wsConnectionFactory
wsConnectionFactory,
)
urls?.add(endpoint.url)
wsApi
@@ -428,7 +430,7 @@ open class ConfiguredUpstreams(
urls: ArrayList<URI>,
forkChoice: ForkChoice,
blockValidator: BlockValidator,
chainsConf: ChainsConfig.ChainConfig
chainsConf: ChainsConfig.ChainConfig,
): EthereumConnectorFactory? {
val wsFactoryApi = buildWsFactory(id, chain, conn, urls)
val httpFactory = buildHttpFactory(conn, urls)
@@ -442,7 +444,7 @@ open class ConfiguredUpstreams(
blockValidator,
wsConnectionResubscribeScheduler,
headScheduler,
chainsConf.expectedBlockTime
chainsConf.expectedBlockTime,
)
if (!connectorFactory.isValid()) {
log.warn("Upstream configuration is invalid (probably no http endpoint)")

View File

@@ -19,6 +19,7 @@ package io.emeraldpay.dshackle.upstream
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import org.slf4j.LoggerFactory
@@ -31,21 +32,21 @@ abstract class DefaultUpstream(
private val hash: Byte,
defaultLag: Long?,
defaultAvail: UpstreamAvailability,
private val options: UpstreamsConfig.Options,
private val options: ChainOptions.Options,
private val role: UpstreamsConfig.UpstreamRole,
private val targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem?,
private val chainConfig: ChainsConfig.ChainConfig
private val chainConfig: ChainsConfig.ChainConfig,
) : Upstream {
constructor(
id: String,
hash: Byte,
options: UpstreamsConfig.Options,
options: ChainOptions.Options,
role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?,
node: QuorumForLabels.QuorumItem?,
chainConfig: ChainsConfig.ChainConfig
chainConfig: ChainsConfig.ChainConfig,
) :
this(id, hash, null, UpstreamAvailability.UNAVAILABLE, options, role, targets, node, chainConfig)
@@ -82,7 +83,7 @@ abstract class DefaultUpstream(
Status(curr.lag, avail, statusByLag(curr.lag, avail))
}.also {
statusStream.emitNext(
it.status
it.status,
) { _, res -> res == Sinks.EmitResult.FAIL_NON_SERIALIZED }
log.trace("Status of upstream [$id] changed to [$it], requested change status to [$avail]")
}
@@ -96,7 +97,9 @@ abstract class DefaultUpstream(
lag > chainConfig.laggingLagSize -> UpstreamAvailability.LAGGING
else -> proposed
}
} else proposed
} else {
proposed
}
}
override fun observeStatus(): Flux<UpstreamAvailability> {
@@ -109,7 +112,7 @@ abstract class DefaultUpstream(
Status(nLag, curr.avail, statusByLag(nLag, curr.avail))
}.also {
statusStream.emitNext(
it.status
it.status,
) { _, res -> res == Sinks.EmitResult.FAIL_NON_SERIALIZED }
log.trace("Status of upstream [$id] changed to [$it], requested change lag to [$lag]")
}
@@ -120,7 +123,7 @@ abstract class DefaultUpstream(
return this.status.get().lag
}
override fun getOptions(): UpstreamsConfig.Options {
override fun getOptions(): ChainOptions.Options {
return options
}

View File

@@ -47,7 +47,7 @@ class FilteredApis(
* Limit of retries
*/
private val retryLimit: Long,
jitter: Int
jitter: Int,
) : ApiSource {
private val internalMatcher: Selector.Matcher
@@ -77,13 +77,13 @@ class FilteredApis(
chain: Chain,
allUpstreams: List<Upstream>,
matcher: Selector.Matcher,
pos: Int
pos: Int,
) : this(chain, allUpstreams, matcher, pos, 10, 7)
constructor(
chain: Chain,
allUpstreams: List<Upstream>,
matcher: Selector.Matcher
matcher: Selector.Matcher,
) : this(chain, allUpstreams, matcher, 0, 10, 10)
private val delay: Int
@@ -132,7 +132,7 @@ class FilteredApis(
}
}
internalMatcher = Selector.MultiMatcher(
listOf(Selector.AvailabilityMatcher(), matcher)
listOf(Selector.AvailabilityMatcher(), matcher),
)
}
@@ -158,7 +158,7 @@ class FilteredApis(
val n = max(rawn, 1)
val time = min(
(n.toDouble().pow(2.0) * delay).roundToLong(),
MAX_WAIT_MILLIS
MAX_WAIT_MILLIS,
)
return Duration.ofMillis(time)
}

View File

@@ -20,6 +20,7 @@ import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.startup.UpstreamChangeEvent
@@ -64,6 +65,7 @@ abstract class Multistream(
private var cacheSubscription: Disposable? = null
private val reconfigLock = ReentrantLock()
private val eventLock = ReentrantLock()
@Volatile
private var callMethods: CallMethods? = null
private var callMethodsFactory: Factory<CallMethods> = Factory {
@@ -73,8 +75,10 @@ abstract class Multistream(
private var seq = 0
protected var lagObserver: HeadLagObserver? = null
private var subscription: Disposable? = null
@Volatile
private var capabilities: Set<Capability> = emptySet()
@Volatile
private var quorumLabels: List<QuorumForLabels.QuorumItem>? = null
private val removed: MutableMap<String, Upstream> = HashMap()
@@ -94,7 +98,7 @@ abstract class Multistream(
Metrics.gauge(
"$metrics.availability",
listOf(Tag.of("chain", chain.chainCode), Tag.of("status", status.name.lowercase())),
this
this,
) {
upstreams.count { it.getStatus() == status }.toDouble()
}
@@ -102,7 +106,8 @@ abstract class Multistream(
Metrics.gauge(
"$metrics.connected",
listOf(Tag.of("chain", chain.chainCode)), this
listOf(Tag.of("chain", chain.chainCode)),
this,
) {
upstreams.size.toDouble()
}
@@ -254,13 +259,13 @@ abstract class Multistream(
val upstreamsFluxes = getAll().map { up ->
Flux.concat(
Mono.just(up.getStatus()),
up.observeStatus()
up.observeStatus(),
).map { UpstreamStatus(up, it) }
}
val onShutdown = stopSignal.asFlux().map { UpstreamAvailability.UNAVAILABLE }
return Flux.merge(
Flux.merge(upstreamsFluxes).map(FilterBestAvailability()).takeUntilOther(stopSignal.asFlux()),
onShutdown
onShutdown,
).distinct()
}
@@ -270,12 +275,15 @@ abstract class Multistream(
override fun getStatus(): UpstreamAvailability {
val upstreams = getAll()
return if (upstreams.isEmpty()) UpstreamAvailability.UNAVAILABLE
else upstreams.minOf { it.getStatus() }
return if (upstreams.isEmpty()) {
UpstreamAvailability.UNAVAILABLE
} else {
upstreams.minOf { it.getStatus() }
}
}
// TODO options for multistream are useless
override fun getOptions(): UpstreamsConfig.Options {
override fun getOptions(): ChainOptions.Options {
throw IllegalStateException("Options are not supported for multistream")
}
@@ -314,7 +322,7 @@ abstract class Multistream(
upstream.observeStatus().map { upstream }
.takeUntilOther(
subscribeRemovedUpstreams()
.filter { it.getId() == upstream.getId() }
.filter { it.getId() == upstream.getId() },
)
}
.subscribe {

View File

@@ -61,19 +61,19 @@ class Selector {
Collections.unmodifiableCollection(
req.andSelector.selectorsList.map {
convertToMatcher(
it
it,
)
}
)
},
),
)
req.hasOrSelector() -> OrMatcher(
Collections.unmodifiableCollection(
req.orSelector.selectorsList.map {
convertToMatcher(
it
it,
)
}
)
},
),
)
req.hasNotSelector() -> NotMatcher(convertToMatcher(req.notSelector.selector))
req.hasExistsSelector() -> ExistsMatcher(req.existsSelector.name)
@@ -138,7 +138,7 @@ class Selector {
}
data class MultiMatcher(
private val matchers: Collection<Matcher>
private val matchers: Collection<Matcher>,
) : Matcher() {
override fun matchesWithCause(up: Upstream): MatchesResponse {
@@ -147,7 +147,7 @@ class Selector {
Success
} else {
MatchesResponse.MultiResponse(
responses.filter { it !is Success }.toSet()
responses.filter { it !is Success }.toSet(),
)
}
}
@@ -175,7 +175,7 @@ class Selector {
}
data class MethodMatcher(
val method: String
val method: String,
) : Matcher() {
override fun matchesWithCause(up: Upstream): MatchesResponse =
@@ -202,7 +202,7 @@ class Selector {
Success
} else {
MatchesResponse.MultiResponse(
labelsResponses.filter { it !is Success }.toSet()
labelsResponses.filter { it !is Success }.toSet(),
)
}
}
@@ -243,7 +243,7 @@ class Selector {
class LabelMatcher(
val name: String,
val values: Collection<String>
val values: Collection<String>,
) : LabelSelectorMatcher() {
override fun matchesWithCause(labels: UpstreamsConfig.Labels): MatchesResponse {
@@ -254,7 +254,8 @@ class Selector {
Success
} else {
MatchesResponse.LabelResponse(
name, values
name,
values,
)
}
}
@@ -263,7 +264,7 @@ class Selector {
return BlockchainOuterClass.Selector.newBuilder().setLabelSelector(
BlockchainOuterClass.LabelSelector.newBuilder()
.setName(name)
.addAllValue(values)
.addAllValue(values),
).build()
}
@@ -286,7 +287,7 @@ class Selector {
Success
} else {
MatchesResponse.MultiResponse(
responses.filter { it !is Success }.toSet()
responses.filter { it !is Success }.toSet(),
)
}
}
@@ -295,7 +296,7 @@ class Selector {
return BlockchainOuterClass.Selector.newBuilder().setOrSelector(
BlockchainOuterClass.OrSelector.newBuilder()
.addAllSelectors(matchers.map { it.asProto() })
.build()
.build(),
).build()
}
@@ -309,7 +310,7 @@ class Selector {
}
class AndMatcher(
val matchers: Collection<LabelSelectorMatcher>
val matchers: Collection<LabelSelectorMatcher>,
) : LabelSelectorMatcher() {
override fun matchesWithCause(labels: UpstreamsConfig.Labels): MatchesResponse {
@@ -318,7 +319,7 @@ class Selector {
Success
} else {
MatchesResponse.MultiResponse(
responses.filter { it !is Success }.toSet()
responses.filter { it !is Success }.toSet(),
)
}
}
@@ -327,7 +328,7 @@ class Selector {
return BlockchainOuterClass.Selector.newBuilder().setAndSelector(
BlockchainOuterClass.AndSelector.newBuilder()
.addAllSelectors(matchers.map { it.asProto() })
.build()
.build(),
).build()
}
@@ -341,7 +342,7 @@ class Selector {
}
class NotMatcher(
val matcher: LabelSelectorMatcher
val matcher: LabelSelectorMatcher,
) : LabelSelectorMatcher() {
override fun matchesWithCause(labels: UpstreamsConfig.Labels): MatchesResponse {
@@ -357,7 +358,7 @@ class Selector {
return BlockchainOuterClass.Selector.newBuilder().setNotSelector(
BlockchainOuterClass.NotSelector.newBuilder()
.setSelector(matcher.asProto())
.build()
.build(),
).build()
}
@@ -371,7 +372,7 @@ class Selector {
}
class ExistsMatcher(
val name: String
val name: String,
) : LabelSelectorMatcher() {
override fun matchesWithCause(labels: UpstreamsConfig.Labels): MatchesResponse =
@@ -385,7 +386,7 @@ class Selector {
return BlockchainOuterClass.Selector.newBuilder().setExistsSelector(
BlockchainOuterClass.ExistsSelector.newBuilder()
.setName(name)
.build()
.build(),
).build()
}

View File

@@ -17,6 +17,7 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import reactor.core.publisher.Flux
@@ -32,7 +33,7 @@ interface Upstream {
*/
fun getIngressReader(): JsonRpcReader
fun getOptions(): UpstreamsConfig.Options
fun getOptions(): ChainOptions.Options
fun getRole(): UpstreamsConfig.UpstreamRole
fun setLag(lag: Long)
fun getLag(): Long?

View File

@@ -88,14 +88,13 @@ open class BitcoinMultistream(
}
}
} else {
val newHead = MergedHead(sourceUpstreams.map { it.getHead() }, MostWorkForkChoice(), headScheduler).apply {
this.start()
}
val newHead = MergedHead(sourceUpstreams.map { it.getHead() }, MostWorkForkChoice(), headScheduler).apply { this.start() }
newHead
}
onHeadUpdated(head)
return head
}
/**
* Finds an API that executed directly on a remote.
*/

View File

@@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.Capability
@@ -33,12 +34,12 @@ open class BitcoinRpcUpstream(
chain: Chain,
private val directApi: JsonRpcReader,
private val head: Head,
options: UpstreamsConfig.Options,
options: ChainOptions.Options,
role: UpstreamsConfig.UpstreamRole,
node: QuorumForLabels.QuorumItem,
callMethods: CallMethods,
esploraClient: EsploraClient? = null,
chainConfig: ChainsConfig.ChainConfig
chainConfig: ChainsConfig.ChainConfig,
) : BitcoinUpstream(id, chain, options, role, callMethods, node, esploraClient, chainConfig), Lifecycle {
private var validatorSubscription: Disposable? = null

View File

@@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.calls.CallMethods
@@ -26,19 +27,19 @@ import io.emeraldpay.dshackle.upstream.calls.DefaultBitcoinMethods
abstract class BitcoinUpstream(
id: String,
val chain: Chain,
options: UpstreamsConfig.Options,
options: ChainOptions.Options,
role: UpstreamsConfig.UpstreamRole,
callMethods: CallMethods,
node: QuorumForLabels.QuorumItem,
val esploraClient: EsploraClient? = null,
chainConfig: ChainsConfig.ChainConfig
chainConfig: ChainsConfig.ChainConfig,
) : DefaultUpstream(id, 0.toByte(), options, role, callMethods, node, chainConfig) {
constructor(
id: String,
chain: Chain,
options: UpstreamsConfig.Options,
options: ChainOptions.Options,
role: UpstreamsConfig.UpstreamRole,
chainConfig: ChainsConfig.ChainConfig
chainConfig: ChainsConfig.ChainConfig,
) : this(id, chain, options, role, DefaultBitcoinMethods(), QuorumForLabels.QuorumItem.empty(), null, chainConfig)
}

View File

@@ -15,7 +15,7 @@
*/
package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
@@ -30,7 +30,7 @@ import java.util.concurrent.Executors
class BitcoinUpstreamValidator(
private val api: JsonRpcReader,
private val options: UpstreamsConfig.Options
private val options: ChainOptions.Options,
) {
companion object {

View File

@@ -24,7 +24,6 @@ import io.emeraldpay.dshackle.quorum.CallQuorum
import io.emeraldpay.dshackle.quorum.MaximumValueQuorum
import io.emeraldpay.dshackle.quorum.NotLaggingQuorum
import io.emeraldpay.dshackle.quorum.NotNullQuorum
import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods.HardcodedData.Companion.createHardcodedData
import io.emeraldpay.etherjar.rpc.RpcException
/**
@@ -32,7 +31,7 @@ import io.emeraldpay.etherjar.rpc.RpcException
* hardcoded results for base methods, such as `net_version`, `web3_clientVersion` and similar
*/
class DefaultEthereumMethods(
private val chain: Chain
private val chain: Chain,
) : CallMethods {
private val version = "\"EmeraldDshackle/${Global.version}\""
@@ -41,7 +40,7 @@ class DefaultEthereumMethods(
val withFilterIdMethods = listOf(
"eth_getFilterChanges",
"eth_getFilterLogs",
"eth_uninstallFilter"
"eth_uninstallFilter",
)
val newFilterMethods = listOf(
@@ -69,79 +68,13 @@ class DefaultEthereumMethods(
"debug_traceBlockByNumber",
"debug_traceCall",
"debug_traceCallMany",
"debug_traceTransaction"
"debug_traceTransaction",
)
val CHAIN_DATA = mapOf(
Chain.ETHEREUM__MAINNET to createHardcodedData("\"1\"", "\"0x1\""),
Chain.ETHEREUM__RINKEBY to createHardcodedData("\"4\"", "\"0x4\""),
Chain.ETHEREUM__ROPSTEN to createHardcodedData("\"3\"", "\"0x3\""),
Chain.ETHEREUM__KOVAN to createHardcodedData("\"42\"", "\"0x2a\""),
Chain.ETHEREUM__GOERLI to createHardcodedData("\"5\"", "\"0x5\""),
Chain.ETHEREUM__SEPOLIA to createHardcodedData("\"11155111\"", "\"0xaa36a7\""),
Chain.ETHEREUM__HOLESKY to createHardcodedData("\"17000\"", "\"0x4268\""),
Chain.ETHEREUM_CLASSIC__MAINNET to createHardcodedData("\"1\"", "\"0x3d\""),
Chain.POLYGON_POS__MAINNET to createHardcodedData("\"137\"", "\"0x89\""),
Chain.POLYGON_POS__MUMBAI to createHardcodedData("\"80001\"", "\"0x13881\""),
Chain.ARBITRUM__MAINNET to createHardcodedData("\"42161\"", "\"0xa4b1\""),
Chain.ARBITRUM__GOERLI to createHardcodedData("\"421613\"", "\"0x66eed\""),
Chain.OPTIMISM__MAINNET to createHardcodedData("\"10\"", "\"0xa\""),
Chain.OPTIMISM__GOERLI to createHardcodedData("\"420\"", "\"0x1A4\""),
Chain.ARBITRUM_NOVA__MAINNET to createHardcodedData("\"42170\"", "\"0xa4ba\""),
Chain.POLYGON_ZKEVM__MAINNET to createHardcodedData("\"1101\"", "\"0x44d\""),
Chain.POLYGON_ZKEVM__TESTNET to createHardcodedData("\"1442\"", "\"0x5a2\""),
Chain.ZKSYNC__MAINNET to createHardcodedData("\"324\"", "\"0x144\""),
Chain.ZKSYNC__TESTNET to createHardcodedData("\"280\"", "\"0x118\""),
Chain.BSC__MAINNET to createHardcodedData("\"56\"", "\"0x38\""),
Chain.BSC__TESTNET to createHardcodedData("\"97\"", "\"0x61\""),
Chain.BASE__MAINNET to createHardcodedData("\"8453\"", "\"0x2105\""),
Chain.BASE__GOERLI to createHardcodedData("\"84531\"", "\"0x14a33\""),
Chain.LINEA__MAINNET to createHardcodedData("\"59144\"", "\"0xe708\""),
Chain.LINEA__GOERLI to createHardcodedData("\"59140\"", "\"0xe704\""),
Chain.FANTOM__MAINNET to createHardcodedData("\"250\"", "\"0xfa\""),
Chain.FANTOM__TESTNET to createHardcodedData("\"4002\"", "\"0xfa2\""),
Chain.GNOSIS__MAINNET to createHardcodedData("\"100\"", "\"0x64\""),
Chain.GNOSIS__CHIADO to createHardcodedData("\"10200\"", "\"0x27d8\""),
Chain.AVALANCHE__MAINNET to createHardcodedData("\"43114\"", "\"0xa86a\""),
Chain.AVALANCHE__FUJI to createHardcodedData("\"43113\"", "\"0xa869\""),
Chain.AURORA__MAINNET to createHardcodedData("\"1313161554\"", "\"0x4e454152\""),
Chain.AURORA__TESTNET to createHardcodedData("\"1313161555\"", "\"0x4e454153\""),
// Chain.CHAIN_SCROLL__MAINNET to createHardcodedData(""43114"", ""0xa86a""), doesn't exist now for L2
Chain.SCROLL__ALPHANET to createHardcodedData("\"534353\"", "\"0x82751\""),
Chain.SCROLL__SEPOLIA to createHardcodedData("\"534351\"", "\"0x8274f\""),
Chain.MANTLE__MAINNET to createHardcodedData("\"5000\"", "\"0x1388\""),
Chain.MANTLE__TESTNET to createHardcodedData("\"5001\"", "\"0x1389\""),
Chain.KLAYTN__MAINNET to createHardcodedData("\"8217\"", "\"0x2019\""),
Chain.KLAYTN__BAOBAB to createHardcodedData("\"1001\"", "\"0x3e9\""),
Chain.MOONBEAM__MAINNET to createHardcodedData("\"1284\"", "\"0x504\""),
Chain.MOONBEAM__MOONRIVER to createHardcodedData("\"1285\"", "\"0x505\""),
Chain.MOONBEAM__ALPHA to createHardcodedData("\"1287\"", "\"0x507\""),
Chain.CELO__MAINNET to createHardcodedData("\"42220\"", "\"0xa4ec\""),
Chain.CELO__ALFAJORES to createHardcodedData("\"44787\"", "\"0xaef3\""),
)
fun getChainByData(data: HardcodedData) = CHAIN_DATA.entries.find { it.value == data }?.key
}
private val anyResponseMethods = listOf(
"eth_gasPrice",
"eth_estimateGas"
"eth_estimateGas",
)
private val possibleNotIndexedMethods = listOf(
@@ -153,7 +86,7 @@ class DefaultEthereumMethods(
"eth_getTransactionByBlockHashAndIndex",
"eth_getTransactionByBlockNumberAndIndex",
"eth_getUncleByBlockHashAndIndex",
"eth_getUncleCountByBlockHash"
"eth_getUncleCountByBlockHash",
)
private val firstValueMethods = listOf(
@@ -162,21 +95,21 @@ class DefaultEthereumMethods(
"eth_getCode",
"eth_getLogs",
"eth_maxPriorityFeePerGas",
"eth_getProof"
"eth_getProof",
)
private val specialMethods = listOf(
"eth_getTransactionCount",
"eth_blockNumber",
"eth_getBalance",
"eth_sendRawTransaction"
"eth_sendRawTransaction",
)
private val headVerifiedMethods = listOf(
"eth_getBlockTransactionCountByNumber",
"eth_getUncleCountByBlockNumber",
"eth_getUncleByBlockNumberAndIndex",
"eth_feeHistory"
"eth_feeHistory",
)
private val filterMethods = withFilterIdMethods + newFilterMethods
@@ -192,7 +125,7 @@ class DefaultEthereumMethods(
"eth_mining",
"eth_hashrate",
"eth_accounts",
"eth_chainId"
"eth_chainId",
)
private val allowedMethods: List<String>
@@ -244,16 +177,16 @@ class DefaultEthereumMethods(
private fun getChainSpecificMethods(chain: Chain): List<String> {
return when (chain) {
Chain.OPTIMISM__MAINNET, Chain.OPTIMISM__GOERLI -> listOf(
"rollup_gasPrices"
"rollup_gasPrices",
)
Chain.POLYGON_POS__MAINNET, Chain.POLYGON_POS__MUMBAI -> listOf(
Chain.POLYGON__MAINNET, Chain.POLYGON__MUMBAI -> listOf(
"bor_getAuthor",
"bor_getCurrentValidators",
"bor_getCurrentProposer",
"bor_getRootHash",
"bor_getSignersAtHash",
"eth_getRootHash"
"eth_getRootHash",
)
Chain.POLYGON_ZKEVM__MAINNET, Chain.POLYGON_ZKEVM__TESTNET -> listOf(
@@ -265,7 +198,7 @@ class DefaultEthereumMethods(
"zkevm_virtualBatchNumber",
"zkevm_verifiedBatchNumber",
"zkevm_getBatchByNumber",
"zkevm_getBroadcastURI"
"zkevm_getBroadcastURI",
)
Chain.ZKSYNC__MAINNET, Chain.ZKSYNC__TESTNET -> listOf(
@@ -286,7 +219,7 @@ class DefaultEthereumMethods(
"zks_getTokenPrice",
"zks_getTransactionDetails",
"zks_L1BatchNumber",
"zks_L1ChainId"
"zks_L1ChainId",
)
else -> emptyList()
@@ -314,8 +247,8 @@ class DefaultEthereumMethods(
// note that the value is in json representation, i.e. if it's a string it should be with quotes,
// that's why "\"0x0\"", "\"1\"", etc. But just "true" for a boolean, or "[]" for array.
val json = when (method) {
"net_version" -> CHAIN_DATA.get(chain)?.netVersion ?: throw RpcException(-32602, "Invalid chain")
"eth_chainId" -> CHAIN_DATA.get(chain)?.chainId ?: throw RpcException(-32602, "Invalid chain")
"net_version" -> "\"${chain.netVersion}\""
"eth_chainId" -> "\"${chain.chainId}\""
"net_peerCount" -> {
"\"0x2a\""
@@ -373,7 +306,7 @@ class DefaultEthereumMethods(
data class HardcodedData private constructor(
val netVersion: String,
val chainId: String
val chainId: String,
) {
companion object {
fun createHardcodedData(netVersion: String, chainId: String): HardcodedData =

View File

@@ -21,6 +21,7 @@ import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.startup.UpstreamChangeEvent
@@ -41,14 +42,14 @@ open class EthereumLikeRpcUpstream(
id: String,
hash: Byte,
val chain: Chain,
options: UpstreamsConfig.Options,
options: ChainOptions.Options,
role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem?,
connectorFactory: ConnectorFactory,
chainConfig: ChainsConfig.ChainConfig,
skipEnhance: Boolean,
private val eventPublisher: ApplicationEventPublisher?
private val eventPublisher: ApplicationEventPublisher?,
) : EthereumLikeUpstream(id, hash, options, role, targets, node, chainConfig), Lifecycle, Upstream, CachesEnabled {
private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(chain, this, getOptions(), chainConfig.callLimitContract)
protected val connector: EthereumConnector = connectorFactory.create(this, validator, chain, skipEnhance)

View File

@@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.DefaultUpstream
@@ -26,11 +27,11 @@ import io.emeraldpay.dshackle.upstream.calls.CallMethods
abstract class EthereumLikeUpstream(
id: String,
hash: Byte,
options: UpstreamsConfig.Options,
options: ChainOptions.Options,
role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem?,
val chainConfig: ChainsConfig.ChainConfig
val chainConfig: ChainsConfig.ChainConfig,
) : DefaultUpstream(id, hash, options, role, targets, node, chainConfig) {
private val capabilities = setOf(Capability.RPC, Capability.BALANCE)

View File

@@ -59,7 +59,7 @@ open class EthereumMultistream(
private var head: DynamicMergedHead = DynamicMergedHead(
PriorityForkChoice(),
"ETH Multistream of ${chain.chainCode}",
headScheduler
headScheduler,
)
private val filteredHeads: MutableMap<String, Head> =
@@ -71,7 +71,6 @@ open class EthereumMultistream(
private val supportsEIP1559set = setOf(
Chain.ETHEREUM__MAINNET, Chain.ETHEREUM__ROPSTEN,
Chain.ETHEREUM__GOERLI,
Chain.ETHEREUM__RINKEBY,
Chain.ETHEREUM__SEPOLIA,
Chain.ARBITRUM__MAINNET,
Chain.OPTIMISM__MAINNET,
@@ -81,13 +80,16 @@ open class EthereumMultistream(
Chain.POLYGON_ZKEVM__TESTNET,
Chain.ZKSYNC__MAINNET,
Chain.ZKSYNC__TESTNET,
Chain.ARBITRUM_NOVA__MAINNET
Chain.ARBITRUM_NOVA__MAINNET,
)
private val supportsEIP1559 = supportsEIP1559set.contains(chain)
private val feeEstimation = if (supportsEIP1559) EthereumPriorityFees(this, reader, 256)
else EthereumLegacyFees(this, reader, 256)
private val feeEstimation = if (supportsEIP1559) {
EthereumPriorityFees(this, reader, 256)
} else {
EthereumLegacyFees(this, reader, 256)
}
init {
this.init()
@@ -161,7 +163,7 @@ open class EthereumMultistream(
override fun tryProxy(
matcher: Selector.Matcher,
request: BlockchainOuterClass.NativeSubscribeRequest
request: BlockchainOuterClass.NativeSubscribeRequest,
): Flux<out Any>? =
upstreams.filter {
matcher.matches(it)
@@ -220,13 +222,15 @@ open class EthereumMultistream(
}.let {
val selected = it.map { source -> source.getHead() }
EnrichedMergedHead(
selected, getHead(), headScheduler,
selected,
getHead(),
headScheduler,
object :
Reader<BlockHash, BlockContainer> {
override fun read(key: BlockHash): Mono<BlockContainer> {
return reader.blocksByHashAsCont().read(key).map { res -> res.data }
}
}
},
)
}
}

View File

@@ -20,12 +20,9 @@ import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods
import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods.Companion.CHAIN_DATA
import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods.Companion.getChainByData
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.etherjar.domain.Address
@@ -46,8 +43,8 @@ import java.util.concurrent.TimeoutException
open class EthereumUpstreamValidator @JvmOverloads constructor(
private val chain: Chain,
private val upstream: Upstream,
private val options: UpstreamsConfig.Options,
private val callLimitContract: String? = null
private val options: ChainOptions.Options,
private val callLimitContract: String? = null,
) {
companion object {
private val log = LoggerFactory.getLogger(EthereumUpstreamValidator::class.java)
@@ -60,7 +57,7 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
open fun validate(): Mono<UpstreamAvailability> {
return Mono.zip(
validateSyncing(),
validatePeers()
validatePeers(),
)
.map(::resolve)
.defaultIfEmpty(UpstreamAvailability.UNAVAILABLE)
@@ -86,7 +83,7 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
.timeout(
Defaults.timeoutInternal,
Mono.fromCallable { log.warn("No response for eth_syncing from ${upstream.getId()}") }
.then(Mono.error(TimeoutException("Validation timeout for Syncing")))
.then(Mono.error(TimeoutException("Validation timeout for Syncing"))),
)
.map {
val isSyncing = it.isSyncing
@@ -113,7 +110,7 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
.timeout(
Defaults.timeoutInternal,
Mono.fromCallable { log.warn("No response for net_peerCount from ${upstream.getId()}") }
.then(Mono.error(TimeoutException("Validation timeout for Peers")))
.then(Mono.error(TimeoutException("Validation timeout for Peers"))),
)
.map { count ->
val minPeers = options.minPeers
@@ -144,7 +141,7 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
return Mono.zip(
validateChain(),
validateCallLimit(),
validateOldBlocks()
validateOldBlocks(),
).map {
it.t1 && it.t2 && it.t3
}.block() ?: false
@@ -156,19 +153,16 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
}
return Mono.zip(
chainId(),
netVersion()
netVersion(),
)
.map {
val chainData = CHAIN_DATA[chain] ?: return@map false
val isChainValid = chainData.chainId == it.t1 && chainData.netVersion == it.t2
val isChainValid = chain.chainId == it.t1 && chain.netVersion.toString() == it.t2
if (!isChainValid) {
val actualChain = getChainByData(
DefaultEthereumMethods.HardcodedData.createHardcodedData(it.t2, it.t1)
)?.chainName
val actualChain = Global.chainByChainId(it.t1).chainName
log.warn(
"${chain.chainName} is specified for upstream ${upstream.getId()} " +
"but actually it is $actualChain with chainId ${it.t1} and net_version ${it.t2}"
"but actually it is $actualChain with chainId ${it.t1} and net_version ${it.t2}",
)
}
@@ -193,11 +187,11 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
Address.from(callLimitContract),
// calling contract with param 200_000, meaning it will generate 200k symbols or response
// f4240 + metadata — ~1 million
HexData.from("0xd8a26e3a00000000000000000000000000000000000000000000000000000000000f4240")
HexData.from("0xd8a26e3a00000000000000000000000000000000000000000000000000000000000f4240"),
),
"latest"
)
)
"latest",
),
),
)
.flatMap(JsonRpcResponse::requireResult)
.map { true }
@@ -206,7 +200,7 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
log.warn(
"Error: ${it.message}. Node ${upstream.getId()} is probably incorrectly configured. " +
"You need to set up your return limit to at least 1_100_000. " +
"Erigon config example: https://github.com/ledgerwatch/erigon/blob/d014da4dc039ea97caf04ed29feb2af92b7b129d/cmd/utils/flags.go#L369"
"Erigon config example: https://github.com/ledgerwatch/erigon/blob/d014da4dc039ea97caf04ed29feb2af92b7b129d/cmd/utils/flags.go#L369",
)
Mono.just(false)
} else {
@@ -216,12 +210,12 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
.timeout(
Defaults.timeoutInternal,
Mono.fromCallable { log.error("No response for eth_call limit check from ${upstream.getId()}") }
.then(Mono.error(TimeoutException("Validation timeout for call limit")))
.then(Mono.error(TimeoutException("Validation timeout for call limit"))),
)
.retryRandomBackoff(3, Duration.ofMillis(100), Duration.ofMillis(500)) { ctx ->
log.warn(
"error during validateCallLimit for ${upstream.getId()}, iteration ${ctx.iteration()}, " +
"message ${ctx.exception().message}"
"message ${ctx.exception().message}",
)
}
.onErrorReturn(false)
@@ -238,14 +232,14 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
.retryRandomBackoff(3, Duration.ofMillis(100), Duration.ofMillis(500)) { ctx ->
log.warn(
"error during old block retrieving for ${upstream.getId()}, iteration ${ctx.iteration()}, " +
"message ${ctx.exception().message}"
"message ${ctx.exception().message}",
)
}
.map { result ->
val receivedResult = result.isNotEmpty() && !Global.nullValue.contentEquals(result)
if (!receivedResult) {
log.warn(
"Node ${upstream.getId()} probably is synced incorrectly, it is not possible to get old blocks"
"Node ${upstream.getId()} probably is synced incorrectly, it is not possible to get old blocks",
)
}
true
@@ -262,12 +256,11 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
.retryRandomBackoff(3, Duration.ofMillis(100), Duration.ofMillis(500)) { ctx ->
log.warn(
"error during chainId retrieving for ${upstream.getId()}, iteration ${ctx.iteration()}, " +
"message ${ctx.exception().message}"
"message ${ctx.exception().message}",
)
}
.doOnError { log.error("Error during execution 'eth_chainId' - ${it.message} for ${upstream.getId()}") }
.flatMap(JsonRpcResponse::requireResult)
.map { String(it) }
.flatMap(JsonRpcResponse::requireStringResult)
}
private fun netVersion(): Mono<String> {
@@ -276,11 +269,10 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
.retryRandomBackoff(3, Duration.ofMillis(100), Duration.ofMillis(500)) { ctx ->
log.warn(
"error during netVersion retrieving for ${upstream.getId()}, iteration ${ctx.iteration()}, " +
"message ${ctx.exception().message}"
"message ${ctx.exception().message}",
)
}
.doOnError { log.error("Error during execution 'net_version' - ${it.message} for ${upstream.getId()}") }
.flatMap(JsonRpcResponse::requireResult)
.map { String(it) }
.flatMap(JsonRpcResponse::requireStringResult)
}
}

View File

@@ -16,7 +16,7 @@ open class EthereumWsConnectionFactory(
private val chain: Chain,
private val uri: URI,
private val origin: URI,
private val scheduler: Scheduler
private val scheduler: Scheduler,
) {
var basicAuth: AuthConfig.ClientBasicAuth? = null
@@ -27,7 +27,7 @@ open class EthereumWsConnectionFactory(
Tag.of("index", connIndex.toString()),
Tag.of("upstream", id),
// UNSPECIFIED shouldn't happen too
Tag.of("chain", chain.chainCode)
Tag.of("chain", chain.chainCode),
)
return RpcMetrics(
@@ -39,7 +39,7 @@ open class EthereumWsConnectionFactory(
Counter.builder("upstream.ws.fail")
.description("Number of failures of WebSocket JSON RPC requests")
.tags(metricsTags)
.register(Metrics.globalRegistry)
.register(Metrics.globalRegistry),
)
}

View File

@@ -51,13 +51,13 @@ open class EthereumPosMultiStream(
val upstreams: MutableList<EthereumLikeUpstream>,
caches: Caches,
private val headScheduler: Scheduler,
tracer: Tracer
tracer: Tracer,
) : Multistream(chain, upstreams as MutableList<Upstream>, caches), EthereumLikeMultistream {
private var head: DynamicMergedHead = DynamicMergedHead(
PriorityForkChoice(),
"ETH Pos Multistream of ${chain.chainCode}",
headScheduler
headScheduler,
)
private val reader: EthereumCachingReader = EthereumCachingReader(this, this.caches, getMethodsFactory(), tracer)
@@ -121,7 +121,7 @@ open class EthereumPosMultiStream(
override fun tryProxy(
matcher: Selector.Matcher,
request: BlockchainOuterClass.NativeSubscribeRequest
request: BlockchainOuterClass.NativeSubscribeRequest,
): Flux<out Any>? =
upstreams.filter {
matcher.matches(it)
@@ -173,7 +173,7 @@ open class EthereumPosMultiStream(
selected,
PriorityForkChoice(),
headScheduler,
"ETH head for ${it.map { it.getId() }}"
"ETH head for ${it.map { it.getId() }}",
).apply {
start()
}
@@ -190,13 +190,15 @@ open class EthereumPosMultiStream(
}.let {
val selected = it.map { source -> source.getHead() }
EnrichedMergedHead(
selected, getHead(), headScheduler,
selected,
getHead(),
headScheduler,
object :
Reader<BlockHash, BlockContainer> {
override fun read(key: BlockHash): Mono<BlockContainer> {
return reader.blocksByHashAsCont().read(key).map { res -> res.data }
}
}
},
)
}
}

View File

@@ -23,6 +23,7 @@ import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.upstream.BuildInfo
import io.emeraldpay.dshackle.upstream.Capability
@@ -59,9 +60,9 @@ class BitcoinGrpcUpstream(
) : BitcoinUpstream(
"${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}",
chain,
UpstreamsConfig.PartialOptions.getDefaults().buildOptions(),
ChainOptions.PartialOptions.getDefaults().buildOptions(),
role,
chainConfig
chainConfig,
),
GrpcUpstream,
Lifecycle {
@@ -70,8 +71,11 @@ class BitcoinGrpcUpstream(
private val defaultReader: JsonRpcReader = client.getReader()
private val blockConverter: Function<BlockchainOuterClass.ChainHead, BlockContainer> = Function { value ->
val parentHash =
if (value.parentBlockId.isBlank()) null
else BlockId.from(value.parentBlockId)
if (value.parentBlockId.isBlank()) {
null
} else {
BlockId.from(value.parentBlockId)
}
val block = BlockContainer(
value.height,
BlockId.from(value.blockId),
@@ -80,7 +84,7 @@ class BitcoinGrpcUpstream(
false,
null,
null,
parentHash
parentHash,
)
block
}
@@ -108,8 +112,14 @@ class BitcoinGrpcUpstream(
}
private val upstreamStatus = GrpcUpstreamStatus(overrideLabels)
private val grpcHead = GrpcHead(
getId(), chain, this, remote, blockConverter, reloadBlock,
MostWorkForkChoice(), headScheduler
getId(),
chain,
this,
remote,
blockConverter,
reloadBlock,
MostWorkForkChoice(),
headScheduler,
)
private val timeout = Defaults.timeout
private var capabilities: Set<Capability> = emptySet()

View File

@@ -24,6 +24,7 @@ import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.BuildInfo
@@ -65,19 +66,22 @@ open class EthereumGrpcUpstream(
) : EthereumLikeUpstream(
"${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}",
hash,
UpstreamsConfig.PartialOptions.getDefaults().buildOptions(),
ChainOptions.PartialOptions.getDefaults().buildOptions(),
role,
null,
null,
chainConfig
chainConfig,
),
GrpcUpstream,
Lifecycle {
private val blockConverter: Function<BlockchainOuterClass.ChainHead, BlockContainer> = Function { value ->
val parentHash =
if (value.parentBlockId.isBlank()) null
else BlockId.from(BlockHash.from("0x" + value.parentBlockId))
if (value.parentBlockId.isBlank()) {
null
} else {
BlockId.from(BlockHash.from("0x" + value.parentBlockId))
}
val block = BlockContainer(
value.height,
BlockId.from(BlockHash.from("0x" + value.blockId)),
@@ -86,7 +90,7 @@ open class EthereumGrpcUpstream(
false,
null,
null,
parentHash
parentHash,
)
block
}
@@ -117,8 +121,14 @@ open class EthereumGrpcUpstream(
private val upstreamStatus = GrpcUpstreamStatus(overrideLabels)
private val grpcHead = GrpcHead(
getId(), chain, this, remote,
blockConverter, reloadBlock, MostWorkForkChoice(), headScheduler
getId(),
chain,
this,
remote,
blockConverter,
reloadBlock,
MostWorkForkChoice(),
headScheduler,
)
private var capabilities: Set<Capability> = emptySet()
private val buildInfo: BuildInfo = BuildInfo()

View File

@@ -23,6 +23,7 @@ import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.BuildInfo
@@ -58,19 +59,22 @@ open class EthereumPosGrpcUpstream(
) : EthereumLikeUpstream(
"${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}",
hash,
UpstreamsConfig.PartialOptions.getDefaults().buildOptions(),
ChainOptions.PartialOptions.getDefaults().buildOptions(),
role,
null,
null,
chainConfig
chainConfig,
),
GrpcUpstream,
Lifecycle {
private val blockConverter: Function<BlockchainOuterClass.ChainHead, BlockContainer> = Function { value ->
val parentHash =
if (value.parentBlockId.isBlank()) null
else BlockId.from(BlockHash.from("0x" + value.parentBlockId))
if (value.parentBlockId.isBlank()) {
null
} else {
BlockId.from(BlockHash.from("0x" + value.parentBlockId))
}
val block = BlockContainer(
value.height,
BlockId.from(BlockHash.from("0x" + value.blockId)),
@@ -79,15 +83,21 @@ open class EthereumPosGrpcUpstream(
false,
null,
null,
parentHash
parentHash,
)
block
}
private val upstreamStatus = GrpcUpstreamStatus(overrideLabels)
private val grpcHead = GrpcHead(
getId(), chain, this, remote, blockConverter, null,
NoChoiceWithPriorityForkChoice(nodeRating, parentId), headScheduler
getId(),
chain,
this,
remote,
blockConverter,
null,
NoChoiceWithPriorityForkChoice(nodeRating, parentId),
headScheduler,
)
private var capabilities: Set<Capability> = emptySet()
private val buildInfo: BuildInfo = BuildInfo()

View File

@@ -25,7 +25,7 @@ import java.util.Collections
import java.util.concurrent.atomic.AtomicReference
class GrpcUpstreamStatus(
private val overrideLabels: UpstreamsConfig.Labels?
private val overrideLabels: UpstreamsConfig.Labels?,
) {
companion object {
private val log = LoggerFactory.getLogger(GrpcUpstreamStatus::class.java)
@@ -47,7 +47,7 @@ class GrpcUpstreamStatus(
overrideLabels?.let { labels.putAll(it) }
val node = QuorumForLabels.QuorumItem(
remoteNode.quorum,
labels
labels,
)
updateNodes.add(node)
updateLabels.add(labels)

View File

@@ -88,7 +88,7 @@ class GrpcUpstreams(
private val clientSpansInterceptor: ClientInterceptor?,
private var maxMetadataSize: Int,
private val headScheduler: Scheduler,
private val grpcAuthContext: GrpcAuthContext
private val grpcAuthContext: GrpcAuthContext,
) {
private val log = LoggerFactory.getLogger(GrpcUpstreams::class.java)
@@ -136,9 +136,11 @@ class GrpcUpstreams(
ReactorAuthGrpc.newReactorStub(channel),
authorizationConfig,
grpcAuthContext,
tokenAuth.publicKeyPath!!
tokenAuth.publicKeyPath!!,
)
} else null
} else {
null
}
val statusSubscriptions = mutableMapOf<Chain, Disposable>()
@@ -161,7 +163,7 @@ class GrpcUpstreams(
if (sub == null || sub.isDisposed) {
val subscription = this.client.subscribeStatus(
StatusRequest.newBuilder()
.addChains(Common.ChainRef.forNumber(it.chain.id)).build()
.addChains(Common.ChainRef.forNumber(it.chain.id)).build(),
).subscribeOn(chainStatusScheduler)
.subscribe { value ->
val chain = Chain.byId(value.chain.number)
@@ -228,7 +230,7 @@ class GrpcUpstreams(
if (StringUtils.isNotEmpty(auth.key) && StringUtils.isNoneEmpty(auth.certificate)) {
sslContext.keyManager(
fileResolver.resolve(auth.certificate!!).inputStream(),
fileResolver.resolve(auth.key!!).inputStream()
fileResolver.resolve(auth.key!!).inputStream(),
)
} else {
log.warn("Connect to remote using only CA certificate")
@@ -237,7 +239,8 @@ class GrpcUpstreams(
ApplicationProtocolConfig.Protocol.ALPN,
ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE,
ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT,
"grpc-exp", "h2"
"grpc-exp",
"h2",
)
sslContext.applicationProtocolConfig(alpn)
return sslContext.build()
@@ -253,8 +256,8 @@ class GrpcUpstreams(
client,
rpcClient,
labels,
chainsConfig.resolve(chain),
headScheduler
chainsConfig.resolve(chain.chainName),
headScheduler,
)
},
BlockchainType.EVM_POS to { chain, rpcClient ->
@@ -267,13 +270,13 @@ class GrpcUpstreams(
rpcClient,
nodeRating,
labels,
chainsConfig.resolve(chain),
headScheduler
chainsConfig.resolve(chain.chainName),
headScheduler,
)
},
BlockchainType.BITCOIN to { chain, rpcClient ->
BitcoinGrpcUpstream(id, role, chain, client, rpcClient, labels, chainsConfig.resolve(chain), headScheduler)
}
BitcoinGrpcUpstream(id, role, chain, client, rpcClient, labels, chainsConfig.resolve(chain.chainCode), headScheduler)
},
)
private fun getOrCreate(chain: Chain): UpstreamChangeEvent {
@@ -285,7 +288,7 @@ class GrpcUpstreams(
private fun makeMetrics(chain: Chain): RpcMetrics {
val metricsTags = listOf(
Tag.of("upstream", id),
Tag.of("chain", chain.chainCode)
Tag.of("chain", chain.chainCode),
)
return RpcMetrics(
@@ -297,14 +300,14 @@ class GrpcUpstreams(
Counter.builder("upstream.grpc.fail")
.description("Number of failures of Dshackle/gRPC requests")
.tags(metricsTags)
.register(Metrics.globalRegistry)
.register(Metrics.globalRegistry),
)
}
private fun getOrCreate(
chain: Chain,
metrics: RpcMetrics,
creator: (chain: Chain, client: JsonRpcGrpcClient) -> DefaultUpstream
creator: (chain: Chain, client: JsonRpcGrpcClient) -> DefaultUpstream,
): UpstreamChangeEvent {
lock.withLock {
val current = known[chain]
@@ -338,7 +341,7 @@ class GrpcUpstreams(
} else {
Mono.error(it)
}
}
},
)
}

View File

@@ -1,211 +0,0 @@
version: v1
chain-settings:
default:
expected-block-time: 12s
lags:
syncing: 6
lagging: 1
chains:
- id: eth
call-validate-contract: 0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96
expected-block-time: 12s
lags:
syncing: 6
lagging: 1
- id: goerli
call-validate-contract: 0xCD9303A1F6da2a68f465A579a24cc2Ee5AE2192f
expected-block-time: 12s
lags:
syncing: 6
lagging: 1
- id: polygon
call-validate-contract: 0x53Daa71B04d589429f6d3DF52db123913B818F23
expected-block-time: 2.7s
lags:
syncing: 20
lagging: 10
- id: polygon-mumbai
call-validate-contract: 0x53Daa71B04d589429f6d3DF52db123913B818F23
expected-block-time: 2.7s
lags:
syncing: 20
lagging: 10
- id: arbitrum
expected-block-time: 260ms
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: arbitrum-testnet
expected-block-time: 1s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: optimism
expected-block-time: 2s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: optimism-testnet
expected-block-time: 2s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: arbitrum-nova
expected-block-time: 1s
options:
disable-validation: true
lags:
syncing: 40
lagging: 20
- id: polygon-zkevm
expected-block-time: 2.7s
options:
disable-validation: true
lags:
syncing: 40
lagging: 20
- id: polygon-zkevm-testnet
expected-block-time: 1m
options:
disable-validation: true
lags:
syncing: 40
lagging: 20
- id: zksync
expected-block-time: 5s
options:
disable-validation: true
lags:
syncing: 40
lagging: 20
- id: zksync-testnet
expected-block-time: 5s
options:
disable-validation: true
lags:
syncing: 40
lagging: 20
- id: base
expected-block-time: 2s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: base-goerli
expected-block-time: 2s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: avalanche
expected-block-time: 2s
options:
validate-peers: false
validate-syncing: false
lags:
syncing: 10
lagging: 5
- id: avalanche-fuji
expected-block-time: 2s
options:
validate-peers: false
validate-syncing: false
lags:
syncing: 10
lagging: 5
- id: gnosis
expected-block-time: 6s
options:
validate-peers: false
lags:
syncing: 10
lagging: 5
- id: gnosis-chiado
expected-block-time: 6s
options:
validate-peers: false
lags:
syncing: 10
lagging: 5
- id: fantom
expected-block-time: 3s
options:
validate-peers: false
lags:
syncing: 10
lagging: 5
- id: fantom-testnet
expected-block-time: 1m
options:
validate-peers: false
lags:
syncing: 10
lagging: 5
- id: aurora
expected-block-time: 1s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: aurora-testnet
expected-block-time: 1s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: scroll-alphanet
expected-block-time: 3s
options:
validate-peers: false
lags:
syncing: 10
lagging: 5
- id: scroll-sepolia
expected-block-time: 4s
options:
validate-peers: false
lags:
syncing: 10
lagging: 5
- id: mantle
expected-block-time: 500ms
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: mantle-testnet
expected-block-time: 500ms
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: klaytn
expected-block-time: 1s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: klaytn-baobab
expected-block-time: 1s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20

View File

@@ -1,59 +0,0 @@
/**
* Copyright (c) 2019 ETCDEV GmbH
* Copyright (c) 2020 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.FileResolver
import spock.lang.Specification
import java.time.Duration
class ChainsConfigReaderSpec extends Specification {
ChainsConfigReader reader = new ChainsConfigReader(
new UpstreamsConfigReader(Stub(FileResolver))
)
def "Parse standard config"() {
setup:
def stream = this.class.getClassLoader().getResourceAsStream("configs/chains-basic.yaml")
when:
def config = reader.read(stream)
def eth = config.resolve(Chain.ETHEREUM__MAINNET)
def pol = config.resolve(Chain.POLYGON_POS__MAINNET)
def opt = config.resolve(Chain.OPTIMISM__MAINNET)
def sep = config.resolve(Chain.ETHEREUM__SEPOLIA)
then:
eth.laggingLagSize == 1
eth.syncingLagSize == 6
eth.callLimitContract == "0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96"
eth.expectedBlockTime == Duration.ofSeconds(12)
pol.laggingLagSize == 10
pol.syncingLagSize == 20
pol.expectedBlockTime == Duration.ofMillis(2700)
opt.laggingLagSize == 3
opt.syncingLagSize == 40
opt.options.validatePeers == false
opt.expectedBlockTime == Duration.ofMillis(400)
sep.laggingLagSize == 1
sep.syncingLagSize == 10
sep.expectedBlockTime == Duration.ofSeconds(12)
}
}

View File

@@ -1,44 +0,0 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.config
import spock.lang.Specification
class EnvVariablesSpec extends Specification {
EnvVariables reader = new EnvVariables()
def "Post process for usual strings"() {
expect:
s == reader.postProcess(s)
where:
s << ["", "a", "13143", "/etc/client1.myservice.com.key", "true", "1a68f20154fc258fe4149c199ad8f281"]
}
def "Post process replaces from env"() {
setup:
System.setProperty("id", "1")
System.setProperty("HOME", "/home/user")
System.setProperty("PASSWORD", "1a68f20154fc258fe4149c199ad8f281")
expect:
replaced == reader.postProcess(orig)
where:
orig | replaced
"p_\${id}" | "p_1"
"home: \${HOME}" | "home: /home/user"
"\${PASSWORD}" | "1a68f20154fc258fe4149c199ad8f281"
}
}

View File

@@ -51,7 +51,7 @@ class MainConfigReaderSpec extends Specification {
port == 8082
tls != null
routes != null
routes.size() == 5
routes.size() == 3
with(routes[0]) {
id == "eth"
blockchain == Chain.ETHEREUM__MAINNET
@@ -61,17 +61,9 @@ class MainConfigReaderSpec extends Specification {
blockchain == Chain.ETHEREUM_CLASSIC__MAINNET
}
with(routes[2]) {
id == "kovan"
blockchain == Chain.ETHEREUM__KOVAN
}
with(routes[3]) {
id == "goerli"
blockchain == Chain.ETHEREUM__GOERLI
}
with(routes[4]) {
id == "rinkeby"
blockchain == Chain.ETHEREUM__RINKEBY
}
}
with(act.health) {
it.enabled

View File

@@ -16,7 +16,8 @@
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.foundation.ChainOptionsReader
import io.emeraldpay.dshackle.test.TestingCommons
import spock.lang.Specification
@@ -24,7 +25,7 @@ import java.time.Duration
class UpstreamsConfigReaderSpec extends Specification {
UpstreamsConfigReader reader = new UpstreamsConfigReader(TestingCommons.fileResolver())
UpstreamsConfigReader reader = new UpstreamsConfigReader(TestingCommons.fileResolver(), new ChainOptionsReader())
def "Parse standard config"() {
setup:
@@ -489,8 +490,8 @@ class UpstreamsConfigReaderSpec extends Specification {
def "Merge options for disableValidation"() {
expect:
def a = new UpstreamsConfig.PartialOptions().tap { disableValidation = base }
def b = new UpstreamsConfig.PartialOptions().tap { disableValidation = overwrite }
def a = new ChainOptions.PartialOptions().tap { disableValidation = base }
def b = new ChainOptions.PartialOptions().tap { disableValidation = overwrite }
def result = a.merge(b).buildOptions()
result.disableValidation == exp
@@ -511,8 +512,8 @@ class UpstreamsConfigReaderSpec extends Specification {
def "Merge options for providesBalance"() {
expect:
def a = new UpstreamsConfig.PartialOptions().tap { providesBalance = base }
def b = new UpstreamsConfig.PartialOptions().tap { providesBalance = overwrite }
def a = new ChainOptions.PartialOptions().tap { providesBalance = base }
def b = new ChainOptions.PartialOptions().tap { providesBalance = overwrite }
def result = a.merge(b).buildOptions()
result.providesBalance == exp
@@ -533,8 +534,8 @@ class UpstreamsConfigReaderSpec extends Specification {
def "Merge options for validatePeers"() {
expect:
def a = new UpstreamsConfig.PartialOptions().tap { validatePeers = base }
def b = new UpstreamsConfig.PartialOptions().tap { validatePeers = overwrite }
def a = new ChainOptions.PartialOptions().tap { validatePeers = base }
def b = new ChainOptions.PartialOptions().tap { validatePeers = overwrite }
def result = a.merge(b).buildOptions()
result.validatePeers == exp
@@ -555,8 +556,8 @@ class UpstreamsConfigReaderSpec extends Specification {
def "Merge options for validateSyncing"() {
expect:
def a = new UpstreamsConfig.PartialOptions().tap { validateSyncing = base }
def b = new UpstreamsConfig.PartialOptions().tap { validateSyncing = overwrite }
def a = new ChainOptions.PartialOptions().tap { validateSyncing = base }
def b = new ChainOptions.PartialOptions().tap { validateSyncing = overwrite }
def result = a.merge(b).buildOptions()
result.validateSyncing == exp
@@ -577,10 +578,10 @@ class UpstreamsConfigReaderSpec extends Specification {
def "Merge options for timeout"() {
expect:
def a = new UpstreamsConfig.PartialOptions().tap {
def a = new ChainOptions.PartialOptions().tap {
timeout = base == null ? null : Duration.ofSeconds(base)
}
def b = new UpstreamsConfig.PartialOptions().tap {
def b = new ChainOptions.PartialOptions().tap {
timeout = overwrite == null ? null : Duration.ofSeconds(overwrite)
}
def result = a.merge(b).buildOptions()
@@ -598,8 +599,8 @@ class UpstreamsConfigReaderSpec extends Specification {
def "Merge options for minPeers"() {
expect:
def a = new UpstreamsConfig.PartialOptions().tap { minPeers = base }
def b = new UpstreamsConfig.PartialOptions().tap { minPeers = overwrite }
def a = new ChainOptions.PartialOptions().tap { minPeers = base }
def b = new ChainOptions.PartialOptions().tap { minPeers = overwrite }
def result = a.merge(b).buildOptions()
result.minPeers == exp
@@ -614,8 +615,8 @@ class UpstreamsConfigReaderSpec extends Specification {
def "Merge options for validationInterval"() {
expect:
def a = new UpstreamsConfig.PartialOptions().tap { validationInterval = base }
def b = new UpstreamsConfig.PartialOptions().tap { validationInterval = overwrite }
def a = new ChainOptions.PartialOptions().tap { validationInterval = base }
def b = new ChainOptions.PartialOptions().tap { validationInterval = overwrite }
def result = a.merge(b).buildOptions()
result.validationInterval == exp
@@ -630,11 +631,11 @@ class UpstreamsConfigReaderSpec extends Specification {
def "Options with default values"() {
setup:
def partialOptions = new UpstreamsConfig.PartialOptions()
def partialOptions = new ChainOptions.PartialOptions()
when:
def options = partialOptions.buildOptions()
then:
options == new UpstreamsConfig.Options(
options == new ChainOptions.Options(
false, false, 30, Duration.ofSeconds(60), null, true, 1, true, true, true
)
}

View File

@@ -1,51 +0,0 @@
/**
* Copyright (c) 2021 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.config
import org.jetbrains.annotations.Nullable
import org.yaml.snakeyaml.Yaml
import org.yaml.snakeyaml.nodes.MappingNode
import spock.lang.Specification
class YamlConfigReaderSpec extends Specification {
def "reads bytes values"() {
setup:
def rdr = new Impl()
expect:
rdr.getValueAsBytes(asNode("test", input), "test") == exp
where:
input | exp
"1024" | 1024
"1k" | 1024
"1kb" | 1024
"1K" | 1024
"16kb" | 16 * 1024
"1M" | 1024 * 1024
"4mb" | 4 * 1024 * 1024
}
private MappingNode asNode(String key, String value) {
return new Yaml().compose(new StringReader("$key: $value")) as MappingNode
}
class Impl extends YamlConfigReader {
@Override
Object read(@Nullable MappingNode input) {
return null
}
}
}

View File

@@ -292,7 +292,7 @@ class NativeCallSpec extends Specification {
def nativeCall = nativeCall(upstreams)
def req = BlockchainOuterClass.NativeCallRequest.newBuilder()
.setChainValue(Chain.ETHEREUM__MORDEN.id)
.setChainValue(Chain.ETHEREUM__GOERLI.id)
.addAllItems([1, 2].collect { id ->
return BlockchainOuterClass.NativeCallItem.newBuilder()
.setId(id)
@@ -300,7 +300,7 @@ class NativeCallSpec extends Specification {
.build()
})
.build()
1 * upstreams.isAvailable(Chain.ETHEREUM__MORDEN) >> false
1 * upstreams.isAvailable(Chain.ETHEREUM__GOERLI) >> false
when:
def resp = nativeCall.prepareCall(req)
then:

View File

@@ -17,10 +17,9 @@
package io.emeraldpay.dshackle.test
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig.Options
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.startup.QuorumForLabels
@@ -31,6 +30,7 @@ import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.jetbrains.annotations.NotNull
import org.reactivestreams.Publisher
import io.emeraldpay.dshackle.foundation.ChainOptions
class EthereumPosRpcUpstreamMock extends EthereumLikeRpcUpstream {
EthereumHeadMock ethereumHeadMock
@@ -80,8 +80,8 @@ class EthereumPosRpcUpstreamMock extends EthereumLikeRpcUpstream {
start()
}
static Options getOpts() {
def opt = UpstreamsConfig.PartialOptions.getDefaults()
static ChainOptions.Options getOpts() {
def opt = ChainOptions.PartialOptions.getDefaults()
opt.setDisableValidation(true)
opt.setDisableUpstreamValidation(true)
return opt.buildOptions()

View File

@@ -28,6 +28,7 @@ import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeRpcUpstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.jetbrains.annotations.NotNull
import io.emeraldpay.dshackle.foundation.ChainOptions
import org.reactivestreams.Publisher
class EthereumRpcUpstreamMock extends EthereumLikeRpcUpstream {
@@ -56,7 +57,7 @@ class EthereumRpcUpstreamMock extends EthereumLikeRpcUpstream {
EthereumRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader<JsonRpcRequest, JsonRpcResponse> api, CallMethods methods) {
super(id, id.hashCode().byteValue(), chain,
UpstreamsConfig.PartialOptions.getDefaults().buildOptions(),
ChainOptions.PartialOptions.getDefaults().buildOptions(),
UpstreamsConfig.UpstreamRole.PRIMARY,
methods,
new QuorumForLabels.QuorumItem(1, new UpstreamsConfig.Labels()),

View File

@@ -29,6 +29,7 @@ import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import reactor.core.scheduler.Schedulers
import reactor.test.StepVerifier
import spock.lang.Retry
import io.emeraldpay.dshackle.foundation.ChainOptions
import spock.lang.Specification
import java.time.Duration
@@ -66,7 +67,7 @@ class FilteredApisSpec extends Specification {
"test",
(byte) 123,
Chain.ETHEREUM__MAINNET,
new UpstreamsConfig.PartialOptions().buildOptions(),
new ChainOptions.PartialOptions().buildOptions(),
UpstreamsConfig.UpstreamRole.PRIMARY,
ethereumTargets,
new QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(it)),

View File

@@ -39,9 +39,7 @@ class DefaultEthereumMethodsSpec extends Specification {
chain | id
Chain.ETHEREUM__MAINNET | '"0x1"'
Chain.ETHEREUM_CLASSIC__MAINNET | '"0x3d"'
Chain.ETHEREUM__KOVAN | '"0x2a"'
Chain.ETHEREUM__GOERLI | '"0x5"'
Chain.ETHEREUM__RINKEBY | '"0x4"'
Chain.ETHEREUM__ROPSTEN | '"0x3"'
}
@@ -61,7 +59,7 @@ class DefaultEthereumMethodsSpec extends Specification {
new DefaultEthereumMethods(chain).getSupportedMethods().containsAll(methods)
where:
chain | methods
Chain.POLYGON_POS__MAINNET | ["bor_getAuthor",
Chain.POLYGON__MAINNET | ["bor_getAuthor",
"bor_getCurrentValidators",
"bor_getCurrentProposer",
"bor_getRootHash",

View File

@@ -16,7 +16,7 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.test.ApiReaderMock
import io.emeraldpay.dshackle.test.TestingCommons
@@ -43,7 +43,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Resolve to final availability"() {
setup:
def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, Stub(EthereumLikeUpstream), UpstreamsConfig.PartialOptions.getDefaults().buildOptions())
def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, Stub(EthereumLikeUpstream), ChainOptions.PartialOptions.getDefaults().buildOptions())
expect:
validator.resolve(Tuples.of(sync, peers, call)) == exp
where:
@@ -61,7 +61,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Doesnt check eth_syncing when disabled"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validateSyncing = false
}.buildOptions()
def up = Mock(EthereumLikeUpstream)
@@ -76,7 +76,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Syncing is OK when false returned from upstream"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validateSyncing = true
}.buildOptions()
def up = TestingCommons.upstream(
@@ -94,7 +94,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Execute onSyncingNode with result of eth_syncing"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validateSyncing = true
}.buildOptions()
def up = Mock(EthereumLikeUpstream) {
@@ -121,7 +121,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Syncing is SYNCING when state returned from upstream"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validateSyncing = true
}.buildOptions()
def up = TestingCommons.upstream(
@@ -139,7 +139,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Syncing is UNAVAILABLE when error returned from upstream"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validateSyncing = true
}.buildOptions()
def up = TestingCommons.upstream(
@@ -157,7 +157,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Doesnt validate peers when disabled"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validatePeers = false
it.minPeers = 10
}.buildOptions()
@@ -173,7 +173,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Doesnt validate peers when zero peers is expected"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validatePeers = true
it.minPeers = 0
}.buildOptions()
@@ -189,7 +189,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Peers is IMMATURE when state returned too few peers"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validatePeers = true
it.minPeers = 10
}.buildOptions()
@@ -208,7 +208,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Peers is OK when state returned exactly min peers"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validatePeers = true
it.minPeers = 10
}.buildOptions()
@@ -227,7 +227,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Peers is OK when state returned more than enough peers"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validatePeers = true
it.minPeers = 10
}.buildOptions()
@@ -246,7 +246,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Peers is UNAVAILABLE when state returned error"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validatePeers = true
it.minPeers = 10
}.buildOptions()
@@ -265,7 +265,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Doesnt validate chan and callLimit when disabled"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validateCalllimit = false
it.validateChain = false
}.buildOptions()
@@ -287,7 +287,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Upstream is valid if not error from call limit check"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validateChain = false
}.buildOptions()
def up = Mock(EthereumLikeRpcUpstream) {
@@ -311,7 +311,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Upstream is not valid if error returned on call limit check"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validateChain = false
}.buildOptions()
def up = Mock(EthereumLikeRpcUpstream) {
@@ -335,7 +335,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Upstream is valid if chain settings are valid"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validateCalllimit = false
}.buildOptions()
def up = Mock(EthereumLikeRpcUpstream) {
@@ -357,7 +357,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Upstream is not valid - specified optimism but got ethereum"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().tap {
def options = ChainOptions.PartialOptions.getDefaults().tap {
it.validateCalllimit = false
}.buildOptions()
def up = Mock(EthereumLikeRpcUpstream) {
@@ -379,7 +379,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Upstream is valid if all setting are valid"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().buildOptions()
def options = ChainOptions.PartialOptions.getDefaults().buildOptions()
def up = Mock(EthereumLikeRpcUpstream) {
5 * getIngressReader() >> Mock(Reader) {
1 * read(new JsonRpcRequest("eth_chainId", emptyList())) >> Mono.just(new JsonRpcResponse('"0x1"'.getBytes(), null))
@@ -403,7 +403,7 @@ class EthereumUpstreamValidatorSpec extends Specification {
def "Upstream is not valid if there are errors"() {
setup:
def options = UpstreamsConfig.PartialOptions.getDefaults().buildOptions()
def options = ChainOptions.PartialOptions.getDefaults().buildOptions()
def up = Mock(EthereumLikeRpcUpstream) {
5 * getIngressReader() >> Mock(Reader) {
1 * read(new JsonRpcRequest("eth_chainId", emptyList())) >> Mono.just(new JsonRpcResponse(null, new JsonRpcError(1, "Too long")))

View File

@@ -38,7 +38,6 @@ import io.grpc.stub.StreamObserver
import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Timer
import reactor.core.scheduler.Schedulers
import reactor.test.StepVerifier
import spock.lang.Specification
import java.time.Duration

View File

@@ -6,6 +6,7 @@ import io.emeraldpay.api.proto.Common.ChainRef
import io.emeraldpay.dshackle.config.MainConfig
import io.emeraldpay.dshackle.config.MainConfigReader
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.github.ganchix.ganache.Account
import io.github.ganchix.ganache.GanacheContainer
import io.grpc.BindableService
@@ -43,8 +44,8 @@ class IntegrationTest {
withAccounts(
listOf(
Account.builder().privateKey(PRIVATE_KEY_0).balance(BigInteger.valueOf(2000000000000000000)).build(),
Account.builder().privateKey(PRIVATE_KEY_1).balance(BigInteger.valueOf(2000000000000000000)).build()
)
Account.builder().privateKey(PRIVATE_KEY_1).balance(BigInteger.valueOf(2000000000000000000)).build(),
),
)
}
@@ -82,7 +83,7 @@ class IntegrationTest {
val reader = MainConfigReader(fileResolver)
val config = reader.read(
ResourceUtils.getFile("classpath:integration/dshackle.yaml")
.inputStream()
.inputStream(),
)!!
patch(config)
return config
@@ -94,7 +95,7 @@ class IntegrationTest {
id = "ganache"
nodeId = 1
chain = "ethereum"
options = UpstreamsConfig.PartialOptions()
options = ChainOptions.PartialOptions()
.apply {
validateChain = false
}
@@ -102,12 +103,12 @@ class IntegrationTest {
execution = UpstreamsConfig.EthereumConnection().apply {
rpc = UpstreamsConfig.HttpEndpoint(
URI.create(
"http://" + ganacheContainer.getHost() + ":" + ganacheContainer.getMappedPort(8545) + "/"
)
"http://" + ganacheContainer.getHost() + ":" + ganacheContainer.getMappedPort(8545) + "/",
),
)
}
}
}
},
)
}
}

View File

@@ -1,51 +0,0 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Chain
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
internal class ChainsConfigTest {
@Test
fun patch() {
val orig = ChainsConfig(
mapOf(
Chain.BITCOIN__MAINNET to createRawChainConfig(0, 0),
Chain.ETHEREUM__MAINNET to createRawChainConfig(1, 2),
Chain.POLYGON_POS__MAINNET to createRawChainConfig(3, 4)
),
createRawChainConfig(1, 2)
)
val patch = ChainsConfig(
mapOf(
Chain.BITCOIN__MAINNET to createRawChainConfig(null, 10000),
Chain.POLYGON_POS__MAINNET to createRawChainConfig(10, 11),
Chain.ARBITRUM__MAINNET to createRawChainConfig(999, 999)
),
createRawChainConfig(100, null)
)
val res = orig.patch(patch)
assertEquals(
ChainsConfig(
mapOf(
Chain.BITCOIN__MAINNET to createRawChainConfig(0, 10000),
Chain.ETHEREUM__MAINNET to createRawChainConfig(1, 2),
Chain.POLYGON_POS__MAINNET to createRawChainConfig(10, 11),
Chain.ARBITRUM__MAINNET to createRawChainConfig(999, 999)
),
createRawChainConfig(100, 2)
),
res
)
}
private fun createRawChainConfig(syncingLagSize: Int?, laggingLagSize: Int?) =
ChainsConfig.RawChainConfig()
.apply {
this.syncingLagSize = syncingLagSize
this.laggingLagSize = laggingLagSize
}
}

View File

@@ -15,21 +15,21 @@ internal class UpstreamsConfigTest {
return Stream.of(
Arguments.of(
UpstreamsConfig.EthereumConnection(),
ConnectorMode.RPC_ONLY
ConnectorMode.RPC_ONLY,
),
Arguments.of(
UpstreamsConfig.EthereumConnection()
.apply {
ws = UpstreamsConfig.WsEndpoint(URI("ws://localhost:8546"))
},
ConnectorMode.WS_ONLY
ConnectorMode.WS_ONLY,
),
Arguments.of(
UpstreamsConfig.EthereumConnection()
.apply {
ws = UpstreamsConfig.WsEndpoint(URI("ws://localhost:8546"))
},
ConnectorMode.WS_ONLY
ConnectorMode.WS_ONLY,
),
Arguments.of(
UpstreamsConfig.EthereumConnection()
@@ -37,7 +37,7 @@ internal class UpstreamsConfigTest {
connectorMode = "RPC_REQUESTS_WITH_WS_HEAD"
ws = UpstreamsConfig.WsEndpoint(URI("ws://localhost:8546"))
},
ConnectorMode.RPC_REQUESTS_WITH_WS_HEAD
ConnectorMode.RPC_REQUESTS_WITH_WS_HEAD,
),
Arguments.of(
UpstreamsConfig.EthereumConnection()
@@ -45,7 +45,7 @@ internal class UpstreamsConfigTest {
connectorMode = "RPC_REQUESTS_WITH_MIXED_HEAD"
ws = UpstreamsConfig.WsEndpoint(URI("ws://localhost:8546"))
},
ConnectorMode.RPC_REQUESTS_WITH_MIXED_HEAD
ConnectorMode.RPC_REQUESTS_WITH_MIXED_HEAD,
),
Arguments.of(
UpstreamsConfig.EthereumConnection()
@@ -53,7 +53,7 @@ internal class UpstreamsConfigTest {
rpc = UpstreamsConfig.HttpEndpoint(URI("http://localhost:8546"))
ws = UpstreamsConfig.WsEndpoint(URI("ws://localhost:8546"))
},
ConnectorMode.RPC_REQUESTS_WITH_WS_HEAD
ConnectorMode.RPC_REQUESTS_WITH_WS_HEAD,
),
)
}

View File

@@ -1,22 +0,0 @@
version: v1
chain-settings:
default:
lags:
syncing: 6
lagging: 1
expected-block-time: 12s
chains:
- id: eth
call-validate-contract: 0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96
expected-block-time: 12s
lags:
syncing: 6
lagging: 1
- id: optimism
expected-block-time: 400ms
lags:
lagging: 3
- id: sepolia
lags:
syncing: 10

View File

@@ -45,13 +45,8 @@ proxy:
blockchain: ethereum
- id: etc
blockchain: ethereum_classic
- id: kovan
blockchain: kovan
- id: goerli
blockchain: goerli
- id: rinkeby
blockchain: rinkeby
cluster:
defaults:
- chains: