StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

problem: suboptimal routing for upstreams in different datacenters

Browse Source 
solution: additional role to separate primary and secondary upstreams
fix: #141
This commit is contained in:
Igor Artamonov
2022-01-05 18:42:35 -05:00
parent 45237592a3
commit c060aeda31
16 changed files with 179 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt
View File

@@ -75,7 +75,7 @@ open class UpstreamsConfig {
var connection: T? = null
val labels = Labels()
var methods: Methods? = null
var role: UpstreamRole = UpstreamRole.STANDARD
var role: UpstreamRole = UpstreamRole.PRIMARY
@Suppress("UNCHECKED_CAST")
fun <Z : UpstreamConnection> cast(type: Class<Z>): Upstream<Z> {
@@ -87,7 +87,8 @@ open class UpstreamsConfig {
}
enum class UpstreamRole {
STANDARD,
PRIMARY,
SECONDARY,
FALLBACK
}

5
src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt
View File

@@ -214,7 +214,10 @@ class UpstreamsConfigReader(
internal fun readUpstreamStandard(upNode: MappingNode, upstream: UpstreamsConfig.Upstream<*>) {
upstream.chain = getValueAsString(upNode, "chain")
getValueAsString(upNode, "role")?.let {
val name = it.trim()
val name = it.trim().let {
// `standard` was initial role, now split into `primary` and `secondary`
if (it == "standard") "primary" else it
}
try {
val role = UpstreamsConfig.UpstreamRole.valueOf(name.uppercase(Locale.getDefault()))
upstream.role = role

1
src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt
View File

@@ -234,6 +234,7 @@ open class ConfiguredUpstreams(
val endpoint = config.connection!!
val ds = GrpcUpstreams(
config.id!!,
config.role,
endpoint.host!!,
endpoint.port,
endpoint.auth,

33
src/main/kotlin/io/emeraldpay/dshackle/upstream/FilteredApis.kt
View File

@@ -82,7 +82,8 @@ class FilteredApis(
) : this(chain, allUpstreams, matcher, 0, 10, 10)
private val delay: Int
private val standardUpstreams: List<Upstream>
private val primaryUpstreams: List<Upstream>
private val secondaryUpstreams: List<Upstream>
private val standardWithFallback: List<Upstream>
private val control = Sinks.many().unicast().onBackpressureBuffer<Boolean>()
@@ -94,8 +95,13 @@ class FilteredApis(
DEFAULT_DELAY_STEP
}
standardUpstreams = allUpstreams.filter {
it.getRole() == UpstreamsConfig.UpstreamRole.STANDARD
primaryUpstreams = allUpstreams.filter {
it.getRole() == UpstreamsConfig.UpstreamRole.PRIMARY
}.let {
startFrom(it, pos)
}
secondaryUpstreams = allUpstreams.filter {
it.getRole() == UpstreamsConfig.UpstreamRole.SECONDARY
}.let {
startFrom(it, pos)
}
@@ -105,12 +111,14 @@ class FilteredApis(
startFrom(it, pos)
}
standardWithFallback = emptyList<Upstream>()
.plus(standardUpstreams)
.plus(primaryUpstreams)
.plus(secondaryUpstreams)
.plus(fallbackUpstreams)
if (Global.metricsExtended) {
getMetrics(chain).let { monitoring ->
monitoring.countStd.record(standardUpstreams.size.toDouble())
monitoring.countPrimary.record(primaryUpstreams.size.toDouble())
monitoring.countSecondary.record(secondaryUpstreams.size.toDouble())
monitoring.countFallback.record(fallbackUpstreams.size.toDouble())
}
}
@@ -145,17 +153,18 @@ class FilteredApis(
override fun subscribe(subscriber: Subscriber<in Upstream>) {
// initially try only standard upstreams
val first = Flux.fromIterable(standardUpstreams)
val first = Flux.fromIterable(primaryUpstreams)
val second = Flux.fromIterable(secondaryUpstreams)
// if all failed, try both standard and fallback upstreams, repeating in cycle
val retries = (0 until (retryLimit - 1)).map { r ->
Flux.fromIterable(standardWithFallback)
// add delay to let upstream to restore if it's a temp failure
// add a delay to let upstream to restore if it's a temp failure
// but delay only start of the check, not between upstreams
// i.e. if all upstreams failed -> wait -> check all without waiting in between
.delaySubscription(waitDuration(r + 1))
}.let { Flux.concat(it) }
var result = Flux.concat(first, retries)
var result = Flux.concat(first, second, retries)
if (Global.metricsExtended) {
var count = 0
@@ -186,9 +195,13 @@ class FilteredApis(
}
class Monitoring(chain: Chain) {
val countStd: DistributionSummary = DistributionSummary.builder("$metricsCode.exist")
val countPrimary: DistributionSummary = DistributionSummary.builder("$metricsCode.exist")
.description("Count of available upstreams to select")
.tags(listOf(Tag.of("chain", chain.chainCode), Tag.of("role", "std")))
.tags(listOf(Tag.of("chain", chain.chainCode), Tag.of("role", "primary")))
.register(Metrics.globalRegistry)
val countSecondary: DistributionSummary = DistributionSummary.builder("$metricsCode.exist")
.description("Count of available upstreams to select")
.tags(listOf(Tag.of("chain", chain.chainCode), Tag.of("role", "secondary")))
.register(Metrics.globalRegistry)
val countFallback: DistributionSummary = DistributionSummary.builder("$metricsCode.exist")
.description("Count of available fallback upstreams to select")

2
src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt
View File

@@ -212,7 +212,7 @@ abstract class Multistream(
// TODO roles for multistream are useless
override fun getRole(): UpstreamsConfig.UpstreamRole {
return UpstreamsConfig.UpstreamRole.STANDARD
return UpstreamsConfig.UpstreamRole.PRIMARY
}
override fun getMethods(): CallMethods {

2
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcUpstream.kt
View File

@@ -33,7 +33,7 @@ open class EthereumRpcUpstream(
constructor(id: String, chain: Chain, api: Reader<JsonRpcRequest, JsonRpcResponse>) :
this(
id, chain, api, null,
UpstreamsConfig.Options.getDefaults(), UpstreamsConfig.UpstreamRole.STANDARD,
UpstreamsConfig.Options.getDefaults(), UpstreamsConfig.UpstreamRole.PRIMARY,
QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels()),
DirectCallMethods()
)

3
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt
View File

@@ -45,6 +45,7 @@ import java.util.function.Function
class BitcoinGrpcUpstream(
private val parentId: String,
role: UpstreamsConfig.UpstreamRole,
chain: Chain,
val remote: ReactorBlockchainGrpc.ReactorBlockchainStub,
private val client: JsonRpcGrpcClient
@@ -52,7 +53,7 @@ class BitcoinGrpcUpstream(
"$parentId/${chain.chainCode}",
chain,
UpstreamsConfig.Options.getDefaults(),
UpstreamsConfig.UpstreamRole.STANDARD
role
),
GrpcUpstream,
Lifecycle {

3
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt
View File

@@ -48,13 +48,14 @@ import java.util.function.Function
open class EthereumGrpcUpstream(
private val parentId: String,
role: UpstreamsConfig.UpstreamRole,
private val chain: Chain,
private val remote: ReactorBlockchainGrpc.ReactorBlockchainStub,
private val client: JsonRpcGrpcClient
) : EthereumUpstream(
"$parentId/${chain.chainCode}",
UpstreamsConfig.Options.getDefaults(),
UpstreamsConfig.UpstreamRole.STANDARD,
role,
null, null
),
GrpcUpstream,

6
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt
View File

@@ -21,6 +21,7 @@ import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.FileResolver
import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.startup.UpstreamChange
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
@@ -51,6 +52,7 @@ import kotlin.concurrent.withLock
class GrpcUpstreams(
private val id: String,
private val role: UpstreamsConfig.UpstreamRole,
private val host: String,
private val port: Int,
private val auth: AuthConfig.ClientTlsAuth? = null,
@@ -198,7 +200,7 @@ class GrpcUpstreams(
val current = known[chain]
return if (current == null) {
val rpcClient = JsonRpcGrpcClient(client!!, chain, metrics)
val created = EthereumGrpcUpstream(id, chain, client!!, rpcClient)
val created = EthereumGrpcUpstream(id, role, chain, client!!, rpcClient)
created.timeout = this.timeout
known[chain] = created
created.start()
@@ -214,7 +216,7 @@ class GrpcUpstreams(
val current = known[chain]
return if (current == null) {
val rpcClient = JsonRpcGrpcClient(client!!, chain, metrics)
val created = BitcoinGrpcUpstream(id, chain, client!!, rpcClient)
val created = BitcoinGrpcUpstream(id, role, chain, client!!, rpcClient)
created.timeout = this.timeout
known[chain] = created
created.start()

23
src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy
View File

@@ -343,8 +343,8 @@ class UpstreamsConfigReaderSpec extends Specification {
then:
act != null
act.upstreams.size() == 2
act.upstreams.get(0).role == UpstreamsConfig.UpstreamRole.STANDARD
act.upstreams.get(1).role == UpstreamsConfig.UpstreamRole.STANDARD
act.upstreams.get(0).role == UpstreamsConfig.UpstreamRole.PRIMARY
act.upstreams.get(1).role == UpstreamsConfig.UpstreamRole.PRIMARY
}
def "Parse config with fallback role"() {
@@ -355,10 +355,23 @@ class UpstreamsConfigReaderSpec extends Specification {
then:
act != null
act.upstreams.size() == 2
act.upstreams.get(0).role == UpstreamsConfig.UpstreamRole.STANDARD
act.upstreams.get(0).role == UpstreamsConfig.UpstreamRole.PRIMARY
act.upstreams.get(1).role == UpstreamsConfig.UpstreamRole.FALLBACK
}
def "Parse config with secondary role"() {
setup:
def config = this.class.getClassLoader().getResourceAsStream("upstreams-roles-2.yaml")
when:
def act = reader.read(config)
then:
act != null
act.upstreams.size() == 3
act.upstreams.get(0).role == UpstreamsConfig.UpstreamRole.PRIMARY
act.upstreams.get(1).role == UpstreamsConfig.UpstreamRole.SECONDARY
act.upstreams.get(2).role == UpstreamsConfig.UpstreamRole.FALLBACK
}
def "Parse config with invalid role"() {
setup:
def config = this.class.getClassLoader().getResourceAsStream("upstreams-roles-invalid.yaml")
@@ -367,7 +380,7 @@ class UpstreamsConfigReaderSpec extends Specification {
then:
act != null
act.upstreams.size() == 2
act.upstreams.get(0).role == UpstreamsConfig.UpstreamRole.STANDARD
act.upstreams.get(1).role == UpstreamsConfig.UpstreamRole.STANDARD
act.upstreams.get(0).role == UpstreamsConfig.UpstreamRole.PRIMARY
act.upstreams.get(1).role == UpstreamsConfig.UpstreamRole.PRIMARY
}
}

14
src/test/groovy/io/emeraldpay/dshackle/quorum/QuorumRpcReaderSpec.groovy
View File

@@ -38,7 +38,7 @@ class QuorumRpcReaderSpec extends Specification {
setup:
def up = Mock(Upstream) {
_ * isAvailable() >> true
_ * getRole() >> UpstreamsConfig.UpstreamRole.STANDARD
_ * getRole() >> UpstreamsConfig.UpstreamRole.PRIMARY
1 * getApi() >> Mock(Reader) {
1 * read(new JsonRpcRequest("eth_test", [])) >> Mono.just(JsonRpcResponse.ok("1"))
}
@@ -71,7 +71,7 @@ class QuorumRpcReaderSpec extends Specification {
}
def up = Mock(Upstream) {
_ * isAvailable() >> true
_ * getRole() >> UpstreamsConfig.UpstreamRole.STANDARD
_ * getRole() >> UpstreamsConfig.UpstreamRole.PRIMARY
_ * getApi() >> api
}
def apis = new FilteredApis(
@@ -98,7 +98,7 @@ class QuorumRpcReaderSpec extends Specification {
setup:
def up = Mock(Upstream) {
_ * isAvailable() >> true
_ * getRole() >> UpstreamsConfig.UpstreamRole.STANDARD
_ * getRole() >> UpstreamsConfig.UpstreamRole.PRIMARY
_ * getApi() >> Mock(Reader) {
2 * read(new JsonRpcRequest("eth_test", [])) >>> [
Mono.just(JsonRpcResponse.ok("null")),
@@ -130,7 +130,7 @@ class QuorumRpcReaderSpec extends Specification {
setup:
def up = Mock(Upstream) {
_ * isAvailable() >> true
_ * getRole() >> UpstreamsConfig.UpstreamRole.STANDARD
_ * getRole() >> UpstreamsConfig.UpstreamRole.PRIMARY
_ * getApi() >> Mock(Reader) {
2 * read(new JsonRpcRequest("eth_test", [])) >>> [
Mono.just(JsonRpcResponse.error(1, "test")),
@@ -161,7 +161,7 @@ class QuorumRpcReaderSpec extends Specification {
setup:
def up = Mock(Upstream) {
_ * isAvailable() >> true
_ * getRole() >> UpstreamsConfig.UpstreamRole.STANDARD
_ * getRole() >> UpstreamsConfig.UpstreamRole.PRIMARY
_ * getApi() >> Mock(Reader) {
3 * read(new JsonRpcRequest("eth_test", [])) >>> [
Mono.just(JsonRpcResponse.ok("null")),
@@ -200,7 +200,7 @@ class QuorumRpcReaderSpec extends Specification {
}
def up = Mock(Upstream) {
_ * isAvailable() >> true
_ * getRole() >> UpstreamsConfig.UpstreamRole.STANDARD
_ * getRole() >> UpstreamsConfig.UpstreamRole.PRIMARY
_ * getApi() >> api
}
def apis = new FilteredApis(
@@ -226,7 +226,7 @@ class QuorumRpcReaderSpec extends Specification {
def up = Mock(Upstream) {
_ * getLag() >> 0
_ * isAvailable() >> true
_ * getRole() >> UpstreamsConfig.UpstreamRole.STANDARD
_ * getRole() >> UpstreamsConfig.UpstreamRole.PRIMARY
_ * getApi() >> Mock(Reader) {
_ * read(new JsonRpcRequest("eth_test", [])) >>> [
Mono.just(JsonRpcResponse.error(-3010, "test")),

2
src/test/groovy/io/emeraldpay/dshackle/test/EthereumUpstreamMock.groovy
View File

@@ -63,7 +63,7 @@ class EthereumUpstreamMock extends EthereumRpcUpstream {
EthereumUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader<JsonRpcRequest, JsonRpcResponse> api, CallMethods methods) {
super(id, chain, api, null,
UpstreamsConfig.Options.getDefaults(),
UpstreamsConfig.UpstreamRole.STANDARD,
UpstreamsConfig.UpstreamRole.PRIMARY,
new QuorumForLabels.QuorumItem(1, new UpstreamsConfig.Labels()),
methods)
setLag(0)

45
src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy
View File

@@ -52,7 +52,7 @@ class FilteredApisSpec extends Specification {
TestingCommons.api().tap { it.id = "${i++}" },
(EthereumWsFactory) null,
new UpstreamsConfig.Options(),
UpstreamsConfig.UpstreamRole.STANDARD,
UpstreamsConfig.UpstreamRole.PRIMARY,
new QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(it)),
ethereumTargets
)
@@ -210,7 +210,7 @@ class FilteredApisSpec extends Specification {
6 | [0, 1]
}
def "Starts with standard"() {
def "Starts with primary"() {
setup:
List<Upstream> standard = (0..1).collect {
TestingCommons.upstream(
@@ -241,4 +241,45 @@ class FilteredApisSpec extends Specification {
.expectComplete()
.verify(Duration.ofSeconds(1))
}
def "Use secondary after primary"() {
setup:
List<Upstream> standard = (0..1).collect {
TestingCommons.upstream(
it.toString(),
new EthereumApiStub(it)
)
}
List<Upstream> fallback = [
Mock([name: "fallback"], Upstream) {
_ * getRole() >> UpstreamsConfig.UpstreamRole.FALLBACK
_ * isAvailable() >> true
}
]
List<Upstream> secondary = [
Mock([name: "secondary"], Upstream) {
_ * getRole() >> UpstreamsConfig.UpstreamRole.SECONDARY
_ * isAvailable() >> true
}
]
when:
def act = new FilteredApis(Chain.ETHEREUM,
[] + fallback + standard + secondary,
Selector.empty, 0, 3, 0)
act.request(11)
then:
StepVerifier.create(act)
.expectNext(standard[0], standard[1]).as("Initial requests with primary")
.expectNext(secondary[0]).as("Initial requests with secondary")
.expectNext(standard[0], standard[1]).as("Retry with primary")
.expectNext(secondary[0]).as("Retry with secondary")
.expectNext(fallback[0]).as("Retry with fallback")
.expectNext(standard[0], standard[1]).as("Second retry with primary")
.expectNext(secondary[0]).as("Second retry with secondary")
.expectNext(fallback[0]).as("Second retry with fallback")
.expectComplete()
.verify(Duration.ofSeconds(1))
}
}

7
src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy
View File

@@ -22,6 +22,7 @@ import io.emeraldpay.api.proto.BlockchainGrpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.test.MockGrpcServer
import io.emeraldpay.dshackle.test.TestingCommons
@@ -80,7 +81,7 @@ class EthereumGrpcUpstreamSpec extends Specification {
)
}
})
def upstream = new EthereumGrpcUpstream("test", chain, client, new JsonRpcGrpcClient(client, chain, metrics))
def upstream = new EthereumGrpcUpstream("test", UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics))
upstream.setLag(0)
upstream.update(BlockchainOuterClass.DescribeChain.newBuilder()
.setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId))
@@ -138,7 +139,7 @@ class EthereumGrpcUpstreamSpec extends Specification {
)
}
})
def upstream = new EthereumGrpcUpstream("test", Chain.ETHEREUM, client, new JsonRpcGrpcClient(client, Chain.ETHEREUM, metrics))
def upstream = new EthereumGrpcUpstream("test", UpstreamsConfig.UpstreamRole.PRIMARY, Chain.ETHEREUM, client, new JsonRpcGrpcClient(client, Chain.ETHEREUM, metrics))
upstream.setLag(0)
upstream.update(BlockchainOuterClass.DescribeChain.newBuilder()
.setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId))
@@ -200,7 +201,7 @@ class EthereumGrpcUpstreamSpec extends Specification {
finished.complete(true)
}
})
def upstream = new EthereumGrpcUpstream("test", chain, client, new JsonRpcGrpcClient(client, chain, metrics))
def upstream = new EthereumGrpcUpstream("test", UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics))
upstream.setLag(0)
upstream.update(BlockchainOuterClass.DescribeChain.newBuilder()
.setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId))

44
src/test/resources/upstreams-roles-2.yaml Normal file
View File

@@ -0,0 +1,44 @@
version: v1
defaults:
- chains:
- ethereum
options:
min-peers: 3
upstreams:
- id: local
chain: ethereum
connection:
ethereum:
rpc:
url: "http://localhost:8545"
ws:
url: "ws://localhost:8546"
origin: "http://localhost"
basic-auth:
username: 9c199ad8f281f20154fc258fe41a6814
password: 258fe4149c199ad8f2811a68f20154fc
- id: closeby
chain: ethereum
role: secondary
connection:
ethereum:
rpc:
url: "http://localhost:8545"
ws:
url: "ws://localhost:8546"
origin: "http://localhost"
basic-auth:
username: 9c199ad8f281f20154fc258fe41a6814
password: 258fe4149c199ad8f2811a68f20154fc
- id: infura
chain: ethereum
role: fallback
connection:
ethereum:
rpc:
url: "https://mainnet.infura.io/v3/fa28c968191849c1aff541ad1d8511f2"
basic-auth:
username: 4fc258fe41a68149c199ad8f281f2015
password: 1a68f20154fc258fe4149c199ad8f281