StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

solution: individual class for quorum based requests

Browse Source
This commit is contained in:
Igor Artamonov
2020-05-15 21:06:32 -04:00
parent 73f55d8d81
commit 114f109722
19 changed files with 611 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/kotlin/io/emeraldpay/dshackle/quorum/AlwaysQuorum.kt
View File

@@ -32,6 +32,10 @@ open class AlwaysQuorum: CallQuorum {
return resolved
}
override fun isFailed(): Boolean {
return false
}
override fun record(response: ByteArray, upstream: Upstream): Boolean {
result = response
resolved = true

4
src/main/kotlin/io/emeraldpay/dshackle/quorum/BroadcastQuorum.kt
View File

@@ -37,6 +37,10 @@ open class BroadcastQuorum(
return calls >= quorum
}
override fun isFailed(): Boolean {
return false
}
override fun getResult(): ByteArray? {
return result
}

2
src/main/kotlin/io/emeraldpay/dshackle/quorum/CallQuorum.kt
View File

@@ -31,6 +31,8 @@ interface CallQuorum {
fun init(head: Head)
fun isResolved(): Boolean
fun isFailed(): Boolean
fun record(response: ByteArray, upstream: Upstream): Boolean
fun record(error: RpcException, upstream: Upstream)
fun getResult(): ByteArray?

8
src/main/kotlin/io/emeraldpay/dshackle/quorum/NonEmptyQuorum.kt
View File

@@ -34,7 +34,11 @@ open class NonEmptyQuorum(
}
override fun isResolved(): Boolean {
return result != null || tries >= maxTries
return result != null
}
override fun isFailed(): Boolean {
return tries >= maxTries
}
override fun recordValue(response: ByteArray, responseValue: Any?, upstream: Upstream) {
@@ -49,9 +53,11 @@ open class NonEmptyQuorum(
}
override fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream) {
tries++
}
override fun record(error: RpcException, upstream: Upstream) {
tries++
}
}

6
src/main/kotlin/io/emeraldpay/dshackle/quorum/NonceQuorum.kt
View File

@@ -41,10 +41,14 @@ open class NonceQuorum(
override fun isResolved(): Boolean {
lock.withLock {
return receivedTimes >= tries || errors >= tries
return receivedTimes >= tries && !isFailed()
}
}
override fun isFailed(): Boolean {
return errors >= tries
}
override fun recordValue(response: ByteArray, responseValue: String?, upstream: Upstream) {
val value = responseValue?.let { str ->
HexQuantity.from(str).value.toLong()

4
src/main/kotlin/io/emeraldpay/dshackle/quorum/NotLaggingQuorum.kt
View File

@@ -32,6 +32,10 @@ class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum {
return result.get() != null
}
override fun isFailed(): Boolean {
return false
}
override fun record(response: ByteArray, upstream: Upstream): Boolean {
val lagging = upstream.getLag() > maxLag
if (!lagging) {

38
src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumReaderFactory.kt Normal file
View File

@@ -0,0 +1,38 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.ApiSource
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
// creates instance of a Quorum based reader
interface QuorumReaderFactory {
companion object {
fun default(): QuorumReaderFactory {
return Default()
}
}
fun create(apis: ApiSource, quorum: CallQuorum): Reader<JsonRpcRequest, QuorumRpcReader.Result>
class Default : QuorumReaderFactory {
override fun create(apis: ApiSource, quorum: CallQuorum): Reader<JsonRpcRequest, QuorumRpcReader.Result> {
return QuorumRpcReader(apis, quorum)
}
}
}

105
src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRpcReader.kt Normal file
View File

@@ -0,0 +1,105 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.ApiSource
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.infinitape.etherjar.rpc.RpcException
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.util.function.Tuples
/**
* Makes request with applying Quorum
*/
class QuorumRpcReader(
private val apis: ApiSource,
private val quorum: CallQuorum
) : Reader<JsonRpcRequest, QuorumRpcReader.Result> {
companion object {
private val log = LoggerFactory.getLogger(QuorumRpcReader::class.java)
}
override fun read(key: JsonRpcRequest): Mono<QuorumRpcReader.Result> {
apis.request(1)
// uses a mix of retry strategy and managed Publisher for calls.
// retry is used when an error happened
// but if no error received, we check quorum and if not enough data received we request more
// eventually source of upstreams is Completed (or something Errored) and if finalizes the result
val retrySpec = reactor.util.retry.Retry.from { signal ->
signal.takeUntil {
it.totalRetries() >= 3 || quorum.isResolved() || quorum.isFailed()
}.doOnNext {
// need one more API source if retried
apis.request(1)
}
}
return Flux.from(apis)
.flatMap { api ->
api.getApi().read(key)
.flatMap(JsonRpcResponse::requireResult)
// on error notify quorum, it may use error message or other details
.doOnError { err ->
if (err is RpcException) {
quorum.record(err, api)
}
}
.map { Tuples.of(it, api) }
}
.retryWhen(retrySpec)
// record all correct responses until quorum reached
.reduce(quorum, { res, a ->
if (res.record(a.t1, a.t2)) {
apis.resolve()
} else {
apis.request(1)
}
res
})
// if last call resulted in error it's still possible that request was resolved correctly. i.e. for BroadcastQuorum
.onErrorResume { err ->
if (quorum.isResolved()) {
Mono.just(quorum)
} else {
Mono.error(err)
}
}
.doOnNext {
if (!it.isResolved()) {
log.debug("No quorum for ${key.method} as ${quorum}")
}
}
// return nothing if not resolved
.filter { it.isResolved() }
.map {
// TODO find actual quorum number
QuorumRpcReader.Result(it.getResult()!!, 1)
}
}
class Result(
val value: ByteArray,
val quorum: Int
)
}

61
src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt
View File

@@ -24,6 +24,8 @@ import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.dshackle.quorum.AlwaysQuorum
import io.emeraldpay.dshackle.quorum.CallQuorum
import io.emeraldpay.dshackle.quorum.QuorumReaderFactory
import io.emeraldpay.dshackle.quorum.QuorumRpcReader
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
@@ -46,6 +48,8 @@ open class NativeCall(
private val log = LoggerFactory.getLogger(NativeCall::class.java)
var quorumReaderFactory: QuorumReaderFactory = QuorumReaderFactory.default()
open fun nativeCall(requestMono: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<BlockchainOuterClass.NativeCallReplyItem> {
return requestMono.flatMapMany(this::prepareCall)
.map(this::setupCallParams)
@@ -138,61 +142,10 @@ open class NativeCall(
if (!ctx.upstream.getMethods().isAllowed(ctx.payload.method)) {
return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method"))
}
//TODO move to routed api
val apis = ctx.getApis()
apis.request(1)
var failures = 0
return Flux.from(apis)
.flatMap { api ->
val upstream = ctx.upstream
api.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
.flatMap(JsonRpcResponse::requireResult)
// on error notify quorum, it may use error message or other details
.doOnError { err ->
if (err is RpcException) {
ctx.callQuorum.record(err, upstream)
}
}
.map { Tuples.of(it, upstream) }
}
.retry {
failures++
if (ctx.callQuorum.isResolved()) {
false
} else if (failures < 3) {
apis.request(1)
true
} else {
false
}
}
// record all correct responses until quorum reached
.reduce(ctx.callQuorum, {res, a ->
if (res.record(a.t1, a.t2)) {
apis.resolve()
} else {
apis.request(1)
}
res
})
// if last call resulted in error it's still possible that request was resolved correctly. i.e. for BroadcastQuorum
.onErrorResume { err ->
if (ctx.callQuorum.isResolved()) {
Mono.just(ctx.callQuorum)
} else {
Mono.error(err)
}
}
.doOnNext {
if (!it.isResolved()) {
log.debug("No quorum for ${ctx.payload.method} as ${ctx.callQuorum}")
}
}
.filter { it.isResolved() }
val reader = quorumReaderFactory.create(ctx.getApis(), ctx.callQuorum)
return reader.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
.map {
val result = it.getResult()
?: throw CallFailure(ctx.id, Exception("No response from upstream for ${ctx.payload.method}"))
ctx.withPayload(result)
ctx.withPayload(it.value)
}
.onErrorMap {
log.error("Failed to make a call", it)

2
src/main/kotlin/io/emeraldpay/dshackle/upstream/ApiSource.kt
View File

@@ -21,7 +21,7 @@ import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.reactivestreams.Publisher
interface ApiSource : Publisher<Reader<JsonRpcRequest, JsonRpcResponse>> {
interface ApiSource : Publisher<Upstream> {
fun resolve()
fun request(tries: Int)

4
src/main/kotlin/io/emeraldpay/dshackle/upstream/FilteredApis.kt
View File

@@ -78,7 +78,7 @@ class FilteredApis(
return Duration.ofMillis(time)
}
override fun subscribe(subscriber: Subscriber<in Reader<JsonRpcRequest, JsonRpcResponse>>) {
override fun subscribe(subscriber: Subscriber<in Upstream>) {
val first = Flux.fromIterable(upstreams)
val retries = (1 until repeatLimit).map { r ->
Flux.fromIterable(upstreams).delaySubscription(waitDuration(r))
@@ -87,7 +87,6 @@ class FilteredApis(
Flux.concat(first, retries)
.filter(Upstream::isAvailable)
.filter(matcher::matches)
.map { it.getApi() }
.zipWith(control)
.map { it.t1 }
.subscribe(subscriber)
@@ -98,6 +97,7 @@ class FilteredApis(
}
override fun request(tries: Int) {
println("requested ${tries}")
//TODO check the buffer size before submitting
repeat(tries) {
control.onNext(true)

1
src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt
View File

@@ -101,6 +101,7 @@ abstract class Multistream(
val apis = getApiSource(matcher)
apis.request(1)
return Mono.from(apis)
.map(Upstream::getApi)
.switchIfEmpty(Mono.error(Exception("No API available for $chain")))
}

15
src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcResponse.kt
View File

@@ -28,6 +28,21 @@ class JsonRpcResponse(
companion object {
private val NULL_VALUE = "null".toByteArray()
@JvmStatic
fun ok(value: ByteArray): JsonRpcResponse {
return JsonRpcResponse(value, null)
}
@JvmStatic
fun ok(value: String): JsonRpcResponse {
return JsonRpcResponse(value.toByteArray(), null)
}
@JvmStatic
fun error(code: Int, msg: String): JsonRpcResponse {
return JsonRpcResponse(null, ResponseError(code, msg))
}
}
fun hasResult(): Boolean {