StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix deadlock, remove setting UNAVAIL status (#333)

Browse Source
This commit is contained in:
KirillPamPam
2023-11-06 16:32:12 +04:00
committed by GitHub
parent 160e5ae26c
commit e4c13ad1ad
34 changed files with 251 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/test/groovy/io/emeraldpay/dshackle/startup/ConfiguredUpstreamsSpec.groovy
View File

@@ -35,6 +35,8 @@ class ConfiguredUpstreamsSpec extends Specification {
Schedulers.boundedElastic(),
null,
Schedulers.boundedElastic(),
Schedulers.boundedElastic(),
Schedulers.boundedElastic(),
AuthorizationConfig.default(),
new GrpcAuthContext()
)
@@ -69,6 +71,8 @@ class ConfiguredUpstreamsSpec extends Specification {
Schedulers.boundedElastic(),
null,
Schedulers.boundedElastic(),
Schedulers.boundedElastic(),
Schedulers.boundedElastic(),
AuthorizationConfig.default(),
new GrpcAuthContext()
)
@@ -102,6 +106,8 @@ class ConfiguredUpstreamsSpec extends Specification {
Schedulers.boundedElastic(),
null,
Schedulers.boundedElastic(),
Schedulers.boundedElastic(),
Schedulers.boundedElastic(),
AuthorizationConfig.default(),
new GrpcAuthContext()
)
@@ -130,6 +136,8 @@ class ConfiguredUpstreamsSpec extends Specification {
Schedulers.boundedElastic(),
null,
Schedulers.boundedElastic(),
Schedulers.boundedElastic(),
Schedulers.boundedElastic(),
AuthorizationConfig.default(),
new GrpcAuthContext()
)
@@ -163,6 +171,8 @@ class ConfiguredUpstreamsSpec extends Specification {
Schedulers.boundedElastic(),
null,
Schedulers.boundedElastic(),
Schedulers.boundedElastic(),
Schedulers.boundedElastic(),
AuthorizationConfig.default(),
new GrpcAuthContext()
)

5
src/test/groovy/io/emeraldpay/dshackle/test/EthereumHeadMock.groovy
View File

@@ -82,4 +82,9 @@ class EthereumHeadMock implements Head {
void onSyncingNode(boolean isSyncing) {
}
@Override
Flux<Boolean> headLiveness() {
return Flux.empty()
}
}

1
src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy
View File

@@ -64,6 +64,7 @@ class FilteredApisSpec extends Specification {
BlockValidator.ALWAYS_VALID,
Schedulers.boundedElastic(),
Schedulers.boundedElastic(),
Schedulers.boundedElastic(),
Duration.ofSeconds(12)
)
new GenericUpstream(

95
src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/GenericWsHeadSpec.groovy
View File

@@ -24,10 +24,10 @@ import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson
import io.emeraldpay.dshackle.upstream.forkchoice.AlwaysForkChoice
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.domain.TransactionId
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.core.publisher.Sinks
@@ -38,6 +38,7 @@ import spock.lang.Specification
import java.time.Duration
import java.time.Instant
import java.time.temporal.ChronoUnit
import java.util.concurrent.atomic.AtomicReference
class GenericWsHeadSpec extends Specification {
@@ -74,7 +75,7 @@ class GenericWsHeadSpec extends Specification {
act == res
1 * ws.subscribe(_) >> new WsSubscriptions.SubscribeData(
Flux.fromIterable([headBlock]), "id"
Flux.fromIterable([headBlock]), "id", new AtomicReference<String>("")
)
}
@@ -96,8 +97,8 @@ class GenericWsHeadSpec extends Specification {
def ws = Mock(WsSubscriptions) {
1 * it.connectionInfoFlux() >> connectionInfoSink.asFlux()
2 * subscribe(_) >>> [
new WsSubscriptions.SubscribeData(Flux.error(new RuntimeException()), "id"),
new WsSubscriptions.SubscribeData(Flux.fromIterable([secondHeadBlock]), "id")
new WsSubscriptions.SubscribeData(Flux.error(new RuntimeException()), "id", new AtomicReference<String>("")),
new WsSubscriptions.SubscribeData(Flux.fromIterable([secondHeadBlock]), "id", new AtomicReference<String>(""))
]
}
@@ -150,8 +151,8 @@ class GenericWsHeadSpec extends Specification {
def ws = Mock(WsSubscriptions) {
1 * it.connectionInfoFlux() >> connectionInfoSink.asFlux()
2 * subscribe(_) >>> [
new WsSubscriptions.SubscribeData(Flux.fromIterable([firstHeadBlock]), "id"),
new WsSubscriptions.SubscribeData(Flux.fromIterable([secondHeadBlock]), "id")
new WsSubscriptions.SubscribeData(Flux.fromIterable([firstHeadBlock]), "id", new AtomicReference<String>("")),
new WsSubscriptions.SubscribeData(Flux.fromIterable([secondHeadBlock]), "id", new AtomicReference<String>(""))
]
}
@@ -191,7 +192,7 @@ class GenericWsHeadSpec extends Specification {
def ws = Mock(WsSubscriptions) {
1 * it.connectionInfoFlux() >> connectionInfoSink.asFlux()
1 * subscribe(_) >>> [
new WsSubscriptions.SubscribeData(Flux.fromIterable([firstHeadBlock]), "id"),
new WsSubscriptions.SubscribeData(Flux.fromIterable([firstHeadBlock]), "id", new AtomicReference<String>("")),
]
}
@@ -230,7 +231,7 @@ class GenericWsHeadSpec extends Specification {
def ws = Mock(WsSubscriptions) {
1 * it.connectionInfoFlux() >> connectionInfoSink.asFlux()
1 * subscribe(_) >>> [
new WsSubscriptions.SubscribeData(Flux.fromIterable([firstHeadBlock]), "id"),
new WsSubscriptions.SubscribeData(Flux.fromIterable([firstHeadBlock]), "id", new AtomicReference<String>("")),
]
}
@@ -282,8 +283,8 @@ class GenericWsHeadSpec extends Specification {
def ws = Mock(WsSubscriptions) {
1 * it.connectionInfoFlux() >> connectionInfoSink.asFlux()
2 * subscribe(_) >>> [
new WsSubscriptions.SubscribeData(Flux.fromIterable([firstHeadBlock]), "id"),
new WsSubscriptions.SubscribeData(Flux.fromIterable([secondHeadBlock]), "id"),
new WsSubscriptions.SubscribeData(Flux.fromIterable([firstHeadBlock]), "id", new AtomicReference<String>("")),
new WsSubscriptions.SubscribeData(Flux.fromIterable([secondHeadBlock]), "id", new AtomicReference<String>("")),
]
}
@@ -316,4 +317,74 @@ class GenericWsHeadSpec extends Specification {
.thenCancel()
.verify(Duration.ofSeconds(1))
}
def "Unsubscribe if there is an error during subscription"() {
setup:
def block = new BlockJson<TransactionRefJson>()
block.number = 100
block.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200")
block.parentHash = parent
block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block.uncles = []
block.totalDifficulty = BigInteger.ONE
def apiMock = TestingCommons.api()
def subId = "subId"
def ws = Mock(WsSubscriptions) {
1 * it.connectionInfoFlux() >> Flux.empty()
1 * it.subscribe(_) >> new WsSubscriptions.SubscribeData(
Flux.error(new RuntimeException()), "id", new AtomicReference<String>(subId)
)
1 * it.unsubscribe(new JsonRpcRequest("eth_unsubscribe", List.of(subId), 2, null, null)) >>
Mono.just(new JsonRpcResponse("".bytes, null))
}
def head = new GenericWsHead(new AlwaysForkChoice(), BlockValidator.ALWAYS_VALID, apiMock, ws, Schedulers.boundedElastic(), Schedulers.boundedElastic(), upstream, EthereumChainSpecific.INSTANCE)
when:
def act = head.listenNewHeads()
then:
StepVerifier.create(act)
.expectComplete()
.verify(Duration.ofSeconds(1))
}
def "If there is ws disconnect then head must emit false its liveness state"() {
setup:
def secondBlock = new BlockJson<TransactionRefJson>()
secondBlock.parentHash = parent
secondBlock.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
secondBlock.number = 105
secondBlock.hash = BlockHash.from("0x29229361dc5aa1ec66c323dc7a299e2b61a8c8dd2a3522d41255ec10eca25dd8")
def secondHeadBlock = secondBlock.with {
Global.objectMapper.writeValueAsBytes(it)
}
def apiMock = TestingCommons.api()
def connectionInfoSink = Sinks.many().multicast().directBestEffort()
def ws = Mock(WsSubscriptions) {
1 * it.connectionInfoFlux() >> connectionInfoSink.asFlux()
1 * subscribe(_) >>> [
new WsSubscriptions.SubscribeData(Flux.fromIterable([secondHeadBlock]), "id", new AtomicReference<String>(""))
]
}
def head = new GenericWsHead(new AlwaysForkChoice(), BlockValidator.ALWAYS_VALID, apiMock, ws, Schedulers.boundedElastic(), Schedulers.boundedElastic(), upstream, EthereumChainSpecific.INSTANCE)
when:
head.start()
def liveness = head.headLiveness()
then:
StepVerifier.create(liveness)
.then {
connectionInfoSink.tryEmitNext(new WsConnection.ConnectionInfo("id", WsConnection.ConnectionState.DISCONNECTED))
}
.expectNext(false)
.thenCancel()
.verify(Duration.ofSeconds(1))
}
}

18
src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/HeadLivenessValidatorSpec.groovy
View File

@@ -2,13 +2,13 @@ package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.test.EthereumHeadMock
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.Head
import reactor.core.publisher.Flux
import reactor.core.scheduler.Schedulers
import reactor.test.StepVerifier
import spock.lang.Specification
import java.time.Duration
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicInteger
class HeadLivenessValidatorSpec extends Specification{
def "emits true"() {
@@ -24,6 +24,20 @@ class HeadLivenessValidatorSpec extends Specification{
}.expectNext(true).thenCancel().verify(Duration.ofSeconds(1))
}
def "emits false if head liveness emits false"() {
when:
def head = Mock(Head) {
1 * it.headLiveness() >> Flux.just(false)
1 * it.getFlux() >> Flux.just(TestingCommons.blockForEthereum(1))
}
def checker = new HeadLivenessValidator(head, Duration.ofSeconds(10), Schedulers.boundedElastic(), "test")
then:
StepVerifier.create(checker.flux)
.expectNext(false)
.thenCancel()
.verify(Duration.ofSeconds(1))
}
def "starts accumulating trues but immediately emits after false"() {
when:
def head = new EthereumHeadMock()

28
src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplRealSpec.groovy
View File

@@ -5,7 +5,6 @@ import io.emeraldpay.dshackle.test.GenericUpstreamMock
import io.emeraldpay.dshackle.test.MockWSServer
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import reactor.core.scheduler.Schedulers
import reactor.test.StepVerifier
@@ -110,33 +109,6 @@ class WsConnectionImplRealSpec extends Specification {
.verify(Duration.ofSeconds(1))
}
def "Gets UNAVAIL status right after disconnect"() {
setup:
def up = Mock(DefaultUpstream) {
_ * getId() >> "test"
}
conn = new WsConnectionPoolFactory(
"test",
1,
new WsConnectionFactory(
"test",
Chain.ETHEREUM__MAINNET,
"ws://localhost:${port}".toURI(),
"http://localhost:${port}".toURI(),
Schedulers.boundedElastic()
)
).create(up).getConnection()
when:
conn.connect()
conn.reconnectIntervalSeconds = 10
Thread.sleep(SLEEP)
server.stop()
Thread.sleep(100)
then:
1 * up.setStatus(UpstreamAvailability.UNAVAILABLE)
}
def "Try to connects to server until it's available"() {
when:
server.stop()

28
src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionMultiPoolSpec.groovy
View File

@@ -30,14 +30,14 @@ class WsConnectionMultiPoolSpec extends Specification {
}
def up = Mock(DefaultUpstream)
def factory = Mock(WsConnectionFactory)
def pool = new WsConnectionMultiPool(factory, up, 3)
def pool = new WsConnectionMultiPool(factory, 3)
pool.scheduler = Stub(ScheduledExecutorService)
when:
pool.connect()
then:
1 * factory.createWsConnection(0, _) >> conn
1 * factory.createWsConnection(0) >> conn
1 * conn.connect()
}
@@ -54,14 +54,14 @@ class WsConnectionMultiPoolSpec extends Specification {
}
def up = Mock(DefaultUpstream)
def factory = Mock(WsConnectionFactory)
def pool = new WsConnectionMultiPool(factory, up, 3)
def pool = new WsConnectionMultiPool(factory, 3)
pool.scheduler = Stub(ScheduledExecutorService)
when:
pool.connect()
then:
1 * factory.createWsConnection(0, _) >> conn1
1 * factory.createWsConnection(0) >> conn1
1 * conn1.connect()
when:
@@ -69,7 +69,7 @@ class WsConnectionMultiPoolSpec extends Specification {
then:
1 * conn1.isConnected() >> true
1 * factory.createWsConnection(1, _) >> conn2
1 * factory.createWsConnection(1) >> conn2
1 * conn2.connect()
when:
@@ -78,7 +78,7 @@ class WsConnectionMultiPoolSpec extends Specification {
then:
1 * conn1.isConnected() >> true
1 * conn2.isConnected() >> true
1 * factory.createWsConnection(2, _) >> conn3
1 * factory.createWsConnection(2) >> conn3
1 * conn3.connect()
when:
@@ -88,7 +88,7 @@ class WsConnectionMultiPoolSpec extends Specification {
1 * conn1.isConnected() >> true
1 * conn2.isConnected() >> true
1 * conn3.isConnected() >> true
0 * factory.createWsConnection(_, _)
0 * factory.createWsConnection(_)
}
def "recreate connection after failure"() {
@@ -107,7 +107,7 @@ class WsConnectionMultiPoolSpec extends Specification {
}
def up = Mock(DefaultUpstream)
def factory = Mock(WsConnectionFactory)
def pool = new WsConnectionMultiPool(factory, up, 3)
def pool = new WsConnectionMultiPool(factory, 3)
pool.scheduler = Stub(ScheduledExecutorService)
when: "initial fill"
@@ -119,9 +119,9 @@ class WsConnectionMultiPoolSpec extends Specification {
_ * conn1.isConnected() >> true
_ * conn2.isConnected() >> true
_ * conn3.isConnected() >> true
1 * factory.createWsConnection(0, _) >> conn1
1 * factory.createWsConnection(1, _) >> conn2
1 * factory.createWsConnection(2, _) >> conn3
1 * factory.createWsConnection(0) >> conn1
1 * factory.createWsConnection(1) >> conn2
1 * factory.createWsConnection(2) >> conn3
1 * conn1.connect()
1 * conn2.connect()
1 * conn3.connect()
@@ -133,7 +133,7 @@ class WsConnectionMultiPoolSpec extends Specification {
1 * conn1.isConnected() >> true
1 * conn2.isConnected() >> true
1 * conn3.isConnected() >> true
0 * factory.createWsConnection(_, _)
0 * factory.createWsConnection(_)
when: "one failed"
pool.connect()
@@ -142,7 +142,7 @@ class WsConnectionMultiPoolSpec extends Specification {
(1.._) * conn1.isConnected() >> true
(1.._) * conn2.isConnected() >> false
(1.._) * conn3.isConnected() >> true
0 * factory.createWsConnection(_, _) // doesn't create immediately, but schedules it for the next adjust
0 * factory.createWsConnection(_) // doesn't create immediately, but schedules it for the next adjust
1 * conn2.close()
when: "needs one more"
@@ -151,7 +151,7 @@ class WsConnectionMultiPoolSpec extends Specification {
then:
1 * conn1.isConnected() >> true
1 * conn3.isConnected() >> true
1 * factory.createWsConnection(3, _) >> conn4
1 * factory.createWsConnection(3) >> conn4
1 * conn4.connect()
}
}

3
src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/WebsocketPendingTxesSpec.groovy
View File

@@ -22,6 +22,7 @@ import reactor.core.publisher.Flux
import spock.lang.Specification
import java.time.Duration
import java.util.concurrent.atomic.AtomicReference
class WebsocketPendingTxesSpec extends Specification {
@@ -42,7 +43,7 @@ class WebsocketPendingTxesSpec extends Specification {
then:
1 * ws.subscribe(new JsonRpcRequest("eth_subscribe", ["newPendingTransactions"])) >> new WsSubscriptions.SubscribeData(
Flux.fromIterable(responses), "id"
Flux.fromIterable(responses), "id", new AtomicReference<String>("")
)
txes.collect {it.toHex() } == [
"0xa61bab14fc9720ea8725622688c2f964666d7c2afdae38af7dad53f12f242d5c",