StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stream responses from node (#361)

Browse Source
This commit is contained in:
KirillPamPam
2023-12-14 12:09:25 +04:00
committed by GitHub
parent a9a3e64c32
commit 6e8f600cdc
39 changed files with 768 additions and 450 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
src/test/kotlin/io/emeraldpay/dshackle/rpc/NativeCallStreamTest.kt
View File

@@ -1,175 +0,0 @@
package io.emeraldpay.dshackle.rpc
import com.fasterxml.jackson.databind.JsonNode
import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.BlockchainOuterClass.NativeCallRequest
import io.emeraldpay.dshackle.Global
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import org.mockito.kotlin.any
import org.mockito.kotlin.doReturn
import org.mockito.kotlin.mock
import org.springframework.util.ResourceUtils
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.test.StepVerifier
import java.time.Duration
class NativeCallStreamTest {
private val upstreamId = "upstreamId"
private val mapper = Global.objectMapper
@Test
fun `streaming response is equal to the original response`() {
val responseFile = ResourceUtils.getFile("classpath:responses/get-by-number-response.json")
val response = mapper.writeValueAsBytes(mapper.readValue(responseFile, JsonNode::class.java))
val nativeCallResponse = BlockchainOuterClass.NativeCallReplyItem.newBuilder()
.setId(1)
.setSucceed(true)
.setUpstreamId(upstreamId)
.setPayload(ByteString.copyFrom(response))
.build()
val nativeCallMock = mock<NativeCall> {
on { nativeCall(any()) } doReturn Flux.just(nativeCallResponse)
}
val nativeCallStream = NativeCallStream(nativeCallMock)
val req = Mono.just(
NativeCallRequest.newBuilder()
.setChunkSize(1000)
.build(),
)
val result = nativeCallStream.nativeCall(req)
.collectList()
.block()!!
.map { it.payload.toByteArray() }
.reduce { acc, bytes -> acc.plus(bytes) }
assertTrue(response.contentEquals(result))
}
@Test
fun `streaming responses is correct`() {
val response = "\"0x1126938\"".toByteArray()
val nativeCallResponse = BlockchainOuterClass.NativeCallReplyItem.newBuilder()
.setId(15)
.setSucceed(true)
.setUpstreamId(upstreamId)
.setPayload(ByteString.copyFrom(response))
.build()
val nativeCallMock = mock<NativeCall> {
on { nativeCall(any()) } doReturn Flux.just(nativeCallResponse)
}
val nativeCallStream = NativeCallStream(nativeCallMock)
val req = Mono.just(
NativeCallRequest.newBuilder()
.setChunkSize(5)
.build(),
)
val chunkResponse: (Int) -> BlockchainOuterClass.NativeCallReplyItem.Builder = { id ->
BlockchainOuterClass.NativeCallReplyItem.newBuilder()
.setId(id)
.setChunked(true)
.setSucceed(true)
.setUpstreamId(upstreamId)
}
val result = nativeCallStream.nativeCall(req)
StepVerifier.create(result)
.expectNext(
chunkResponse(15)
.setPayload(ByteString.copyFrom("\"0x11".toByteArray()))
.build(),
)
.expectNext(
chunkResponse(15)
.setPayload(ByteString.copyFrom("26938".toByteArray()))
.build(),
)
.expectNext(
chunkResponse(15)
.setFinalChunk(true)
.setPayload(ByteString.copyFrom("\"".toByteArray()))
.build(),
)
.expectComplete()
.verify(Duration.ofSeconds(3))
}
@Test
fun `no streaming if response is too small`() {
val response = "\"0x1\"".toByteArray()
val nativeCallResponse = BlockchainOuterClass.NativeCallReplyItem.newBuilder()
.setId(15)
.setSucceed(true)
.setUpstreamId(upstreamId)
.setPayload(ByteString.copyFrom(response))
.build()
val nativeCallMock = mock<NativeCall> {
on { nativeCall(any()) } doReturn Flux.just(nativeCallResponse)
}
val nativeCallStream = NativeCallStream(nativeCallMock)
val req = Mono.just(
NativeCallRequest.newBuilder()
.setChunkSize(1000)
.build(),
)
val result = nativeCallStream.nativeCall(req)
StepVerifier.create(result)
.expectNext(
nativeCallResponse,
)
.expectComplete()
.verify(Duration.ofSeconds(3))
}
@Test
fun `sort responses by request id is correct`() {
val response = "\"0x1\"".toByteArray()
val response2 = "\"0x2\"".toByteArray()
val response3 = "\"0x3\"".toByteArray()
val nativeCallResponse: (Int, ByteArray) -> BlockchainOuterClass.NativeCallReplyItem = { id, resp ->
BlockchainOuterClass.NativeCallReplyItem.newBuilder()
.setId(id)
.setChunked(true)
.setSucceed(true)
.setUpstreamId(upstreamId)
.setPayload(ByteString.copyFrom(resp))
.build()
}
val nativeCallMock = mock<NativeCall> {
on { nativeCall(any()) } doReturn Flux.just(
nativeCallResponse(1, response),
nativeCallResponse(2, response2),
nativeCallResponse(3, response3),
).flatMap {
when (it.id) {
1 -> Mono.just(it).delayElement(Duration.ofMillis(200))
2 -> Mono.just(it).delayElement(Duration.ofMillis(100))
else -> Mono.just(it)
}
}
}
val nativeCallStream = NativeCallStream(nativeCallMock)
val req = Mono.just(
NativeCallRequest.newBuilder()
.setSorted(true)
.build(),
)
val result = nativeCallStream.nativeCall(req)
StepVerifier.create(result)
.expectNextMatches { it.payload.toByteArray().contentEquals(response) }
.expectNextMatches { it.payload.toByteArray().contentEquals(response2) }
.expectNextMatches { it.payload.toByteArray().contentEquals(response3) }
.expectComplete()
.verify(Duration.ofSeconds(3))
}
}

131
src/test/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/stream/JsonRpcStreamParserTest.kt Normal file
View File

@@ -0,0 +1,131 @@
package io.emeraldpay.dshackle.upstream.rpcclient.stream
import org.junit.jupiter.api.Assertions.assertNotNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.Arguments
import org.junit.jupiter.params.provider.MethodSource
import reactor.core.publisher.Flux
import reactor.test.StepVerifier
import java.time.Duration
class JsonRpcStreamParserTest {
private val streamParser = JsonRpcStreamParser()
@Test
fun `if first part couldn't be parsed then aggregate response`() {
val statusCode = 200
val bytes = "{\"strangeResponse\": 2}".toByteArray()
val stream: Flux<ByteArray> = Flux.just(bytes)
StepVerifier.create(streamParser.streamParse(statusCode, stream))
.expectNext(AggregateResponse(bytes, statusCode))
.expectComplete()
.verify(Duration.ofSeconds(1))
}
@ParameterizedTest
@MethodSource("data")
fun `if first part has result field then single response`(
response: ByteArray,
result: ByteArray,
) {
val statusCode = 200
val stream: Flux<ByteArray> = Flux.just(response)
StepVerifier.create(streamParser.streamParse(statusCode, stream))
.expectNext(SingleResponse(result, null))
.expectComplete()
.verify(Duration.ofSeconds(1))
}
@ParameterizedTest
@MethodSource("dataStream")
fun `if big result then stream response`(
response: List<ByteArray>,
chunks: List<Chunk>,
) {
val statusCode = 200
val stream: Flux<ByteArray> = Flux.fromIterable(response)
val result = streamParser.streamParse(statusCode, stream).block()
assertTrue(result is StreamResponse)
assertNotNull(result)
StepVerifier.create((result as StreamResponse).stream)
.expectNextSequence(chunks)
.expectComplete()
.verify(Duration.ofSeconds(5))
}
companion object {
@JvmStatic
fun data(): List<Arguments> = listOf(
Arguments.of("{\"id\": 2,\"result\": \"0x12\"}".toByteArray(), "\"0x12\"".toByteArray()),
Arguments.of("{\"id\": 2,\"result\": 11}".toByteArray(), "11".toByteArray()),
Arguments.of("{\"id\": 2,\"result\": false}".toByteArray(), "false".toByteArray()),
Arguments.of("{\"id\": 2,\"result\": null}".toByteArray(), "null".toByteArray()),
Arguments.of("{\"id\": 2,\"result\": {\"name\": \"value\"}".toByteArray(), "{\"name\": \"value\"}".toByteArray()),
Arguments.of("{\"id\": 2,\"result\": [{\"name\": \"value\"}]".toByteArray(), "[{\"name\": \"value\"}]".toByteArray()),
)
@JvmStatic
fun dataStream(): List<Arguments> = listOf(
Arguments.of(
listOf("{\"id\": 2,\"result\": \"0x12".toByteArray(), "222\"}".toByteArray()),
listOf(
Chunk("\"0x12".toByteArray(), false),
Chunk("222\"".toByteArray(), true),
),
),
Arguments.of(
listOf(
"{\"id\": 2,\"result\": \"0x12".toByteArray(),
"123\\\"".toByteArray(),
"222\"}".toByteArray(),
),
listOf(
Chunk("\"0x12".toByteArray(), false),
Chunk("123\\\"".toByteArray(), false),
Chunk("222\"".toByteArray(), true),
),
),
Arguments.of(
listOf(
"{\"id\": 2,\"result\": \"0x12".toByteArray(),
"1\\n23\\\"".toByteArray(),
"456\\".toByteArray(),
"\\222\\\\\\\\\"}".toByteArray(),
),
listOf(
Chunk("\"0x12".toByteArray(), false),
Chunk("1\\n23\\\"".toByteArray(), false),
Chunk("456\\".toByteArray(), false),
Chunk("\\222\\\\\\\\\"".toByteArray(), true),
),
),
Arguments.of(
listOf("{\"id\": 2,\"result\": {\"name\": ".toByteArray(), "\"bigName\"".toByteArray(), "}".toByteArray()),
listOf(
Chunk("{\"name\": ".toByteArray(), false),
Chunk("\"bigName\"".toByteArray(), false),
Chunk("}".toByteArray(), true),
),
),
Arguments.of(
listOf(
"{\"id\": 2,\"result\": [{\"name\": ".toByteArray(),
"\"bigName\"".toByteArray(),
"}],".toByteArray(),
"\"field\": \"value\"}".toByteArray(),
),
listOf(
Chunk("[{\"name\": ".toByteArray(), false),
Chunk("\"bigName\"".toByteArray(), false),
Chunk("}]".toByteArray(), true),
),
),
)
}
}