Collect error spans from providers (#198)

This commit is contained in:
KirillPamPam
2023-04-11 16:54:55 +04:00
committed by GitHub
parent fc488e1aab
commit 6b0e201c0e
12 changed files with 540 additions and 10 deletions

View File

@@ -22,6 +22,7 @@ class Defaults {
companion object {
const val maxMessageSize: Int = 32 * 1024 * 1024
const val maxMetadataSize = 16384
val timeout: Duration = Duration.ofSeconds(60)
val timeoutInternal: Duration = timeout.dividedBy(4)
val retryConnection: Duration = Duration.ofSeconds(10)

View File

@@ -29,6 +29,9 @@ import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Tag
import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.beans.factory.annotation.Value
import org.springframework.scheduling.concurrent.CustomizableThreadFactory
import org.springframework.stereotype.Service
import java.net.InetSocketAddress
@@ -42,8 +45,13 @@ open class GrpcServer(
private val mainConfig: MainConfig,
private val tlsSetup: TlsSetup,
private val accessHandler: AccessHandlerGrpc,
private val grpcServerBraveInterceptor: ServerInterceptor
private val grpcServerBraveInterceptor: ServerInterceptor,
@Autowired(required = false)
@Qualifier("serverSpansInterceptor")
private val serverSpansInterceptor: ServerInterceptor?
) {
@Value("\${spring.application.max-metadata-size}")
private var maxMetadataSize: Int = Defaults.maxMetadataSize
private val log = LoggerFactory.getLogger(GrpcServer::class.java)
@@ -68,6 +76,7 @@ open class GrpcServer(
val serverBuilder = NettyServerBuilder
.forAddress(InetSocketAddress(mainConfig.host, mainConfig.port))
.maxInboundMessageSize(Defaults.maxMessageSize)
.maxInboundMetadataSize(maxMetadataSize)
if (mainConfig.accessLogConfig.enabled) {
serverBuilder.intercept(accessHandler)
@@ -78,6 +87,10 @@ open class GrpcServer(
}
serverBuilder.intercept(grpcServerBraveInterceptor)
serverSpansInterceptor?.let {
serverBuilder.intercept(it)
log.info("Collect spans from provider is enabled")
}
tlsSetup.setupServer("Native gRPC", mainConfig.tls, true)?.let {
serverBuilder.sslContext(it)

View File

@@ -0,0 +1,62 @@
package io.emeraldpay.dshackle.config.spans
import brave.Tracer
import brave.handler.MutableSpan
import brave.handler.SpanHandler
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import io.grpc.CallOptions
import io.grpc.Channel
import io.grpc.ClientCall
import io.grpc.ClientInterceptor
import io.grpc.ForwardingClientCall.SimpleForwardingClientCall
import io.grpc.ForwardingClientCallListener
import io.grpc.Metadata
import io.grpc.Metadata.ASCII_STRING_MARSHALLER
import io.grpc.MethodDescriptor
import org.springframework.beans.factory.annotation.Qualifier
class ClientSpansInterceptor(
private val zipkinSpanHandler: SpanHandler,
private val tracer: Tracer,
@Qualifier("spanMapper")
private val spanMapper: ObjectMapper
) : ClientInterceptor {
override fun <ReqT : Any?, RespT : Any?> interceptCall(
method: MethodDescriptor<ReqT, RespT>,
callOptions: CallOptions,
next: Channel
): ClientCall<ReqT, RespT> {
return if (method.fullMethodName == "emerald.Blockchain/NativeCall") {
SpanClientCall(next.newCall(method, callOptions))
} else {
next.newCall(method, callOptions)
}
}
private inner class SpanClientCall<ReqT, RespT>(
delegate: ClientCall<ReqT, RespT>?
) : SimpleForwardingClientCall<ReqT, RespT>(delegate) {
override fun start(responseListener: Listener<RespT>, headers: Metadata) {
val spanResponseListener = SpanClientCallListener(responseListener)
super.start(spanResponseListener, headers)
}
}
private inner class SpanClientCallListener<RespT>(
delegate: ClientCall.Listener<RespT>,
) : ForwardingClientCallListener.SimpleForwardingClientCallListener<RespT>(delegate) {
override fun onHeaders(headers: Metadata) {
headers[Metadata.Key.of(SPAN_HEADER, ASCII_STRING_MARSHALLER)]
?.takeIf { it.isNotBlank() }
?.let {
val spansFromProvider = spanMapper.readValue<List<MutableSpan>>(it)
spansFromProvider.forEach { span ->
zipkinSpanHandler.end(tracer.currentSpan().context(), span, SpanHandler.Cause.FINISHED)
}
}
super.onHeaders(headers)
}
}
}

View File

@@ -0,0 +1,76 @@
package io.emeraldpay.dshackle.config.spans
import brave.handler.MutableSpan
import brave.handler.SpanHandler
import brave.propagation.TraceContext
import com.fasterxml.jackson.databind.ObjectMapper
import com.github.benmanes.caffeine.cache.Caffeine
import io.emeraldpay.dshackle.commons.SPAN_ERROR
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.cloud.sleuth.Span
import java.time.Duration
class ErrorSpanHandler(
@Qualifier("spanMapper")
private val spanMapper: ObjectMapper,
) : SpanHandler() {
private val spans = Caffeine
.newBuilder()
.expireAfterWrite(Duration.ofMinutes(5))
.build<String, MutableList<MutableSpan>>()
override fun end(context: TraceContext, span: MutableSpan, cause: Cause): Boolean {
if (span.traceId().length > 20 && span.parentId() != null) {
val spanList = spans.asMap().computeIfAbsent(span.parentId()) { mutableListOf() }
spanList.add(span)
}
return super.end(context, span, cause)
}
fun getErrorSpans(spanId: String, currentSpan: Span): String {
val spansInfo = SpansInfo()
enrichErrorSpans(spanId, spansInfo)
currentSpan.end()
currentSpan.context().parentId()?.let {
spans.getIfPresent(it)?.let { mutableSpans ->
if (mutableSpans.isNotEmpty()) {
spansInfo.spans.add(mutableSpans[0])
}
}
}
spansInfo.spans
.map { it.parentId() }
.forEach {
if (it != null) {
spans.invalidate(it)
}
}
return if (spansInfo.hasError) {
spanMapper.writeValueAsString(spansInfo.spans)
} else {
""
}
}
private fun enrichErrorSpans(spanId: String, spansInfo: SpansInfo) {
val currentSpans: List<MutableSpan>? = spans.getIfPresent(spanId)
currentSpans?.forEach {
spansInfo.spans.add(it)
if (it.tags().containsKey(SPAN_ERROR)) {
spansInfo.hasError = true
}
if (spanId != it.id()) {
enrichErrorSpans(it.id(), spansInfo)
}
}
}
private data class SpansInfo(
var hasError: Boolean = false,
val spans: MutableList<MutableSpan> = mutableListOf()
)
}

View File

@@ -0,0 +1,44 @@
package io.emeraldpay.dshackle.config.spans
import io.grpc.ForwardingServerCall.SimpleForwardingServerCall
import io.grpc.Metadata
import io.grpc.ServerCall
import io.grpc.ServerCallHandler
import io.grpc.ServerInterceptor
import org.springframework.cloud.sleuth.Tracer
class ServerSpansInterceptor(
private val tracer: Tracer,
private val errorSpanHandler: ErrorSpanHandler
) : ServerInterceptor {
override fun <ReqT : Any?, RespT : Any?> interceptCall(
call: ServerCall<ReqT, RespT>,
headers: Metadata,
next: ServerCallHandler<ReqT, RespT>
): ServerCall.Listener<ReqT> {
val serverCall = if (call.methodDescriptor.fullMethodName == "emerald.Blockchain/NativeCall") {
SpanServerCall(call)
} else {
call
}
return next.startCall(serverCall, headers)
}
private inner class SpanServerCall<ReqT, RespT>(
private val call: ServerCall<ReqT, RespT>
) : SimpleForwardingServerCall<ReqT, RespT>(call) {
override fun sendHeaders(headers: Metadata) {
tracer.currentSpan()?.let {
val parentId = it.context().parentId()
if (parentId != null) {
val spans = errorSpanHandler.getErrorSpans(it.context().spanId(), it)
if (spans.isNotBlank()) {
headers.put(Metadata.Key.of(SPAN_HEADER, Metadata.ASCII_STRING_MARSHALLER), spans)
}
}
call.sendHeaders(headers)
}
}
}
}

View File

@@ -0,0 +1,68 @@
package io.emeraldpay.dshackle.config.spans
import brave.Tracer
import brave.handler.SpanHandler
import com.fasterxml.jackson.annotation.JsonAutoDetect
import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.ObjectMapper
import io.grpc.ClientInterceptor
import io.grpc.ServerInterceptor
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
const val SPAN_HEADER = "spans"
@Configuration
@ConditionalOnProperty(value = ["spans.collect.enabled"], havingValue = "true")
open class SpanConfig {
@Bean
open fun spanMapper(): ObjectMapper =
ObjectMapper()
.apply {
setSerializationInclusion(JsonInclude.Include.NON_DEFAULT)
configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
setVisibility(
this.serializationConfig.defaultVisibilityChecker
.withFieldVisibility(JsonAutoDetect.Visibility.ANY)
.withGetterVisibility(JsonAutoDetect.Visibility.NONE)
.withSetterVisibility(JsonAutoDetect.Visibility.NONE)
.withCreatorVisibility(JsonAutoDetect.Visibility.NONE)
)
}
@Configuration
@ConditionalOnBean(SpanConfig::class)
@ConditionalOnProperty(value = ["spans.collect.main.enabled"], havingValue = "true")
open class MainSpanConfig {
@Bean
open fun clientSpansInterceptor(
zipkinSpanHandler: SpanHandler,
tracer: Tracer,
@Qualifier("spanMapper")
spanMapper: ObjectMapper
): ClientInterceptor = ClientSpansInterceptor(zipkinSpanHandler, tracer, spanMapper)
}
@Configuration
@ConditionalOnBean(SpanConfig::class)
@ConditionalOnProperty(value = ["spans.collect.provider.enabled"], havingValue = "true")
open class ProviderSpanConfig {
@Bean
open fun errorSpanHandler(
@Qualifier("spanMapper")
spanMapper: ObjectMapper
): ErrorSpanHandler = ErrorSpanHandler(spanMapper)
@Bean
open fun serverSpansInterceptor(
tracer: org.springframework.cloud.sleuth.Tracer,
errorSpanHandler: ErrorSpanHandler
): ServerInterceptor = ServerSpansInterceptor(tracer, errorSpanHandler)
}
}

View File

@@ -20,6 +20,7 @@ import brave.grpc.GrpcTracing
import com.google.common.annotations.VisibleForTesting
import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.FileResolver
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.ChainsConfig
@@ -50,8 +51,11 @@ import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.forkchoice.NoChoiceWithPriorityForkChoice
import io.emeraldpay.dshackle.upstream.grpc.GrpcUpstreams
import io.grpc.ClientInterceptor
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.beans.factory.annotation.Value
import org.springframework.boot.ApplicationArguments
import org.springframework.boot.ApplicationRunner
import org.springframework.context.ApplicationEventPublisher
@@ -76,8 +80,12 @@ open class ConfiguredUpstreams(
private val channelExecutor: Executor,
private val chainsConfig: ChainsConfig,
private val grpcTracing: GrpcTracing,
private val wsConnectionResubscribeScheduler: Scheduler
private val wsConnectionResubscribeScheduler: Scheduler,
@Autowired(required = false)
private val clientSpansInterceptor: ClientInterceptor?
) : ApplicationRunner {
@Value("\${spring.application.max-metadata-size}")
private var maxMetadataSize: Int = Defaults.maxMetadataSize
private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java)
private var seq = AtomicInteger(0)
@@ -340,7 +348,9 @@ open class ConfiguredUpstreams(
grpcUpstreamsScheduler,
channelExecutor,
chainsConfig,
grpcTracing
grpcTracing,
clientSpansInterceptor,
maxMetadataSize
).apply {
timeout = options.timeout
}

View File

@@ -35,6 +35,7 @@ import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
import io.grpc.ClientInterceptor
import io.grpc.Codec
import io.grpc.netty.NettyChannelBuilder
import io.micrometer.core.instrument.Counter
@@ -72,7 +73,9 @@ class GrpcUpstreams(
private val chainStatusScheduler: Scheduler,
private val grpcExecutor: Executor,
private val chainsConfig: ChainsConfig,
private val grpcTracing: GrpcTracing
private val grpcTracing: GrpcTracing,
private val clientSpansInterceptor: ClientInterceptor?,
private var maxMetadataSize: Int
) {
private val log = LoggerFactory.getLogger(GrpcUpstreams::class.java)
@@ -86,10 +89,15 @@ class GrpcUpstreams(
val chanelBuilder = NettyChannelBuilder.forAddress(host, port)
// some messages are very large. many of them in megabytes, some even in gigabytes (ex. ETH Traces)
.maxInboundMessageSize(Defaults.maxMessageSize)
.maxInboundMetadataSize(maxMetadataSize)
.enableRetry()
.intercept(grpcTracing.newClientInterceptor())
.executor(grpcExecutor)
.maxRetryAttempts(3)
clientSpansInterceptor?.let {
chanelBuilder.intercept(it)
log.info("Collect spans is enabled")
}
if (auth != null && StringUtils.isNotEmpty(auth.ca)) {
chanelBuilder
.useTransportSecurity()

View File

@@ -1,5 +1,6 @@
spring:
application:
max-metadata-size: ${MAX_METADATA_SIZE:16384}
name: ${DRPC_APP_NAME:dshackle}
zipkin:
enabled: ${ZIPKIN_ENABLE:false}
@@ -14,4 +15,12 @@ spring:
- ".+SubscribeTxStatus"
- ".+SubscribeStatus"
- ".+SubscribeNodeStatus"
- ".+Describe"
- ".+Describe"
spans:
collect:
enabled: ${ENABLE_COLLECT_SPANS:false}
provider:
enabled: ${ENABLE_PROVIDER_COLLECT_SPANS:true}
main:
enabled: ${ENABLE_MAIN_COLLECT_SPANS:false}

View File

@@ -30,7 +30,8 @@ class ConfiguredUpstreamsSpec extends Specification {
Executors.newFixedThreadPool(1),
ChainsConfig.default(),
GrpcTracing.create(Tracing.newBuilder().build()),
Schedulers.boundedElastic()
Schedulers.boundedElastic(),
null
)
def methods = new UpstreamsConfig.Methods(
[
@@ -60,7 +61,8 @@ class ConfiguredUpstreamsSpec extends Specification {
Executors.newFixedThreadPool(1),
ChainsConfig.default(),
GrpcTracing.create(Tracing.newBuilder().build()),
Schedulers.boundedElastic()
Schedulers.boundedElastic(),
null
)
def methods = new UpstreamsConfig.Methods(
[
@@ -89,7 +91,8 @@ class ConfiguredUpstreamsSpec extends Specification {
Executors.newFixedThreadPool(1),
ChainsConfig.default(),
GrpcTracing.create(Tracing.newBuilder().build()),
Schedulers.boundedElastic()
Schedulers.boundedElastic(),
null
)
expect:
configurer.getHash(node, src) == expected
@@ -113,7 +116,8 @@ class ConfiguredUpstreamsSpec extends Specification {
Executors.newFixedThreadPool(1),
ChainsConfig.default(),
GrpcTracing.create(Tracing.newBuilder().build()),
Schedulers.boundedElastic()
Schedulers.boundedElastic(),
null
)
when:
def h1 = configurer.getHash(null, "hohoho")
@@ -142,7 +146,8 @@ class ConfiguredUpstreamsSpec extends Specification {
Executors.newFixedThreadPool(1),
ChainsConfig.default(),
GrpcTracing.create(Tracing.newBuilder().build()),
Schedulers.boundedElastic()
Schedulers.boundedElastic(),
null
)
def methodsGroup = new UpstreamsConfig.MethodGroups(
["filter"] as Set,

View File

@@ -0,0 +1,138 @@
package io.emeraldpay.dshackle.config.spans
import brave.handler.SpanHandler
import org.junit.jupiter.api.Assertions.assertThrows
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertDoesNotThrow
import org.junit.jupiter.api.extension.ExtendWith
import org.mockito.Mockito.mock
import org.springframework.beans.factory.NoSuchBeanDefinitionException
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.context.TestConfiguration
import org.springframework.cloud.sleuth.Tracer
import org.springframework.context.ApplicationContext
import org.springframework.context.annotation.Bean
import org.springframework.test.context.ContextConfiguration
import org.springframework.test.context.TestPropertySource
import org.springframework.test.context.junit.jupiter.SpringExtension
class CollectSpanConfigTest {
@ContextConfiguration(
classes = [SpanConfig::class],
)
@ExtendWith(SpringExtension::class)
@TestPropertySource(
properties = [
"spans.collect.enabled=false"
]
)
class SpanConfigTest {
@Autowired
private lateinit var appCtx: ApplicationContext
@Test
fun testSpanConfig() {
assertThrows(NoSuchBeanDefinitionException::class.java) {
appCtx.getBean(SpanConfig::class.java)
}
assertThrows(NoSuchBeanDefinitionException::class.java) {
appCtx.getBean(ErrorSpanHandler::class.java)
}
assertThrows(NoSuchBeanDefinitionException::class.java) {
appCtx.getBean(ServerSpansInterceptor::class.java)
}
assertThrows(NoSuchBeanDefinitionException::class.java) {
appCtx.getBean(ClientSpansInterceptor::class.java)
}
assertThrows(NoSuchBeanDefinitionException::class.java) {
appCtx.getBean("spanMapper")
}
}
}
@ContextConfiguration(
classes = [SpanConfig::class, SpanProviderConfigTest.Config::class],
)
@ExtendWith(SpringExtension::class)
@TestPropertySource(
properties = [
"spans.collect.enabled=true",
"spans.collect.provider.enabled=true",
"spans.collect.main.enabled=false"
]
)
class SpanProviderConfigTest {
@Autowired
private lateinit var appCtx: ApplicationContext
@Test
fun testSpanProviderConfig() {
assertDoesNotThrow {
appCtx.getBean(SpanConfig::class.java)
}
assertDoesNotThrow {
appCtx.getBean(ErrorSpanHandler::class.java)
}
assertDoesNotThrow {
appCtx.getBean(ServerSpansInterceptor::class.java)
}
assertThrows(NoSuchBeanDefinitionException::class.java) {
appCtx.getBean(ClientSpansInterceptor::class.java)
}
assertDoesNotThrow {
appCtx.getBean("spanMapper")
}
}
@TestConfiguration
open class Config {
@Bean
open fun tracer(): Tracer = mock(Tracer::class.java)
}
}
@ContextConfiguration(
classes = [SpanConfig::class, SpanMainConfigTest.Config::class],
)
@ExtendWith(SpringExtension::class)
@TestPropertySource(
properties = [
"spans.collect.enabled=true",
"spans.collect.provider.enabled=false",
"spans.collect.main.enabled=true"
]
)
class SpanMainConfigTest {
@Autowired
private lateinit var appCtx: ApplicationContext
@Test
fun testSpanMainConfig() {
assertDoesNotThrow {
appCtx.getBean(SpanConfig::class.java)
}
assertThrows(NoSuchBeanDefinitionException::class.java) {
appCtx.getBean(ErrorSpanHandler::class.java)
}
assertThrows(NoSuchBeanDefinitionException::class.java) {
appCtx.getBean(ServerSpansInterceptor::class.java)
}
assertDoesNotThrow {
appCtx.getBean(ClientSpansInterceptor::class.java)
}
assertDoesNotThrow {
appCtx.getBean("spanMapper")
}
}
@TestConfiguration
open class Config {
@Bean
open fun tracer(): brave.Tracer = mock(brave.Tracer::class.java)
@Bean
open fun zipkinSpanHandler(): SpanHandler = mock(SpanHandler::class.java)
}
}
}

View File

@@ -0,0 +1,96 @@
package io.emeraldpay.dshackle.config.spans
import brave.handler.MutableSpan
import brave.handler.SpanHandler
import brave.propagation.TraceContext
import com.fasterxml.jackson.module.kotlin.readValue
import io.emeraldpay.dshackle.commons.SPAN_ERROR
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import org.mockito.Mockito
import org.mockito.Mockito.`when`
import org.springframework.cloud.sleuth.Span
import org.springframework.cloud.sleuth.brave.bridge.BraveTraceContext
class ErrorSpanHandlerTest {
private val mapper = SpanConfig().spanMapper()
private val ctx = TraceContext.newBuilder()
.traceId(1223324)
.spanId(234235)
.build()
@Test
fun `span with length of traceId less than 20 is not collected`() {
val spanId = "f7e83f2b69ec684d"
val currentSpan = Mockito.mock(Span::class.java)
val handler = spanHandler()
`when`(currentSpan.context()).thenReturn(BraveTraceContext(ctx))
handler.end(ctx, span("f7e83f2b69ec684d", spanId), SpanHandler.Cause.FINISHED)
val result = handler.getErrorSpans(spanId, currentSpan)
assertEquals("", result)
}
@Test
fun `span without parenId is not collected`() {
val spanId = "f7e83f2b69ec684d"
val currentSpan = Mockito.mock(Span::class.java)
val handler = spanHandler()
`when`(currentSpan.context()).thenReturn(BraveTraceContext(ctx))
handler.end(ctx, span("6666632728347823749827349723985", spanId), SpanHandler.Cause.FINISHED)
val result = handler.getErrorSpans(spanId, currentSpan)
assertEquals("", result)
}
@Test
fun `span with length of traceId greater than 20 and with parentId is collected`() {
val spanId = "f7e83f2b69ec684d"
val currentSpan = Mockito.mock(Span::class.java)
val handler = spanHandler()
val span = span("6666632728347823749827349723985", spanId)
.apply { parentId("f7e83f2b69ec682d") }
`when`(currentSpan.context()).thenReturn(BraveTraceContext(ctx))
handler.end(ctx, span, SpanHandler.Cause.FINISHED)
val result = handler.getErrorSpans("f7e83f2b69ec682d", currentSpan)
val collectedSpans = mapper.readValue<List<MutableSpan>>(result)
assertTrue(collectedSpans.size == 1)
assertEquals(span, collectedSpans[0])
}
@Test
fun `span without error tag is not collected`() {
val spanId = "f7e83f2b69ec684d"
val currentSpan = Mockito.mock(Span::class.java)
val handler = spanHandler()
val span = span("6666632728347823749827349723985", spanId)
.apply {
parentId("f7e83f2b69ec682d")
removeTag(SPAN_ERROR)
}
`when`(currentSpan.context()).thenReturn(BraveTraceContext(ctx))
handler.end(ctx, span, SpanHandler.Cause.FINISHED)
val result = handler.getErrorSpans("f7e83f2b69ec682d", currentSpan)
assertEquals("", result)
}
private fun span(traceId: String, spanId: String) = MutableSpan()
.apply {
traceId(traceId)
id(spanId)
tag(SPAN_ERROR, "true")
}
private fun spanHandler() = ErrorSpanHandler(mapper)
}