Collect error spans from providers (#198)

src/main/kotlin/io/emeraldpay/dshackle/Defaults.kt

@@ -22,6 +22,7 @@ class Defaults {
 
     companion object {
         const val maxMessageSize: Int = 32 * 1024 * 1024
+        const val maxMetadataSize = 16384
         val timeout: Duration = Duration.ofSeconds(60)
         val timeoutInternal: Duration = timeout.dividedBy(4)
         val retryConnection: Duration = Duration.ofSeconds(10)

src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt

@@ -29,6 +29,9 @@ import io.micrometer.core.instrument.Metrics
 import io.micrometer.core.instrument.Tag
 import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics
 import org.slf4j.LoggerFactory
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.beans.factory.annotation.Qualifier
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.scheduling.concurrent.CustomizableThreadFactory
 import org.springframework.stereotype.Service
 import java.net.InetSocketAddress
@@ -42,8 +45,13 @@ open class GrpcServer(
     private val mainConfig: MainConfig,
     private val tlsSetup: TlsSetup,
     private val accessHandler: AccessHandlerGrpc,
-    private val grpcServerBraveInterceptor: ServerInterceptor
+    private val grpcServerBraveInterceptor: ServerInterceptor,
+    @Autowired(required = false)
+    @Qualifier("serverSpansInterceptor")
+    private val serverSpansInterceptor: ServerInterceptor?
 ) {
+    @Value("\${spring.application.max-metadata-size}")
+    private var maxMetadataSize: Int = Defaults.maxMetadataSize
 
     private val log = LoggerFactory.getLogger(GrpcServer::class.java)
 
@@ -68,6 +76,7 @@ open class GrpcServer(
         val serverBuilder = NettyServerBuilder
             .forAddress(InetSocketAddress(mainConfig.host, mainConfig.port))
             .maxInboundMessageSize(Defaults.maxMessageSize)
+            .maxInboundMetadataSize(maxMetadataSize)
 
         if (mainConfig.accessLogConfig.enabled) {
             serverBuilder.intercept(accessHandler)
@@ -78,6 +87,10 @@ open class GrpcServer(
         }
 
         serverBuilder.intercept(grpcServerBraveInterceptor)
+        serverSpansInterceptor?.let {
+            serverBuilder.intercept(it)
+            log.info("Collect spans from provider is enabled")
+        }
 
         tlsSetup.setupServer("Native gRPC", mainConfig.tls, true)?.let {
             serverBuilder.sslContext(it)

src/main/kotlin/io/emeraldpay/dshackle/config/spans/ClientSpansInterceptor.kt

@@ -0,0 +1,62 @@
+package io.emeraldpay.dshackle.config.spans
+
+import brave.Tracer
+import brave.handler.MutableSpan
+import brave.handler.SpanHandler
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.kotlin.readValue
+import io.grpc.CallOptions
+import io.grpc.Channel
+import io.grpc.ClientCall
+import io.grpc.ClientInterceptor
+import io.grpc.ForwardingClientCall.SimpleForwardingClientCall
+import io.grpc.ForwardingClientCallListener
+import io.grpc.Metadata
+import io.grpc.Metadata.ASCII_STRING_MARSHALLER
+import io.grpc.MethodDescriptor
+import org.springframework.beans.factory.annotation.Qualifier
+
+class ClientSpansInterceptor(
+    private val zipkinSpanHandler: SpanHandler,
+    private val tracer: Tracer,
+    @Qualifier("spanMapper")
+    private val spanMapper: ObjectMapper
+) : ClientInterceptor {
+
+    override fun <ReqT : Any?, RespT : Any?> interceptCall(
+        method: MethodDescriptor<ReqT, RespT>,
+        callOptions: CallOptions,
+        next: Channel
+    ): ClientCall<ReqT, RespT> {
+        return if (method.fullMethodName == "emerald.Blockchain/NativeCall") {
+            SpanClientCall(next.newCall(method, callOptions))
+        } else {
+            next.newCall(method, callOptions)
+        }
+    }
+
+    private inner class SpanClientCall<ReqT, RespT>(
+        delegate: ClientCall<ReqT, RespT>?
+    ) : SimpleForwardingClientCall<ReqT, RespT>(delegate) {
+        override fun start(responseListener: Listener<RespT>, headers: Metadata) {
+            val spanResponseListener = SpanClientCallListener(responseListener)
+            super.start(spanResponseListener, headers)
+        }
+    }
+
+    private inner class SpanClientCallListener<RespT>(
+        delegate: ClientCall.Listener<RespT>,
+    ) : ForwardingClientCallListener.SimpleForwardingClientCallListener<RespT>(delegate) {
+        override fun onHeaders(headers: Metadata) {
+            headers[Metadata.Key.of(SPAN_HEADER, ASCII_STRING_MARSHALLER)]
+                ?.takeIf { it.isNotBlank() }
+                ?.let {
+                    val spansFromProvider = spanMapper.readValue<List<MutableSpan>>(it)
+                    spansFromProvider.forEach { span ->
+                        zipkinSpanHandler.end(tracer.currentSpan().context(), span, SpanHandler.Cause.FINISHED)
+                    }
+                }
+            super.onHeaders(headers)
+        }
+    }
+}

src/main/kotlin/io/emeraldpay/dshackle/config/spans/ErrorSpanHandler.kt

@@ -0,0 +1,76 @@
+package io.emeraldpay.dshackle.config.spans
+
+import brave.handler.MutableSpan
+import brave.handler.SpanHandler
+import brave.propagation.TraceContext
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.github.benmanes.caffeine.cache.Caffeine
+import io.emeraldpay.dshackle.commons.SPAN_ERROR
+import org.springframework.beans.factory.annotation.Qualifier
+import org.springframework.cloud.sleuth.Span
+import java.time.Duration
+
+class ErrorSpanHandler(
+    @Qualifier("spanMapper")
+    private val spanMapper: ObjectMapper,
+) : SpanHandler() {
+    private val spans = Caffeine
+        .newBuilder()
+        .expireAfterWrite(Duration.ofMinutes(5))
+        .build<String, MutableList<MutableSpan>>()
+
+    override fun end(context: TraceContext, span: MutableSpan, cause: Cause): Boolean {
+        if (span.traceId().length > 20 && span.parentId() != null) {
+            val spanList = spans.asMap().computeIfAbsent(span.parentId()) { mutableListOf() }
+            spanList.add(span)
+        }
+        return super.end(context, span, cause)
+    }
+
+    fun getErrorSpans(spanId: String, currentSpan: Span): String {
+        val spansInfo = SpansInfo()
+
+        enrichErrorSpans(spanId, spansInfo)
+        currentSpan.end()
+        currentSpan.context().parentId()?.let {
+            spans.getIfPresent(it)?.let { mutableSpans ->
+                if (mutableSpans.isNotEmpty()) {
+                    spansInfo.spans.add(mutableSpans[0])
+                }
+            }
+        }
+
+        spansInfo.spans
+            .map { it.parentId() }
+            .forEach {
+                if (it != null) {
+                    spans.invalidate(it)
+                }
+            }
+
+        return if (spansInfo.hasError) {
+            spanMapper.writeValueAsString(spansInfo.spans)
+        } else {
+            ""
+        }
+    }
+
+    private fun enrichErrorSpans(spanId: String, spansInfo: SpansInfo) {
+        val currentSpans: List<MutableSpan>? = spans.getIfPresent(spanId)
+
+        currentSpans?.forEach {
+            spansInfo.spans.add(it)
+            if (it.tags().containsKey(SPAN_ERROR)) {
+                spansInfo.hasError = true
+            }
+            if (spanId != it.id()) {
+                enrichErrorSpans(it.id(), spansInfo)
+            }
+        }
+    }
+
+    private data class SpansInfo(
+        var hasError: Boolean = false,
+        val spans: MutableList<MutableSpan> = mutableListOf()
+    )
+}

src/main/kotlin/io/emeraldpay/dshackle/config/spans/ServerSpansInterceptor.kt

@@ -0,0 +1,44 @@
+package io.emeraldpay.dshackle.config.spans
+
+import io.grpc.ForwardingServerCall.SimpleForwardingServerCall
+import io.grpc.Metadata
+import io.grpc.ServerCall
+import io.grpc.ServerCallHandler
+import io.grpc.ServerInterceptor
+import org.springframework.cloud.sleuth.Tracer
+
+class ServerSpansInterceptor(
+    private val tracer: Tracer,
+    private val errorSpanHandler: ErrorSpanHandler
+) : ServerInterceptor {
+    override fun <ReqT : Any?, RespT : Any?> interceptCall(
+        call: ServerCall<ReqT, RespT>,
+        headers: Metadata,
+        next: ServerCallHandler<ReqT, RespT>
+    ): ServerCall.Listener<ReqT> {
+        val serverCall = if (call.methodDescriptor.fullMethodName == "emerald.Blockchain/NativeCall") {
+            SpanServerCall(call)
+        } else {
+            call
+        }
+
+        return next.startCall(serverCall, headers)
+    }
+
+    private inner class SpanServerCall<ReqT, RespT>(
+        private val call: ServerCall<ReqT, RespT>
+    ) : SimpleForwardingServerCall<ReqT, RespT>(call) {
+        override fun sendHeaders(headers: Metadata) {
+            tracer.currentSpan()?.let {
+                val parentId = it.context().parentId()
+                if (parentId != null) {
+                    val spans = errorSpanHandler.getErrorSpans(it.context().spanId(), it)
+                    if (spans.isNotBlank()) {
+                        headers.put(Metadata.Key.of(SPAN_HEADER, Metadata.ASCII_STRING_MARSHALLER), spans)
+                    }
+                }
+                call.sendHeaders(headers)
+            }
+        }
+    }
+}

src/main/kotlin/io/emeraldpay/dshackle/config/spans/SpanConfig.kt

@@ -0,0 +1,68 @@
+package io.emeraldpay.dshackle.config.spans
+
+import brave.Tracer
+import brave.handler.SpanHandler
+import com.fasterxml.jackson.annotation.JsonAutoDetect
+import com.fasterxml.jackson.annotation.JsonInclude
+import com.fasterxml.jackson.databind.DeserializationFeature
+import com.fasterxml.jackson.databind.ObjectMapper
+import io.grpc.ClientInterceptor
+import io.grpc.ServerInterceptor
+import org.springframework.beans.factory.annotation.Qualifier
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBean
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+
+const val SPAN_HEADER = "spans"
+
+@Configuration
+@ConditionalOnProperty(value = ["spans.collect.enabled"], havingValue = "true")
+open class SpanConfig {
+
+    @Bean
+    open fun spanMapper(): ObjectMapper =
+        ObjectMapper()
+            .apply {
+                setSerializationInclusion(JsonInclude.Include.NON_DEFAULT)
+                configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+                setVisibility(
+                    this.serializationConfig.defaultVisibilityChecker
+                        .withFieldVisibility(JsonAutoDetect.Visibility.ANY)
+                        .withGetterVisibility(JsonAutoDetect.Visibility.NONE)
+                        .withSetterVisibility(JsonAutoDetect.Visibility.NONE)
+                        .withCreatorVisibility(JsonAutoDetect.Visibility.NONE)
+                )
+            }
+
+    @Configuration
+    @ConditionalOnBean(SpanConfig::class)
+    @ConditionalOnProperty(value = ["spans.collect.main.enabled"], havingValue = "true")
+    open class MainSpanConfig {
+        @Bean
+        open fun clientSpansInterceptor(
+            zipkinSpanHandler: SpanHandler,
+            tracer: Tracer,
+            @Qualifier("spanMapper")
+            spanMapper: ObjectMapper
+        ): ClientInterceptor = ClientSpansInterceptor(zipkinSpanHandler, tracer, spanMapper)
+    }
+
+    @Configuration
+    @ConditionalOnBean(SpanConfig::class)
+    @ConditionalOnProperty(value = ["spans.collect.provider.enabled"], havingValue = "true")
+    open class ProviderSpanConfig {
+
+        @Bean
+        open fun errorSpanHandler(
+            @Qualifier("spanMapper")
+            spanMapper: ObjectMapper
+        ): ErrorSpanHandler = ErrorSpanHandler(spanMapper)
+
+        @Bean
+        open fun serverSpansInterceptor(
+            tracer: org.springframework.cloud.sleuth.Tracer,
+            errorSpanHandler: ErrorSpanHandler
+        ): ServerInterceptor = ServerSpansInterceptor(tracer, errorSpanHandler)
+    }
+}

src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt

@@ -20,6 +20,7 @@ import brave.grpc.GrpcTracing
 import com.google.common.annotations.VisibleForTesting
 import io.emeraldpay.dshackle.BlockchainType
 import io.emeraldpay.dshackle.Chain
+import io.emeraldpay.dshackle.Defaults
 import io.emeraldpay.dshackle.FileResolver
 import io.emeraldpay.dshackle.Global
 import io.emeraldpay.dshackle.config.ChainsConfig
@@ -50,8 +51,11 @@ import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
 import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
 import io.emeraldpay.dshackle.upstream.forkchoice.NoChoiceWithPriorityForkChoice
 import io.emeraldpay.dshackle.upstream.grpc.GrpcUpstreams
+import io.grpc.ClientInterceptor
 import org.slf4j.LoggerFactory
+import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.beans.factory.annotation.Qualifier
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.boot.ApplicationArguments
 import org.springframework.boot.ApplicationRunner
 import org.springframework.context.ApplicationEventPublisher
@@ -76,8 +80,12 @@ open class ConfiguredUpstreams(
     private val channelExecutor: Executor,
     private val chainsConfig: ChainsConfig,
     private val grpcTracing: GrpcTracing,
-    private val wsConnectionResubscribeScheduler: Scheduler
+    private val wsConnectionResubscribeScheduler: Scheduler,
+    @Autowired(required = false)
+    private val clientSpansInterceptor: ClientInterceptor?
 ) : ApplicationRunner {
+    @Value("\${spring.application.max-metadata-size}")
+    private var maxMetadataSize: Int = Defaults.maxMetadataSize
 
     private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java)
     private var seq = AtomicInteger(0)
@@ -340,7 +348,9 @@ open class ConfiguredUpstreams(
             grpcUpstreamsScheduler,
             channelExecutor,
             chainsConfig,
-            grpcTracing
+            grpcTracing,
+            clientSpansInterceptor,
+            maxMetadataSize
         ).apply {
             timeout = options.timeout
         }

src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt

@@ -35,6 +35,7 @@ import io.emeraldpay.dshackle.upstream.Lifecycle
 import io.emeraldpay.dshackle.upstream.UpstreamAvailability
 import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient
 import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
+import io.grpc.ClientInterceptor
 import io.grpc.Codec
 import io.grpc.netty.NettyChannelBuilder
 import io.micrometer.core.instrument.Counter
@@ -72,7 +73,9 @@ class GrpcUpstreams(
     private val chainStatusScheduler: Scheduler,
     private val grpcExecutor: Executor,
     private val chainsConfig: ChainsConfig,
-    private val grpcTracing: GrpcTracing
+    private val grpcTracing: GrpcTracing,
+    private val clientSpansInterceptor: ClientInterceptor?,
+    private var maxMetadataSize: Int
 ) {
     private val log = LoggerFactory.getLogger(GrpcUpstreams::class.java)
 
@@ -86,10 +89,15 @@ class GrpcUpstreams(
         val chanelBuilder = NettyChannelBuilder.forAddress(host, port)
             // some messages are very large. many of them in megabytes, some even in gigabytes (ex. ETH Traces)
             .maxInboundMessageSize(Defaults.maxMessageSize)
+            .maxInboundMetadataSize(maxMetadataSize)
             .enableRetry()
             .intercept(grpcTracing.newClientInterceptor())
             .executor(grpcExecutor)
             .maxRetryAttempts(3)
+        clientSpansInterceptor?.let {
+            chanelBuilder.intercept(it)
+            log.info("Collect spans is enabled")
+        }
         if (auth != null && StringUtils.isNotEmpty(auth.ca)) {
             chanelBuilder
                 .useTransportSecurity()

src/main/resources/application.yml

@@ -1,5 +1,6 @@
 spring:
   application:
+    max-metadata-size: ${MAX_METADATA_SIZE:16384}
     name: ${DRPC_APP_NAME:dshackle}
   zipkin:
     enabled: ${ZIPKIN_ENABLE:false}
@@ -14,4 +15,12 @@ spring:
         - ".+SubscribeTxStatus"
         - ".+SubscribeStatus"
         - ".+SubscribeNodeStatus"
-        - ".+Describe"
+        - ".+Describe"
+
+spans:
+  collect:
+    enabled: ${ENABLE_COLLECT_SPANS:false}
+    provider:
+      enabled: ${ENABLE_PROVIDER_COLLECT_SPANS:true}
+    main:
+      enabled: ${ENABLE_MAIN_COLLECT_SPANS:false}

src/test/groovy/io/emeraldpay/dshackle/startup/ConfiguredUpstreamsSpec.groovy

@@ -30,7 +30,8 @@ class ConfiguredUpstreamsSpec extends Specification {
                 Executors.newFixedThreadPool(1),
                 ChainsConfig.default(),
                 GrpcTracing.create(Tracing.newBuilder().build()),
-                Schedulers.boundedElastic()
+                Schedulers.boundedElastic(),
+                null
         )
         def methods = new UpstreamsConfig.Methods(
                 [
@@ -60,7 +61,8 @@ class ConfiguredUpstreamsSpec extends Specification {
                 Executors.newFixedThreadPool(1),
                 ChainsConfig.default(),
                 GrpcTracing.create(Tracing.newBuilder().build()),
-                Schedulers.boundedElastic()
+                Schedulers.boundedElastic(),
+                null
         )
         def methods = new UpstreamsConfig.Methods(
                 [
@@ -89,7 +91,8 @@ class ConfiguredUpstreamsSpec extends Specification {
                 Executors.newFixedThreadPool(1),
                 ChainsConfig.default(),
                 GrpcTracing.create(Tracing.newBuilder().build()),
-                Schedulers.boundedElastic()
+                Schedulers.boundedElastic(),
+                null
         )
         expect:
         configurer.getHash(node, src) == expected
@@ -113,7 +116,8 @@ class ConfiguredUpstreamsSpec extends Specification {
                 Executors.newFixedThreadPool(1),
                 ChainsConfig.default(),
                 GrpcTracing.create(Tracing.newBuilder().build()),
-                Schedulers.boundedElastic()
+                Schedulers.boundedElastic(),
+                null
         )
         when:
         def h1 = configurer.getHash(null, "hohoho")
@@ -142,7 +146,8 @@ class ConfiguredUpstreamsSpec extends Specification {
                 Executors.newFixedThreadPool(1),
                 ChainsConfig.default(),
                 GrpcTracing.create(Tracing.newBuilder().build()),
-                Schedulers.boundedElastic()
+                Schedulers.boundedElastic(),
+                null
         )
         def methodsGroup = new UpstreamsConfig.MethodGroups(
                 ["filter"] as Set,

src/test/kotlin/io/emeraldpay/dshackle/config/spans/CollectSpanConfigTest.kt

@@ -0,0 +1,138 @@
+package io.emeraldpay.dshackle.config.spans
+
+import brave.handler.SpanHandler
+import org.junit.jupiter.api.Assertions.assertThrows
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertDoesNotThrow
+import org.junit.jupiter.api.extension.ExtendWith
+import org.mockito.Mockito.mock
+import org.springframework.beans.factory.NoSuchBeanDefinitionException
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.boot.test.context.TestConfiguration
+import org.springframework.cloud.sleuth.Tracer
+import org.springframework.context.ApplicationContext
+import org.springframework.context.annotation.Bean
+import org.springframework.test.context.ContextConfiguration
+import org.springframework.test.context.TestPropertySource
+import org.springframework.test.context.junit.jupiter.SpringExtension
+
+class CollectSpanConfigTest {
+
+    @ContextConfiguration(
+        classes = [SpanConfig::class],
+    )
+    @ExtendWith(SpringExtension::class)
+    @TestPropertySource(
+        properties = [
+            "spans.collect.enabled=false"
+        ]
+    )
+    class SpanConfigTest {
+        @Autowired
+        private lateinit var appCtx: ApplicationContext
+
+        @Test
+        fun testSpanConfig() {
+            assertThrows(NoSuchBeanDefinitionException::class.java) {
+                appCtx.getBean(SpanConfig::class.java)
+            }
+            assertThrows(NoSuchBeanDefinitionException::class.java) {
+                appCtx.getBean(ErrorSpanHandler::class.java)
+            }
+            assertThrows(NoSuchBeanDefinitionException::class.java) {
+                appCtx.getBean(ServerSpansInterceptor::class.java)
+            }
+            assertThrows(NoSuchBeanDefinitionException::class.java) {
+                appCtx.getBean(ClientSpansInterceptor::class.java)
+            }
+            assertThrows(NoSuchBeanDefinitionException::class.java) {
+                appCtx.getBean("spanMapper")
+            }
+        }
+    }
+
+    @ContextConfiguration(
+        classes = [SpanConfig::class, SpanProviderConfigTest.Config::class],
+    )
+    @ExtendWith(SpringExtension::class)
+    @TestPropertySource(
+        properties = [
+            "spans.collect.enabled=true",
+            "spans.collect.provider.enabled=true",
+            "spans.collect.main.enabled=false"
+        ]
+    )
+    class SpanProviderConfigTest {
+        @Autowired
+        private lateinit var appCtx: ApplicationContext
+
+        @Test
+        fun testSpanProviderConfig() {
+            assertDoesNotThrow {
+                appCtx.getBean(SpanConfig::class.java)
+            }
+            assertDoesNotThrow {
+                appCtx.getBean(ErrorSpanHandler::class.java)
+            }
+            assertDoesNotThrow {
+                appCtx.getBean(ServerSpansInterceptor::class.java)
+            }
+            assertThrows(NoSuchBeanDefinitionException::class.java) {
+                appCtx.getBean(ClientSpansInterceptor::class.java)
+            }
+            assertDoesNotThrow {
+                appCtx.getBean("spanMapper")
+            }
+        }
+
+        @TestConfiguration
+        open class Config {
+            @Bean
+            open fun tracer(): Tracer = mock(Tracer::class.java)
+        }
+    }
+
+    @ContextConfiguration(
+        classes = [SpanConfig::class, SpanMainConfigTest.Config::class],
+    )
+    @ExtendWith(SpringExtension::class)
+    @TestPropertySource(
+        properties = [
+            "spans.collect.enabled=true",
+            "spans.collect.provider.enabled=false",
+            "spans.collect.main.enabled=true"
+        ]
+    )
+    class SpanMainConfigTest {
+        @Autowired
+        private lateinit var appCtx: ApplicationContext
+
+        @Test
+        fun testSpanMainConfig() {
+            assertDoesNotThrow {
+                appCtx.getBean(SpanConfig::class.java)
+            }
+            assertThrows(NoSuchBeanDefinitionException::class.java) {
+                appCtx.getBean(ErrorSpanHandler::class.java)
+            }
+            assertThrows(NoSuchBeanDefinitionException::class.java) {
+                appCtx.getBean(ServerSpansInterceptor::class.java)
+            }
+            assertDoesNotThrow {
+                appCtx.getBean(ClientSpansInterceptor::class.java)
+            }
+            assertDoesNotThrow {
+                appCtx.getBean("spanMapper")
+            }
+        }
+
+        @TestConfiguration
+        open class Config {
+            @Bean
+            open fun tracer(): brave.Tracer = mock(brave.Tracer::class.java)
+
+            @Bean
+            open fun zipkinSpanHandler(): SpanHandler = mock(SpanHandler::class.java)
+        }
+    }
+}

src/test/kotlin/io/emeraldpay/dshackle/config/spans/ErrorSpanHandlerTest.kt

@@ -0,0 +1,96 @@
+package io.emeraldpay.dshackle.config.spans
+
+import brave.handler.MutableSpan
+import brave.handler.SpanHandler
+import brave.propagation.TraceContext
+import com.fasterxml.jackson.module.kotlin.readValue
+import io.emeraldpay.dshackle.commons.SPAN_ERROR
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.Test
+import org.mockito.Mockito
+import org.mockito.Mockito.`when`
+import org.springframework.cloud.sleuth.Span
+import org.springframework.cloud.sleuth.brave.bridge.BraveTraceContext
+
+class ErrorSpanHandlerTest {
+    private val mapper = SpanConfig().spanMapper()
+    private val ctx = TraceContext.newBuilder()
+        .traceId(1223324)
+        .spanId(234235)
+        .build()
+
+    @Test
+    fun `span with length of traceId less than 20 is not collected`() {
+        val spanId = "f7e83f2b69ec684d"
+        val currentSpan = Mockito.mock(Span::class.java)
+        val handler = spanHandler()
+
+        `when`(currentSpan.context()).thenReturn(BraveTraceContext(ctx))
+
+        handler.end(ctx, span("f7e83f2b69ec684d", spanId), SpanHandler.Cause.FINISHED)
+
+        val result = handler.getErrorSpans(spanId, currentSpan)
+        assertEquals("", result)
+    }
+
+    @Test
+    fun `span without parenId is not collected`() {
+        val spanId = "f7e83f2b69ec684d"
+        val currentSpan = Mockito.mock(Span::class.java)
+        val handler = spanHandler()
+
+        `when`(currentSpan.context()).thenReturn(BraveTraceContext(ctx))
+
+        handler.end(ctx, span("6666632728347823749827349723985", spanId), SpanHandler.Cause.FINISHED)
+
+        val result = handler.getErrorSpans(spanId, currentSpan)
+        assertEquals("", result)
+    }
+
+    @Test
+    fun `span with length of traceId greater than 20 and with parentId is collected`() {
+        val spanId = "f7e83f2b69ec684d"
+        val currentSpan = Mockito.mock(Span::class.java)
+        val handler = spanHandler()
+        val span = span("6666632728347823749827349723985", spanId)
+            .apply { parentId("f7e83f2b69ec682d") }
+
+        `when`(currentSpan.context()).thenReturn(BraveTraceContext(ctx))
+
+        handler.end(ctx, span, SpanHandler.Cause.FINISHED)
+
+        val result = handler.getErrorSpans("f7e83f2b69ec682d", currentSpan)
+        val collectedSpans = mapper.readValue<List<MutableSpan>>(result)
+        assertTrue(collectedSpans.size == 1)
+        assertEquals(span, collectedSpans[0])
+    }
+
+    @Test
+    fun `span without error tag is not collected`() {
+        val spanId = "f7e83f2b69ec684d"
+        val currentSpan = Mockito.mock(Span::class.java)
+        val handler = spanHandler()
+        val span = span("6666632728347823749827349723985", spanId)
+            .apply {
+                parentId("f7e83f2b69ec682d")
+                removeTag(SPAN_ERROR)
+            }
+
+        `when`(currentSpan.context()).thenReturn(BraveTraceContext(ctx))
+
+        handler.end(ctx, span, SpanHandler.Cause.FINISHED)
+
+        val result = handler.getErrorSpans("f7e83f2b69ec682d", currentSpan)
+        assertEquals("", result)
+    }
+
+    private fun span(traceId: String, spanId: String) = MutableSpan()
+        .apply {
+            traceId(traceId)
+            id(spanId)
+            tag(SPAN_ERROR, "true")
+        }
+
+    private fun spanHandler() = ErrorSpanHandler(mapper)
+}