solution: improve logging of upstream selection

src/main/kotlin/io/emeraldpay/dshackle/quorum/AlwaysQuorum.kt

@@ -56,4 +56,8 @@ open class AlwaysQuorum: CallQuorum {
     override fun getError(): JsonRpcError? {
         return rpcError
     }
+
+    override fun toString(): String {
+        return "Quorum: Accept Any"
+    }
 }

src/main/kotlin/io/emeraldpay/dshackle/quorum/BroadcastQuorum.kt

@@ -60,4 +60,7 @@ open class BroadcastQuorum(
         }
     }
 
+    override fun toString(): String {
+        return "Quorum: Broadcast to $quorum upstreams"
+    }
 }

src/main/kotlin/io/emeraldpay/dshackle/quorum/NonEmptyQuorum.kt

@@ -56,4 +56,7 @@ open class NonEmptyQuorum(
         tries++
     }
 
+    override fun toString(): String {
+        return "Quorum: Accept Non Error Result"
+    }
 }

src/main/kotlin/io/emeraldpay/dshackle/quorum/NonceQuorum.kt

@@ -71,4 +71,7 @@ open class NonceQuorum(
         errors++
     }
 
+    override fun toString(): String {
+        return "Quorum: Confirm with $tries upstreams"
+    }
 }

src/main/kotlin/io/emeraldpay/dshackle/quorum/NotLaggingQuorum.kt

@@ -64,4 +64,8 @@ class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum {
     override fun getError(): JsonRpcError? {
         return rpcError
     }
+
+    override fun toString(): String {
+        return "Quorum: late <= $maxLag blocks"
+    }
 }

src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRpcReader.kt

@@ -62,7 +62,7 @@ class QuorumRpcReader(
                                 ?: RpcException(-32000, "Unknown Upstream error")
                 )
             } else {
-                log.warn("Empty result for ${key.method} as ${q}")
+                log.warn("Did get any result from upstream. Method [${key.method}] using [$q]")
                 Mono.empty<Result>()
             }
         }
@@ -119,7 +119,7 @@ class QuorumRpcReader(
                 }
                 .doOnNext {
                     if (!it.isResolved()) {
-                        log.debug("No quorum for ${key.method} as ${quorum.javaClass.name}: ${it.getError()?.message ?: ""}")
+                        log.debug("No quorum for ${key.method} using [${quorum}]. Error: ${it.getError()?.message ?: ""}")
                     }
                 }
                 // return nothing if not resolved

src/main/kotlin/io/emeraldpay/dshackle/upstream/FilteredApis.kt

@@ -28,7 +28,7 @@ import kotlin.math.roundToLong
 import kotlin.random.Random
 
 class FilteredApis(
-        allUpstreams: List<Upstream>,
+        private val allUpstreams: List<Upstream>,
         private val matcher: Selector.Matcher,
         pos: Int,
         /**
@@ -128,4 +128,8 @@ class FilteredApis(
             control.onNext(true)
         }
     }
+
+    override fun toString(): String {
+        return "Filter API: ${allUpstreams.size} upstreams with $matcher"
+    }
 }

src/main/kotlin/io/emeraldpay/dshackle/upstream/Selector.kt

@@ -104,6 +104,8 @@ class Selector {
 
     interface Matcher {
         fun matches(up: Upstream): Boolean
+
+        fun describeInternal(): String
     }
 
     class MultiMatcher(
@@ -113,9 +115,17 @@ class Selector {
             return matchers.all { it.matches(up) }
         }
 
-        fun <T: Matcher> getMatcher(type: Class<T>): T? {
+        fun <T : Matcher> getMatcher(type: Class<T>): T? {
             return matchers.find { type.isAssignableFrom(it.javaClass) } as T?
         }
+
+        override fun describeInternal(): String {
+            return if (matchers.size == 1) {
+                matchers.first().describeInternal()
+            } else {
+                "ALLOF[" + matchers.joinToString(",") { it.describeInternal() } + "]"
+            }
+        }
     }
 
     class MethodMatcher(
@@ -124,6 +134,10 @@ class Selector {
         override fun matches(up: Upstream): Boolean {
             return up.getMethods().isAllowed(method)
         }
+
+        override fun describeInternal(): String {
+            return "allow method $method"
+        }
     }
 
     abstract class LabelSelectorMatcher: Matcher {
@@ -139,6 +153,14 @@ class Selector {
         override fun matches(up: Upstream): Boolean {
             return true
         }
+
+        override fun describeInternal(): String {
+            return "empty"
+        }
+
+        override fun toString(): String {
+            return "Matcher: ${describeInternal()}"
+        }
     }
 
     class AnyLabelMatcher: LabelSelectorMatcher() {
@@ -154,6 +176,14 @@ class Selector {
         override fun matches(up: Upstream): Boolean {
             return true
         }
+
+        override fun describeInternal(): String {
+            return "any label"
+        }
+
+        override fun toString(): String {
+            return "Matcher: ${describeInternal()}"
+        }
     }
 
     class LocalAndMatcher(vararg val matchers: Matcher) : Matcher {
@@ -162,6 +192,13 @@ class Selector {
             return matchers.all { it.matches(up) }
         }
 
+        override fun describeInternal(): String {
+            return "local upstream"
+        }
+
+        override fun toString(): String {
+            return "Matcher: ${describeInternal()}"
+        }
     }
 
     class LabelMatcher(val name: String, val values: Collection<String>) : LabelSelectorMatcher() {
@@ -178,6 +215,14 @@ class Selector {
                             .addAllValue(values)
             ).build()
         }
+
+        override fun describeInternal(): String {
+            return "label '$name'=" + values.joinToString(",")
+        }
+
+        override fun toString(): String {
+            return "Matcher: ${describeInternal()}"
+        }
     }
 
     class OrMatcher(val matchers: Collection<LabelSelectorMatcher>): LabelSelectorMatcher() {
@@ -192,6 +237,14 @@ class Selector {
                             .build()
             ).build()
         }
+
+        override fun describeInternal(): String {
+            return "ALLOF[" + matchers.joinToString(",") { it.describeInternal() } + "]"
+        }
+
+        override fun toString(): String {
+            return "Matcher: ${describeInternal()}"
+        }
     }
 
     class AndMatcher(val matchers: Collection<LabelSelectorMatcher>): LabelSelectorMatcher() {
@@ -206,6 +259,14 @@ class Selector {
                             .build()
             ).build()
         }
+
+        override fun describeInternal(): String {
+            return "ALLOF[" + matchers.joinToString(",") { it.describeInternal() } + "]"
+        }
+
+        override fun toString(): String {
+            return "Matcher: ${describeInternal()}"
+        }
     }
 
     class NotMatcher(val matcher: LabelSelectorMatcher): LabelSelectorMatcher() {
@@ -220,6 +281,14 @@ class Selector {
                             .build()
             ).build()
         }
+
+        override fun describeInternal(): String {
+            return "NOT[${matcher.describeInternal()}]"
+        }
+
+        override fun toString(): String {
+            return "Matcher: ${describeInternal()}"
+        }
     }
 
     class ExistsMatcher(val name: String): LabelSelectorMatcher() {
@@ -234,18 +303,42 @@ class Selector {
                             .build()
             ).build()
         }
+
+        override fun describeInternal(): String {
+            return "label '${name}' exists"
+        }
+
+        override fun toString(): String {
+            return "Matcher: ${describeInternal()}"
+        }
     }
 
     class CapabilityMatcher(val capability: Capability) : Matcher {
         override fun matches(up: Upstream): Boolean {
             return up.getCapabilities().contains(capability)
         }
+
+        override fun describeInternal(): String {
+            return "provides $capability API"
+        }
+
+        override fun toString(): String {
+            return "Matcher: ${describeInternal()}"
+        }
     }
 
     class GrpcMatcher() : Matcher {
         override fun matches(up: Upstream): Boolean {
             return up.isGrpc()
         }
+
+        override fun describeInternal(): String {
+            return "is gRPC"
+        }
+
+        override fun toString(): String {
+            return "Matcher: ${describeInternal()}"
+        }
     }
 
     class HeightMatcher(val height: Long): Matcher {
@@ -265,5 +358,13 @@ class Selector {
         override fun hashCode(): Int {
             return height.hashCode()
         }
+
+        override fun describeInternal(): String {
+            return "height $height"
+        }
+
+        override fun toString(): String {
+            return "Matcher: ${describeInternal()}"
+        }
     }
 }