Merge pull request #34 from p2p-org/evm_networks_support

support evm networks: Arbitrum, Optimism, Binance Smart Chain

dshackle-cli/package.json

@@ -18,8 +18,7 @@
     "@grpc/proto-loader": "^0.7.2",
     "arg": "^5.0.2",
     "cli-color": "^2.0.3",
-    "esm": "^3.2.25",
+    "esm": "^3.2.25"
-    "inquirer": "^9.1.1"
   },
   "files": [
     "bin/",

dshackle-cli/src/cli.js

@@ -1,47 +1,109 @@
-import {describe} from "./grpc-clent";
+import {describe, connect, nativeCall} from "./grpc-clent";
 import arg from 'arg';
 import clc from "cli-color";
 import util from "util";
 
+const chains = {
+    CHAIN_BSC: 1006,
+    CHAIN_OPTIMISM: 1005,
+    CHAIN_ARBITRUM: 1004,
+    CHAIN_MATIC: 1002,
+    CHAIN_ETHEREUM: 100
+}
+
 export function cli(args) {
     let opts = parseArgumentsIntoOptions(args);
     if (!opts.url) {
         console.log("Err: URL not specified!!!")
         return
     }
-    describe(opts.url, opts.ca, opts.cert, opts.key, (error, response) => {
-        if (error) {
-            console.error(clc.red('Connection to ' + opts.url + ' failed! [' + error.message + ']'));
-        } else {
-            console.error(clc.green('Connected to ', opts.url));
-            if (opts.print) {
-                console.log(util.inspect(response, false, null, true /* enable colors */));
-            } else {
-                response.chains.forEach(function (item) {
-                    var state = item.status.availability;
 
-                    switch (state) {
+    const client = connect(opts.url, opts.ca, opts.cert, opts.key)
-                        case 'AVAIL_OK':
+    describe(client, (error, response) => {
-                            state = clc.green(state);
+        if (error) {
-                            break
+            console.error(clc.red('Connection to ' + opts.url + ' failed:  ' + error.message));
-                        case 'AVAIL_UNKNOWN':
+            return
-                        case 'AVAIL_UNAVAILABLE':
+        }
-                            state = clc.red(state);
+        console.log(clc.green('Connected to ', opts.url));
-                            break
+        if (opts.print) {
-                        default:
+            console.log(util.inspect(response, false, null, true /* enable colors */));
-                            state = clc.yellow(state);
+        }
-                    }
+        processDescribe(client, response, opts.testRun)
-                    console.log(item.status.chain + ' -> ' + clc.bold(state))
+    })
-                })
+}
+
+function processDescribe(client, response, testRun) {
+    let promises = []
+    let statuses = new Map()
+
+    response.chains.forEach((item) => {
+        const state = item.status.availability
+        const chain = item.status.chain
+        let status = {
+            state: 'AVAIL_UNKNOWN',
+            grpc: clc.yellow('UNKNOWN'),
+            failed: false
+        }
+
+        switch (state) {
+            case 'AVAIL_OK':
+                status.state = clc.green(state);
+                break
+            case 'AVAIL_UNKNOWN':
+            case 'AVAIL_UNAVAILABLE':
+                status.state = clc.red(state);
+                break
+            default:
+                status.state = clc.yellow(state);
+        }
+
+        if (state === 'AVAIL_OK') {
+            promises.push(nativeCall(client, chains[chain], chain))
+        } else {
+            status.failed = true
+        }
+        statuses.set(chain, status)
+    })
+
+    Promise.all(promises).then(responses => {
+        responses.forEach((resp) => {
+            let status = statuses.get(resp.chain)
+            if (resp.error) {
+                status.failed = true
+                status.grpc = clc.red(resp.error.message)
+            } else {
+                if (resp.payload.succeed) {
+                    status.grpc = clc.green('OK')
+                } else {
+                    status.grpc = clc.red('FAILED')
+                    status.failed = true
+                }
             }
+        })
+
+        let hasError = false
+        statuses.forEach((status, chain) => {
+            printState(chain, status)
+            if (status.failed) {
+                hasError = true
+            }
+        })
+
+        if (hasError && testRun) {
+            process.exit(1)
         }
     })
 }
 
+function printState(chain, status) {
+    console.log(chain + ' -> ' + "state: " + clc.bold(status.state) + " gRPC: " + clc.bold(status.grpc))
+}
+
 function parseArgumentsIntoOptions(rawArgs) {
     const args = arg(
         {
             '--print': Boolean,
+            '--test-run': Boolean,
             '--ca': String,
             '--cert': String,
             '--key': String,
@@ -53,6 +115,7 @@ function parseArgumentsIntoOptions(rawArgs) {
     );
     return {
         print: args['--print'] || false,
+        testRun: args['--test-run'] || false,
         url: args._[0],
         ca: args['--ca'],
         cert: args['--cert'],

dshackle-cli/src/grpc-clent.js

@@ -16,7 +16,10 @@ const options = {
 const packageDefinition = protoLoader.loadSync(PROTO_PATH, options);
 const emerald = grpc.loadPackageDefinition(packageDefinition).emerald
 
-export function describe(url, ca, cert, key, handler) {
+
+var id = 100
+
+export function connect(url, ca, cert, key) {
     let credentials = grpc.credentials.createInsecure()
     if (ca || cert || key) {
         console.log("Using TLS")
@@ -26,11 +29,41 @@ export function describe(url, ca, cert, key, handler) {
             cert ? fs.readFileSync(cert) : null
         );
     }
-    const client = new emerald.Blockchain(
+    console.log('Connecting to: ' + url + '...')
+    return new emerald.Blockchain(
         url,
         credentials
     );
+}
 
-    console.log('Connecting to: ' + url + '...')
+export function describe(client, handler) {
     client.Describe({}, handler);
 }
+
+export function nativeCall(client, chainCode, chain) {
+    return new Promise((resolve, reject) => {
+        const call = client.NativeCall({
+            chain: chainCode,
+            items: [{
+                id: id++,
+                method: "eth_getBalance",
+                payload: "WyIweDhEOTc2ODlDOTgxODg5MkI3MDBlMjdGMzE2Y2MzRTQxZTE3ZkJlYjkiLCAibGF0ZXN0Il0="
+            }],
+            quorum: 1,
+            min_availability: 0
+        })
+        call.on('data', (item) => {
+            resolve(toResult(chain, item, null))
+        })
+        call.on('end', () => resolve(toResult(chain, null, null)))
+        call.on('error', (e) => reject(toResult(chain, null, e)))
+    })
+}
+
+function toResult(chain, obj, err) {
+    return {
+        chain: chain,
+        payload: obj,
+        error: err
+    }
+}

gradle/libs.versions.toml

@@ -2,16 +2,16 @@
 detekt = "1.18.1"
 etherjar = "0.11.1"
 groovy = "2.5.14"
-protoc = "3.9.0"
+protoc = "3.21.7"
 slf4j = "1.7.32"
 jackson = "2.11.0"
-grpc = "1.38.0"
+grpc = "1.49.2"
 reactive-grpc = "1.2.0"
 spring-boot = "2.5.6"
 spring-security = "5.5.3"
 reactor = "3.4.10"
 netty = "4.1.70.Final"
-netty-tcnative = "2.0.45.Final"
+netty-tcnative = "2.0.47.Final"
 kotlin = "1.5.31"
 httpcomponents = "4.5.8"
 

src/main/kotlin/io/emeraldpay/dshackle/Global.kt

@@ -44,8 +44,14 @@ class Global {
             "ethereum" to Chain.ETHEREUM,
             "ethereum-classic" to Chain.ETHEREUM_CLASSIC,
             "eth" to Chain.ETHEREUM,
-            "polygon" to Chain.MATIC,
+            "polygon" to Chain.POLYGON,
-            "matic" to Chain.MATIC,
+            "matic" to Chain.POLYGON,
+            "arbitrum" to Chain.ARBITRUM,
+            "arb" to Chain.ARBITRUM,
+            "optimism" to Chain.OPTIMISM,
+            "binance" to Chain.BSC,
+            "bsc" to Chain.BSC,
+            "bnb-smart-chain" to Chain.BSC,
             "etc" to Chain.ETHEREUM_CLASSIC,
             "morden" to Chain.TESTNET_MORDEN,
             "kovan" to Chain.TESTNET_KOVAN,

src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfig.kt

@@ -41,7 +41,7 @@ class TokensConfig(
                 type == null -> type
                 address.isNullOrBlank() -> "address"
                 blockchain != null &&
-                    (BlockchainType.from(blockchain!!) == BlockchainType.ETHEREUM_POS || BlockchainType.from(blockchain!!) == BlockchainType.ETHEREUM) &&
+                    (BlockchainType.from(blockchain!!) == BlockchainType.EVM_POS || BlockchainType.from(blockchain!!) == BlockchainType.EVM_POW) &&
                     !Address.isValidAddress(address) -> "address"
                 else -> null
             }

src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt

@@ -31,6 +31,7 @@ import io.emeraldpay.dshackle.upstream.Multistream
 import io.emeraldpay.dshackle.upstream.MultistreamHolder
 import io.emeraldpay.dshackle.upstream.Selector
 import io.emeraldpay.dshackle.upstream.calls.EthereumCallSelector
+import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
 import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream
 import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
 import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
@@ -66,14 +67,16 @@ open class NativeCall(
     private val ethereumCallSelectors = EnumMap<Chain, EthereumCallSelector>(Chain::class.java)
 
     init {
+        val casting = mapOf(
+            BlockchainType.EVM_POS to EthereumPosMultiStream::class.java,
+            BlockchainType.EVM_POW to EthereumMultistream::class.java,
+        )
+
         multistreamHolder.observeChains().subscribe { chain ->
-            if ((BlockchainType.from(chain) == BlockchainType.ETHEREUM_POS || BlockchainType.from(chain) == BlockchainType.ETHEREUM) && !ethereumCallSelectors.containsKey(
+            casting[BlockchainType.from(chain)]?.let { cast ->
-                    chain
-                )
-            ) {
                 multistreamHolder.getUpstream(chain)?.let { up ->
-                    val reader = up.cast(EthereumPosMultiStream::class.java).getReader()
+                    val reader = up.cast(cast).getReader()
-                    ethereumCallSelectors[chain] = EthereumCallSelector(reader.heightByHash())
+                    ethereumCallSelectors.putIfAbsent(chain, EthereumCallSelector(reader.heightByHash()))
                 }
             }
         }
@@ -213,7 +216,7 @@ open class NativeCall(
         }
         // for ethereum the actual block needed for the call may be specified in the call parameters
         val callSpecificMatcher: Mono<Selector.Matcher> =
-            if (BlockchainType.from(upstream.chain) == BlockchainType.ETHEREUM_POS || BlockchainType.from(upstream.chain) == BlockchainType.ETHEREUM) {
+            if (BlockchainType.from(upstream.chain) == BlockchainType.EVM_POS || BlockchainType.from(upstream.chain) == BlockchainType.EVM_POW) {
                 ethereumCallSelectors[chain]?.getMatcher(method, params, upstream.getHead())
             } else {
                 null

src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt

@@ -57,7 +57,7 @@ open class NativeSubscribe(
 
     fun start(request: BlockchainOuterClass.NativeSubscribeRequest): Publisher<ResponseHolder> {
         val chain = Chain.byId(request.chainValue)
-        if (BlockchainType.from(chain) != BlockchainType.ETHEREUM_POS && BlockchainType.from(chain) != BlockchainType.ETHEREUM) {
+        if (BlockchainType.from(chain) != BlockchainType.EVM_POS && BlockchainType.from(chain) != BlockchainType.EVM_POW) {
             return Mono.error(UnsupportedOperationException("Native subscribe is not supported for ${chain.chainCode}"))
         }
 

src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackERC20Address.kt

@@ -70,7 +70,7 @@ class TrackERC20Address(
 
     override fun isSupported(chain: Chain, asset: String): Boolean {
         return tokens.containsKey(TokenId(chain, asset.lowercase(Locale.getDefault()))) &&
-            (BlockchainType.from(chain) == BlockchainType.ETHEREUM_POS || BlockchainType.from(chain) == BlockchainType.ETHEREUM) && multistreamHolder.isAvailable(chain)
+            (BlockchainType.from(chain) == BlockchainType.EVM_POS || BlockchainType.from(chain) == BlockchainType.EVM_POW) && multistreamHolder.isAvailable(chain)
     }
 
     override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {

src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumAddress.kt

@@ -43,7 +43,7 @@ class TrackEthereumAddress(
 
     override fun isSupported(chain: Chain, asset: String): Boolean {
         return asset == "ether" &&
-            (BlockchainType.from(chain) == BlockchainType.ETHEREUM_POS || BlockchainType.from(chain) == BlockchainType.ETHEREUM) && multistreamHolder.isAvailable(chain)
+            (BlockchainType.from(chain) == BlockchainType.EVM_POS || BlockchainType.from(chain) == BlockchainType.EVM_POW) && multistreamHolder.isAvailable(chain)
     }
 
     override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux<BlockchainOuterClass.AddressBalance> {

src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumTx.kt

@@ -62,7 +62,7 @@ class TrackEthereumTx(
     private val log = LoggerFactory.getLogger(TrackEthereumTx::class.java)
 
     override fun isSupported(chain: Chain): Boolean {
-        return (BlockchainType.from(chain) == BlockchainType.ETHEREUM_POS || BlockchainType.from(chain) == BlockchainType.ETHEREUM) && multistreamHolder.isAvailable(chain)
+        return (BlockchainType.from(chain) == BlockchainType.EVM_POS || BlockchainType.from(chain) == BlockchainType.EVM_POW) && multistreamHolder.isAvailable(chain)
     }
 
     override fun subscribe(request: BlockchainOuterClass.TxStatusRequest): Flux<BlockchainOuterClass.TxStatus> {

src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt

@@ -92,7 +92,7 @@ open class ConfiguredUpstreams(
                 val options = (up.options ?: UpstreamsConfig.Options())
                     .merge(defaultOptions[chain] ?: UpstreamsConfig.Options.getDefaults())
                 val upstream = when (BlockchainType.from(chain)) {
-                    BlockchainType.ETHEREUM -> {
+                    BlockchainType.EVM_POW -> {
                         buildEthereumUpstream(up.nodeId, up.cast(UpstreamsConfig.EthereumConnection::class.java), chain, options)
                     }
 
@@ -100,7 +100,7 @@ open class ConfiguredUpstreams(
                         buildBitcoinUpstream(up.cast(UpstreamsConfig.BitcoinConnection::class.java), chain, options)
                     }
 
-                    BlockchainType.ETHEREUM_POS -> {
+                    BlockchainType.EVM_POS -> {
                         buildEthereumPosUpstream(
                             up.nodeId,
                             up.cast(UpstreamsConfig.EthereumPosConnection::class.java),

src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolder.kt

@@ -62,7 +62,7 @@ open class CurrentMultistreamHolder(
             val chain = change.chain
             try {
                 when (BlockchainType.from(chain)) {
-                    BlockchainType.ETHEREUM -> {
+                    BlockchainType.EVM_POW -> {
                         val up = change.upstream.cast(EthereumUpstream::class.java)
                         val current = chainMapping[chain]
                         val factory = Callable<Multistream> {
@@ -70,7 +70,7 @@ open class CurrentMultistreamHolder(
                         }
                         processUpdate(change, up, current, factory)
                     }
-                    BlockchainType.ETHEREUM_POS -> {
+                    BlockchainType.EVM_POS -> {
                         val up = change.upstream.cast(EthereumPosUpstream::class.java)
                         val current = chainMapping[chain]
                         val factory = Callable<Multistream> {
@@ -145,9 +145,9 @@ open class CurrentMultistreamHolder(
 
     fun setupDefaultMethods(chain: Chain): CallMethods {
         val created = when (BlockchainType.from(chain)) {
-            BlockchainType.ETHEREUM -> DefaultEthereumMethods(chain)
+            BlockchainType.EVM_POW -> DefaultEthereumMethods(chain)
             BlockchainType.BITCOIN -> DefaultBitcoinMethods()
-            BlockchainType.ETHEREUM_POS -> DefaultEthereumMethods(chain)
+            BlockchainType.EVM_POS -> DefaultEthereumMethods(chain)
             else -> throw IllegalStateException("Unsupported chain: $chain")
         }
         callTargets[chain] = created

src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultEthereumMethods.kt

@@ -133,7 +133,7 @@ class DefaultEthereumMethods(
                     Chain.ETHEREUM_CLASSIC == chain -> {
                         "\"1\""
                     }
-                    Chain.MATIC == chain -> {
+                    Chain.POLYGON == chain -> {
                         "\"137\""
                     }
                     Chain.TESTNET_MORDEN == chain -> {
@@ -159,7 +159,7 @@ class DefaultEthereumMethods(
                     Chain.ETHEREUM == chain -> {
                         "\"0x1\""
                     }
-                    Chain.MATIC == chain -> {
+                    Chain.POLYGON == chain -> {
                         "\"0x89\""
                     }
                     Chain.TESTNET_ROPSTEN == chain -> {

src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumBlockValidator.kt

@@ -111,7 +111,7 @@ class EthereumBlockValidator : BlockValidator {
                     )
                 }
 
-                val timestampValid = it.timestamp > cur.timestamp
+                val timestampValid = it.timestamp >= cur.timestamp
                 if (!timestampValid) {
                     log.warn(
                         "Block timestamp {} not valid for {}. Must be greater than {}",

src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsConnection.kt

@@ -240,6 +240,7 @@ open class WsConnection(
             .aggregateFrames(msgSizeLimit)
             .receiveFrames()
             .map { ByteBufInputStream(it.content()).readAllBytes() }
+            .filter { it.isNotEmpty() }
             .flatMap {
                 try {
                     val msg = parser.parse(it)

src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt

@@ -191,11 +191,11 @@ class GrpcUpstreams(
         )
 
         val blockchainType = BlockchainType.from(chain)
-        if (blockchainType == BlockchainType.ETHEREUM) {
+        if (blockchainType == BlockchainType.EVM_POW) {
             return getOrCreateEthereum(chain, metrics)
         } else if (blockchainType == BlockchainType.BITCOIN) {
             return getOrCreateBitcoin(chain, metrics)
-        } else if (blockchainType == BlockchainType.ETHEREUM_POS) {
+        } else if (blockchainType == BlockchainType.EVM_POS) {
             return getOrCreateEthereumPos(chain, metrics)
         } else {
             throw IllegalArgumentException("Unsupported blockchain: $chain")

src/test/groovy/io/emeraldpay/dshackle/test/MultistreamHolderMock.groovy

@@ -27,11 +27,9 @@ import io.emeraldpay.dshackle.upstream.calls.CallMethods
 import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods
 import io.emeraldpay.dshackle.upstream.Upstream
 import io.emeraldpay.dshackle.upstream.MultistreamHolder
-import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
 import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream
 import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosRpcUpstream
 import io.emeraldpay.dshackle.upstream.ethereum.EthereumReader
-import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcUpstream
 import io.emeraldpay.grpc.BlockchainType
 import io.emeraldpay.grpc.Chain
 import org.jetbrains.annotations.NotNull
@@ -48,7 +46,7 @@ class MultistreamHolderMock implements MultistreamHolder {
 
     Multistream addUpstream(@NotNull Chain chain, @NotNull Upstream up) {
         if (!upstreams.containsKey(chain)) {
-            if (BlockchainType.from(chain) == BlockchainType.ETHEREUM_POS) {
+            if (BlockchainType.from(chain) == BlockchainType.EVM_POS) {
                 if (up instanceof EthereumPosMultiStream) {
                     upstreams[chain] = up
                 } else if (up instanceof EthereumPosRpcUpstream) {