added total difficulty & blocks sequence checks

src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractHead.kt

@@ -57,7 +57,7 @@ abstract class AbstractHead(
             }
             .subscribeOn(Schedulers.boundedElastic())
             .subscribe { block ->
-                if (blockValidator.isValid(block)) {
+                if (blockValidator.isValid(forkChoice.getHead(), block)) {
                     notifyBeforeBlock()
                     when (val choiceResult = forkChoice.choose(block)) {
                         is ForkChoice.ChoiceResult.Updated -> {

src/main/kotlin/io/emeraldpay/dshackle/upstream/BlockValidator.kt

@@ -4,10 +4,10 @@ import io.emeraldpay.dshackle.data.BlockContainer
 
 interface BlockValidator {
 
-    fun isValid(block: BlockContainer): Boolean
+    fun isValid(currentHead: BlockContainer?, newHead: BlockContainer): Boolean
 
     class AlwaysValid : BlockValidator {
-        override fun isValid(block: BlockContainer): Boolean = true
+        override fun isValid(currentHead: BlockContainer?, newHead: BlockContainer): Boolean = true
     }
 
     companion object {

src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumBlockValidator.kt

@@ -10,13 +10,16 @@ import io.emeraldpay.dshackle.upstream.ethereum.RLP.Companion.encodeBigInt
 import io.emeraldpay.dshackle.upstream.ethereum.RLP.Companion.encodeList
 import io.emeraldpay.dshackle.upstream.ethereum.RLP.Companion.fromHexString
 import io.emeraldpay.dshackle.upstream.ethereum.RLP.Companion.fromHexStringI
+import io.emeraldpay.etherjar.rpc.json.BlockJson
 import org.bouncycastle.jcajce.provider.digest.Keccak
 import org.slf4j.LoggerFactory
+import java.math.BigDecimal
 import java.math.BigInteger
 
 class EthereumBlockValidator : BlockValidator {
 
     companion object {
+        val ACCURACY = 0.02.toBigDecimal()
         val MAX_BIG_INT: BigInteger = BigInteger.valueOf(2).pow(256)
         val NUMBER_ELEMENTS = setOf("difficulty", "number", "gasLimit", "gasUsed", "timestamp", "baseFeePerGas")
         val ELEMENTS = setOf(
@@ -41,22 +44,22 @@ class EthereumBlockValidator : BlockValidator {
         private val log = LoggerFactory.getLogger(EthereumBlockValidator::class.java)
     }
 
-    override fun isValid(block: BlockContainer): Boolean =
+    override fun isValid(currentHead: BlockContainer?, newHead: BlockContainer): Boolean =
-        block.json?.let {
+        newHead.json?.let {
             println(String(it))
+            val validBlocksSequence = currentHead?.let { cur -> validateTotalDifficulty(cur, newHead) } ?: true
             val node = Global.objectMapper.readTree(it)
             val rlpEncoded = rlp(node)
-            val hashValid = validateHash(block.hash, rlpEncoded)
+            val hashValid = validateHash(newHead.hash, rlpEncoded)
-
             if (!hashValid) {
-                log.warn("Hash not valid for block ${block.hash}")
+                log.warn("Hash '${newHead.hash}' not valid for block ${newHead.hash}")
             }
             val difficultyValid = validateDifficulty(node, rlpEncoded)
             if (!difficultyValid) {
-                log.warn("PoW not valid for block ${block.hash}")
+                log.warn("PoW not valid for block ${newHead.hash}")
             }
 
-            hashValid && difficultyValid
+            hashValid && difficultyValid && validBlocksSequence
         } ?: false
 
     private fun rlp(node: JsonNode): Map<String, ByteArray> =
@@ -77,8 +80,11 @@ class EthereumBlockValidator : BlockValidator {
     }
 
     private fun validateDifficulty(node: JsonNode, rlpEncoded: Map<String, ByteArray>): Boolean {
-        val nonce = fromHexStringI(node["nonce"].asText())
         val difficulty = fromHexStringI(node["difficulty"].asText())
+        if (difficulty.signum() < 0) {
+            return false
+        }
+        val nonce = fromHexStringI(node["nonce"].asText())
         val mix = fromHexString(node["mixHash"].asText())
 
         val target = MAX_BIG_INT.divide(difficulty)
@@ -87,14 +93,90 @@ class EthereumBlockValidator : BlockValidator {
                 .mapNotNull { rlpEncoded[it] }
                 .toList()
         )
-        val sha3 = sha3(encoded)
+        val headerHash = sha3(encoded)
-        val seed = Keccak.Digest512().digest(sha3.plus(nonce.asUint64()))
+        val seed = Keccak.Digest512().digest(headerHash.plus(nonce.asUint64()))
         val result = sha3(seed.plus(mix))
         return target >= BigInteger(1, result)
     }
 
+    private fun validateTotalDifficulty(currentHead: BlockContainer, newHead: BlockContainer): Boolean =
+        currentHead.getParsed(BlockJson::class.java)?.let { cur ->
+            newHead.getParsed(BlockJson::class.java)?.let {
+                val numberValid = it.number >= cur.number
+                if (!numberValid) {
+                    log.warn(
+                        "Block number {} not valid for {}. Must be greater than {}",
+                        it.number,
+                        it.hash,
+                        cur.number
+                    )
+                }
+
+                val timestampValid = it.timestamp > cur.timestamp
+                if (!timestampValid) {
+                    log.warn(
+                        "Block timestamp {} not valid for {}. Must be greater than {}",
+                        it.timestamp,
+                        it.hash,
+                        cur.timestamp
+                    )
+                }
+
+                numberValid && timestampValid && approxTotalDifficultyValid(
+                    curTotalDifficulty = cur.totalDifficulty,
+                    curDifficulty = cur.difficulty,
+                    curNumber = cur.number,
+                    blockTotalDifficulty = it.totalDifficulty,
+                    blockNumber = it.number,
+                    blockHash = it.hash.toHex()
+                )
+            } ?: false
+        } ?: true
+
+    private fun approxTotalDifficultyValid(
+        curTotalDifficulty: BigInteger,
+        curDifficulty: BigInteger,
+        curNumber: Long,
+        blockTotalDifficulty: BigInteger,
+        blockNumber: Long,
+        blockHash: String
+    ): Boolean {
+        if (blockTotalDifficulty < curTotalDifficulty) {
+            log.warn(
+                "Block totalDifficulty {} not valid for {}. Must be greater than {}",
+                blockTotalDifficulty,
+                blockHash,
+                curTotalDifficulty
+            )
+            return false
+        }
+
+        val decimalRef = curDifficulty.toBigDecimal()
+        val shift = BigDecimal.valueOf(blockNumber - curNumber)
+        val totalAccuracy = shift * ACCURACY
+
+        val blockApprox = (blockTotalDifficulty - curTotalDifficulty).toBigDecimal() / shift
+        val min = decimalRef * (BigDecimal.ONE - totalAccuracy)
+        val max = decimalRef * (BigDecimal.ONE + totalAccuracy)
+
+        val valid = blockApprox in min..max
+        if (!valid) {
+            log.warn(
+                "Block totalDifficulty {} not valid for {}. Expected to be int [{}, {}]",
+                blockTotalDifficulty,
+                blockHash,
+                min.toBigInteger(),
+                max.toBigInteger()
+            )
+        }
+        return valid
+    }
+
     private fun BigInteger.asUint64() = this.toByteArray().let { bytes ->
-        ByteArray(8) { bytes[bytes.size - it - 1] }
+        ByteArray(8) {
+            val index = bytes.size - it - 1
+            if (index < bytes.size) bytes[index] else 0
+        }
     }
 
     private fun sha3(byteArray: ByteArray): ByteArray =

src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumBlockValidatorSpec.groovy

@@ -1,29 +1,57 @@
 package io.emeraldpay.dshackle.upstream.ethereum
 
-import com.fasterxml.jackson.databind.JsonNode
+
 import io.emeraldpay.dshackle.Global
 import io.emeraldpay.dshackle.data.BlockContainer
 import io.emeraldpay.dshackle.data.BlockId
+import io.emeraldpay.etherjar.rpc.json.BlockJson
 import org.apache.commons.io.FileUtils
 import spock.lang.Specification
 
-import java.time.Instant
-
 class EthereumBlockValidatorSpec extends Specification {
 
+    def mapper = Global.getObjectMapper()
     def validator = new EthereumBlockValidator()
 
     def "test validate block"() {
         expect:
         def path = "src/test/resources/blocks/${file}.json"
-        def bytes = FileUtils.readFileToByteArray(new File(path))
+        validator.isValid(null, fromFile(path)) == expect
-
-        def container = new BlockContainer(1L, BlockId.from(hash), BigInteger.ONE, Instant.now(), false, bytes, null, Collections.emptyList(), 1)
-        validator.isValid(container) == expect
 
         where:
-        file                || expect || hash
+        file                      || expect
-        "eth_valid_block_1" || true   || "0xae174c5cf816f820b17ae13d120fff057406ce12f448422f0d522459d4ae646b"
+        "eth_valid_block_1"       || true
-        "eth_valid_block_2" || true   || "0x95fd7b85ed04cc67884d7c33d13df3ebb568a2c1532021316c296038bda71776"
+        "eth_block_sec_1"         || true
+        "eth_block_sec_2"         || true
+        "eth_block_sec_2_invalid" || true
+    }
+
+    def "test validate blocks sequence"() {
+        expect:
+        def curPath = "src/test/resources/blocks/${cur}.json"
+        def nextPath = "src/test/resources/blocks/${next}.json"
+
+        validator.isValid(fromFile(curPath), fromFile(nextPath)) == expect
+
+        where:
+        cur               || next                      || expect
+        "eth_block_sec_1" || "eth_block_sec_2"         || true
+        "eth_block_sec_2" || "eth_block_sec_1"         || false
+        "eth_block_sec_1" || "eth_block_sec_2_invalid" || false
+    }
+
+    def fromFile(String path) {
+        def bytes = FileUtils.readFileToByteArray(new File(path))
+        def block = mapper.readValue(bytes, BlockJson.class)
+
+        return new BlockContainer(
+                block.number,
+                BlockId.from(block.hash.toHex()),
+                block.difficulty,
+                block.timestamp,
+                true, bytes,
+                block,
+                Collections.emptyList(),
+                1)
     }
 }

src/test/resources/blocks/eth_block_sec_1.json

@@ -0,0 +1,23 @@
+{
+  "baseFeePerGas": "0x121e9f5ec",
+  "difficulty": "0x2a76b9975fb402",
+  "extraData": "0x457468657265756d50504c4e532f326d696e6572735f455536",
+  "gasLimit": "0x1c9c380",
+  "gasUsed": "0x1c995f7",
+  "hash": "0xb359a505ab41388cb97f3d7a270f9a5a70ee46f3b496c5f5f367f3b13c6858bf",
+  "logsBloom": "0x37fbc56e6b8fd9bef7f7ffd0ff7557f386b67de3cd2c433a7a7fdafa3db9c96f7de997ecde4c7f75540e9bf7607f1baebe79bffaae7ffb9ff783dce2b5f7e3e3f75fc8a75bf8a5fff9cf63fb5be179e10ad75eec9f7e9b5b31fbfed7c85749bfff6dc82532ff7fa999dfdddcf7e46efb3a5f2c7f8e5fa7777f23dff6f5ffe60afb359fee6e639e7fd94dafe1eff7a8676f78efc39f7cf3ddadafbffb4eb6b4dbffd263f9ffbfff708ef7ffbbfd56edbdcfefbed6aaae0f9b7b6fdff7feffe67dc7afbcfeedb2ad73ffefbd2edc9f5ceb1f6c37f1b17feeb77b5fdf3f727f69fbb476ffcaf8773de7e7ff3725097c4a8db577de793ede8df472ccde9d7e0aff7e",
+  "miner": "0x00192fb10df37c9fb26829eb2cc623cd1bf599e8",
+  "mixHash": "0x61b14f607b1c9ff2aa14e65ce8e87f2831cd37e1a0b46cfeac8e6fb8d35b9cb0",
+  "nonce": "0x1da600070ccbad45",
+  "number": "0xeb1db1",
+  "parentHash": "0x17b1fe61ce7690725af12865b0f63e20c9c5944748cfbba6041362acc8ca25b4",
+  "receiptsRoot": "0xcec742556a657826cd4d9ceaf00c2a241a28d6145b6e4b6af077820c710930c6",
+  "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+  "size": "0x2397d",
+  "stateRoot": "0x2a2773fd8b4d55e2e777bb71adc4d99d913d9520be2026e26dd6c8c22527a204",
+  "timestamp": "0x63074d3c",
+  "totalDifficulty": "0xc1b02a29da16ecdfb95",
+  "transactions": [],
+  "transactionsRoot": "0xb74a34aeee1e746f60747225b9671d71df22dd368a308ef436059a0e7877c896",
+  "uncles": []
+}

src/test/resources/blocks/eth_block_sec_2.json

@@ -0,0 +1,23 @@
+{
+  "baseFeePerGas": "0x1461ffefd",
+  "difficulty": "0x2a6c5be8f9dc16",
+  "extraData": "0x4554482e4352415a59504f4f4c2e4f5247",
+  "gasLimit": "0x1ca35ef",
+  "gasUsed": "0x1ca11b3",
+  "hash": "0xc3c93de6e76e5da05df8a48ab15dfb0cc610d21932a0220e797e0adc05118ed8",
+  "logsBloom": "0xffb4778ec39a1aeef1b072f2bb1f47bb91266fd117530aeb7fbdda78deee7d6ff39e5ff8b7d83ad061bf1b9e083709ff0abfbe6afbfffdd73e49f77f80ff377d87dfa1de3f7d94bff9f66bbf5060f9ed20d74468ffff9c29ff07df4de9fdbffa1eed92e043fbc7dbf95efff9aef7ace1bace1e7a94de65e7527beed2454deeceeb609fbc9256d1fbbf6dbee5975335775f72fcd5e7e2585bad718cd278bbf66cefdbefe17acb76375befcbf9fd9ceb9d57d6fbe6ef7ae03fcb7b3eeed9fde946ffb9f5be4d8becf76f6c2633c73ed7c20df1f6e29c7e5bff1f9c79365e3d6dd5bc7afbbbcd7e1afbe5b657f8ab66ac768ee6ace959f9a77f78f9bf95babbffa7",
+  "miner": "0x4f9bebe3adc3c7f647c0023c60f91ac9dffa52d5",
+  "mixHash": "0xf6d6b09cae8efa94f2caa018c38c5c17ab1d89ab3d84feca076d1894f2f07a8f",
+  "nonce": "0xf7d487f95819f012",
+  "number": "0xeb1db2",
+  "parentHash": "0xb359a505ab41388cb97f3d7a270f9a5a70ee46f3b496c5f5f367f3b13c6858bf",
+  "receiptsRoot": "0x4e3cd00367e2a59339612ed2d10dcec5ec9ccdcbeb66a343bf1a8a115b9ce04e",
+  "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+  "size": "0x2086f",
+  "stateRoot": "0x8e2f50554d5e8bb783f12db999b5ff8a948e980a80593e877574eaa089ad362b",
+  "timestamp": "0x63074d58",
+  "totalDifficulty": "0xc1b02cd09fd57c7d7ab",
+  "transactions": [],
+  "transactionsRoot": "0x22b3c3ce7398ea4ee2665a5f13a7b77c4828802f160fc261f82b551f0a37a97c",
+  "uncles": []
+}

src/test/resources/blocks/eth_block_sec_2_invalid.json

@@ -0,0 +1,23 @@
+{
+  "baseFeePerGas": "0x1461ffefd",
+  "difficulty": "0x2a6c5be8f9dc16",
+  "extraData": "0x4554482e4352415a59504f4f4c2e4f5247",
+  "gasLimit": "0x1ca35ef",
+  "gasUsed": "0x1ca11b3",
+  "hash": "0xc3c93de6e76e5da05df8a48ab15dfb0cc610d21932a0220e797e0adc05118ed8",
+  "logsBloom": "0xffb4778ec39a1aeef1b072f2bb1f47bb91266fd117530aeb7fbdda78deee7d6ff39e5ff8b7d83ad061bf1b9e083709ff0abfbe6afbfffdd73e49f77f80ff377d87dfa1de3f7d94bff9f66bbf5060f9ed20d74468ffff9c29ff07df4de9fdbffa1eed92e043fbc7dbf95efff9aef7ace1bace1e7a94de65e7527beed2454deeceeb609fbc9256d1fbbf6dbee5975335775f72fcd5e7e2585bad718cd278bbf66cefdbefe17acb76375befcbf9fd9ceb9d57d6fbe6ef7ae03fcb7b3eeed9fde946ffb9f5be4d8becf76f6c2633c73ed7c20df1f6e29c7e5bff1f9c79365e3d6dd5bc7afbbbcd7e1afbe5b657f8ab66ac768ee6ace959f9a77f78f9bf95babbffa7",
+  "miner": "0x4f9bebe3adc3c7f647c0023c60f91ac9dffa52d5",
+  "mixHash": "0xf6d6b09cae8efa94f2caa018c38c5c17ab1d89ab3d84feca076d1894f2f07a8f",
+  "nonce": "0xf7d487f95819f012",
+  "number": "0xeb1db2",
+  "parentHash": "0xb359a505ab41388cb97f3d7a270f9a5a70ee46f3b496c5f5f367f3b13c6858bf",
+  "receiptsRoot": "0x4e3cd00367e2a59339612ed2d10dcec5ec9ccdcbeb66a343bf1a8a115b9ce04e",
+  "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+  "size": "0x2086f",
+  "stateRoot": "0x8e2f50554d5e8bb783f12db999b5ff8a948e980a80593e877574eaa089ad362b",
+  "timestamp": "0x63074d58",
+  "totalDifficulty": "0xc1b03cd09fd57c7d7ab",
+  "transactions": [],
+  "transactionsRoot": "0x22b3c3ce7398ea4ee2665a5f13a7b77c4828802f160fc261f82b551f0a37a97c",
+  "uncles": []
+}

src/test/resources/blocks/eth_valid_block_2.json

@@ -1,22 +0,0 @@
-{
-  "baseFeePerGas": "0x1bee2dd60",
-  "difficulty": "0x2a392ce7577bb6",
-  "extraData": "0x706f6f6c696e2e636f6d203bb704c2ea57e9ea",
-  "gasLimit": "0x1c95111",
-  "gasUsed": "0x1c92201",
-  "hash": "0x95fd7b85ed04cc67884d7c33d13df3ebb568a2c1532021316c296038bda71776",
-  "logsBloom": "0xcebecdfb7deb7ced9da83f9dfbbadf778315b7cb669d85bee9af5a76dcaebdbf9f7eb5ed9dd15d1ff4a1bb6f67ff4bdc4f1db5a95efbbbf4a655b2ff6d3ff7fde4fb98b99f59b5fefa921fd962296df2bcdbdfbf8ff55d7d12c2df1fbcbddadc5f12abc5e72af47ec084be69a17a3fdbff7dea6fff7d5ff60fdbecd2cf6cbebdcf789f4ce95fbbe416ef9bffcf6ea4f65fa9adf3fb6bdd1a7efffbfe49bbbbb5fbddab6df5ea7e689fa9ebf6ed76bdb497dc2bceff39f31a1e33fff7dcfd637cdb53f70fb3cf0cfb6ff0fbbebdfbd7dcaffaff46ea5f8b9e6e757ffefcfdf7dea17f7f7cffef7f4d5ea055fd7f6c7cafaeeb98b9496d0e5df7eedab1be5b79ef",
-  "miner": "0x8f03f1a3f10c05e7cccf75c1fd10168e06659be7",
-  "mixHash": "0x1071fcdfa97b5a7a7c65ff9376ad212815d89986c1530c8b23f84d31c13cad12",
-  "nonce": "0xe6de1dde14d49aeb",
-  "number": "0xeaf0bb",
-  "parentHash": "0x431533a3fec65477d91c310a78ac766c2c660b4c410db6c429cb71ca069cc441",
-  "receiptsRoot": "0x01253f659cdf7e7223ef4b254fa7e654c1813bb1f39676033ff34b18c65ccb83",
-  "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
-  "size": "0x230b2",
-  "stateRoot": "0x482376d49e5a49695f28fbc43cadb7c09ec1486d0951280ca1d92590b5b83c8d",
-  "timestamp": "0x6304e043",
-  "totalDifficulty": "0xc1370904c375c59ceb9",
-  "transactionsRoot": "0x7ada19c3ef58d5470452dfa2a45611c632e36204b90a77da0419225f0364f713",
-  "uncles": []
-}