From 54e708159986b5dc2cb1fdb30323f5276c9e8fba Mon Sep 17 00:00:00 2001 From: Igor Artamonov Date: Fri, 7 Jun 2019 19:39:04 -0400 Subject: [PATCH] solution: initial version --- .gitignore | 4 + build.gradle | 108 +++++++++++++++++ gradle.properties | 21 ++++ .../kotlin/io/emeraldpay/dshackle/Config.kt | 37 ++++++ .../io/emeraldpay/dshackle/GrpcServer.kt | 49 ++++++++ .../kotlin/io/emeraldpay/dshackle/Starter.kt | 14 +++ .../emeraldpay/dshackle/rpc/BlockchainRpc.kt | 17 +++ .../io/emeraldpay/dshackle/rpc/NativeCall.kt | 113 ++++++++++++++++++ .../dshackle/upstream/EthereumUpstream.kt | 24 ++++ .../emeraldpay/dshackle/upstream/Upstreams.kt | 49 ++++++++ src/main/resources/application.properties | 2 + 11 files changed, 438 insertions(+) create mode 100644 .gitignore create mode 100644 build.gradle create mode 100644 gradle.properties create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/Config.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/Starter.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/EthereumUpstream.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstreams.kt create mode 100644 src/main/resources/application.properties diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..2469c991 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.gradle/ +build/ +out/ +*.iml \ No newline at end of file diff --git a/build.gradle b/build.gradle new file mode 100644 index 00000000..7cc53610 --- /dev/null +++ b/build.gradle @@ -0,0 +1,108 @@ +buildscript { + repositories { + mavenLocal() + mavenCentral() + } + dependencies { + classpath 'org.jetbrains.kotlin:kotlin-gradle-plugin:1.3.11' + } +} + +plugins { + id 'org.jetbrains.kotlin.jvm' version '1.3.11' +} + +apply plugin: 'war' +apply plugin: 'java' +apply plugin: 'groovy' +apply plugin: "kotlin" +apply plugin: 'idea' +apply plugin: 'maven' +apply plugin: 'application' + +group = 'io.emeraldpay.dshackle' +version = '0.1-SNAPSHOT' + +targetCompatibility = '1.8' +sourceCompatibility = '1.8' + +repositories { + mavenLocal() + mavenCentral() + maven { url "http://repo.spring.io/snapshot" } + maven { url "http://repo.spring.io/milestone" } + maven { url "https://dl.bintray.com/infinitape/etherjar" } +} + +configurations { + compile.exclude group: "commons-logging" + compile.exclude group: "ch.qos.logback" + compile.exclude group: "org.slf4j", module: "slf4j-jdk14" + compile.exclude group: "org.slf4j", module: "log4j-over-slf4j" +} + +dependencies { + compile "io.emeraldpay:emerald-grpc:0.1-SNAPSHOT" + + compile "io.grpc:grpc-protobuf:${grpcVersion}" + compile "io.grpc:grpc-stub:${grpcVersion}" + compile "io.grpc:grpc-netty:${grpcVersion}" + compile "io.netty:netty-tcnative-boringssl-static:2.0.22.Final" + + compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8" + compile "org.jetbrains.kotlin:kotlin-reflect" + + compile "org.springframework:spring-core:$springVersion" + compile "org.springframework:spring-context:$springVersion" + compile "org.springframework.security:spring-security-core:$springVersion" + compile "org.springframework.security:spring-security-web:$springVersion" + compile "org.springframework.security:spring-security-config:$springVersion" + compile 'io.projectreactor:reactor-core:3.2.9.RELEASE' + compile 'io.projectreactor.addons:reactor-extra:3.2.3.RELEASE' + compile 'io.projectreactor.kotlin:reactor-kotlin-extensions:1.0.0.M1' + + + compile "io.infinitape:etherjar-domain:$etherjarVersion" + compile "io.infinitape:etherjar-hex:$etherjarVersion" + compile "io.infinitape:etherjar-rpc-http:$etherjarVersion" + compile "io.infinitape:etherjar-tx:$etherjarVersion" + + compile 'org.apache.httpcomponents:httpmime:4.5.8' + compile 'org.apache.httpcomponents:httpclient:4.5.8' + compile 'com.fasterxml.jackson.core:jackson-core:2.9.8' + compile 'com.fasterxml.jackson.core:jackson-databind:2.9.8' + compile 'commons-io:commons-io:2.6' + compile 'org.apache.commons:commons-lang3:3.9' + compile 'org.apache.commons:commons-collections4:4.3' + compile 'javax.annotation:javax.annotation-api:1.3.2' + compile 'org.bouncycastle:bcprov-jdk15on:1.61' + + compile("org.springframework.boot:spring-boot-starter:$springBootVersion") { + exclude module: 'spring-boot-starter-logging' + } + + compile "org.slf4j:slf4j-api:$slf4jVersion" + compile "org.apache.logging.log4j:log4j-slf4j-impl:2.11.1" + compile "org.slf4j:jul-to-slf4j:$slf4jVersion" + compile "org.slf4j:jcl-over-slf4j:$slf4jVersion" + + + testCompile "org.codehaus.groovy:groovy:$groovyVersion" + testCompile "org.spockframework:spock-core:$spockVersion" +} + +compileKotlin { + kotlinOptions { + jvmTarget = "1.8" + } +} +compileTestKotlin { + kotlinOptions { + jvmTarget = "1.8" + } +} + +application { + mainClassName = 'io.emeraldpay.dshackle.StarterKt' +} + diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 00000000..8c9494f8 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,21 @@ +# Languages +groovyVersion=2.5.5 +kotlinVersion=1.3.11 + +# Main Libs +slf4jVersion=1.7.25 +jacksonVersion=2.9.8 +grpcVersion=1.20.0 +protocVersion=3.7.1 +protobufVersion=3.7.1 + +# Core +springBootVersion=2.1.4.RELEASE +springVersion=5.1.4.RELEASE + +# Our Libs +etherjarVersion=0.6.0 + +# Testing +spockVersion=1.2-groovy-2.5 + diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt new file mode 100644 index 00000000..41d73b83 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt @@ -0,0 +1,37 @@ +package io.emeraldpay.dshackle + +import com.fasterxml.jackson.core.Version +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.module.SimpleModule +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.scheduling.annotation.EnableAsync +import org.springframework.scheduling.annotation.EnableScheduling +import org.springframework.scheduling.annotation.Scheduled +import java.text.SimpleDateFormat +import java.util.* + +@Configuration +@EnableScheduling +@EnableAsync +open class Config { + + @Bean + open fun objectMapper(): ObjectMapper { + val module = SimpleModule("EmeraldDShackle", Version(1, 0, 0, null, null, null)) + + val objectMapper = ObjectMapper() + objectMapper.registerModule(module) + objectMapper + .setDateFormat(SimpleDateFormat("yyyy-MM-dd\'T\'HH:mm:ss.SSS")) + .setTimeZone(TimeZone.getTimeZone("UTC")) + + return objectMapper + } + + // Temporally hack to let Spring Boot know it has something active (i.e shouldn't shutdown, as non-web server) + @Scheduled(fixedRate = 60000) + fun readCurrentTime() { + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt b/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt new file mode 100644 index 00000000..127e0086 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt @@ -0,0 +1,49 @@ +package io.emeraldpay.dshackle + +import io.grpc.Server +import io.grpc.ServerBuilder +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.core.io.ResourceLoader +import org.springframework.stereotype.Service +import javax.annotation.PostConstruct +import javax.annotation.PreDestroy + +@Service +open class GrpcServer( + @Autowired val rpcs: List, + @Autowired val resourceLoader: ResourceLoader +) { + + private val log = LoggerFactory.getLogger(GrpcServer::class.java) + + private var server: Server? = null; + + @PostConstruct + fun start() { + log.info("Starting GRPC Server...") + val serverBuilder = ServerBuilder.forPort(8090) + rpcs.forEach { + serverBuilder.addService(it) + } + +// serverBuilder +// .useTransportSecurity( +// resourceLoader.getResource("127.0.0.1.crt").inputStream, +// resourceLoader.getResource("127.0.0.1.p8.key").inputStream +// ) + + val server = serverBuilder.build() + this.server = server + + Thread { server.start() }.run() + log.info("GRPC Server started") + } + + @PreDestroy + fun stop() { + log.info("Shutting down GRPC Server...") + server?.shutdownNow() + log.info("GRPC Server shot down") + } +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt b/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt new file mode 100644 index 00000000..95b68165 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt @@ -0,0 +1,14 @@ +package io.emeraldpay.dshackle + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.context.annotation.Import + +@SpringBootApplication(scanBasePackages = [ "io.emeraldpay.dshackle" ]) +@Import(Config::class) +open class Starter + +fun main(args: Array) { + val app = SpringApplication(Starter::class.java) + app.run() +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt new file mode 100644 index 00000000..8b33213b --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt @@ -0,0 +1,17 @@ +package io.emeraldpay.dshackle.rpc + +import io.emeraldpay.api.proto.BlockchainGrpc +import io.emeraldpay.api.proto.BlockchainOuterClass +import io.grpc.stub.StreamObserver +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + +@Service +class BlockchainRpc( + @Autowired private val nativeCall: NativeCall +): BlockchainGrpc.BlockchainImplBase() { + + override fun nativeCall(request: BlockchainOuterClass.CallBlockchainRequest, responseObserver: StreamObserver) { + nativeCall.nativeCall(request, responseObserver) + } +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt new file mode 100644 index 00000000..ce253a0d --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt @@ -0,0 +1,113 @@ +package io.emeraldpay.dshackle.rpc + +import com.fasterxml.jackson.databind.ObjectMapper +import com.google.protobuf.ByteString +import io.emeraldpay.api.proto.BlockchainOuterClass +import io.emeraldpay.dshackle.upstream.Upstreams +import io.emeraldpay.grpc.Chain +import io.grpc.stub.StreamObserver +import io.infinitape.etherjar.rpc.json.ResponseJson +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import reactor.core.publisher.toFlux +import reactor.core.publisher.toMono +import reactor.util.function.Tuples +import java.lang.Exception + +@Service +class NativeCall( + @Autowired private val upstreams: Upstreams, + @Autowired private val objectMapper: ObjectMapper +) { + + private val log = LoggerFactory.getLogger(NativeCall::class.java) + + private val allowedMethods = listOf( + "eth_gasPrice", + "eth_blockNumber", + "eth_getBalance", + "eth_getStorageAt", + "eth_getTransactionCount", + "eth_getBlockTransactionCountByHash", + "eth_getBlockTransactionCountByNumber", + "eth_getUncleCountByBlockHash", + "eth_getUncleCountByBlockNumber", + "eth_getCode", + "eth_sendRawTransaction", + "eth_call", + "eth_estimateGas", + "eth_getBlockByHash", + "eth_getBlockByNumber", + "eth_getTransactionByHash", + "eth_getTransactionByBlockHashAndIndex", + "eth_getTransactionByBlockNumberAndIndex", + "eth_getTransactionReceipt", + "eth_getUncleByBlockHashAndIndex", + "eth_getUncleByBlockNumberAndIndex" + ) + + open fun nativeCall(request: BlockchainOuterClass.CallBlockchainRequest, responseObserver: StreamObserver) { + val chain= Chain.byId(request.chain.number) + if (chain == Chain.UNSPECIFIED) { + throw Exception("Invalid chain id: ${request.chain.number}") + } + val upstream = upstreams.ethereumUpstream(chain) ?: throw Exception("Chain ${chain.id} is unavailable") + request.itemsList.toFlux() + .map { + val method = it.target + val params = it.payload.toStringUtf8() + return@map CallContext(it.id, Tuples.of(method, params)) + } + .map { + val params = extractParams(it.payload.t2) + return@map it.withPayload(Tuples.of(it.payload.t1, params)) + } + .flatMap { ctx -> + upstream.execute(ctx.id, ctx.payload.t1, ctx.payload.t2).map { resp -> + ctx.withPayload(resp) + }.onErrorMap { + CallFailure(ctx.id, it) + } + } + .map { + BlockchainOuterClass.CallBlockchainReplyItem.newBuilder() + .setSucceed(true) + .setId(it.id) + .setPayload(ByteString.copyFrom(it.payload)) + .build() + } + .onErrorResume() { + val id: Int = if (it != null && CallFailure::class.isInstance(it)) { + (it as CallFailure).id + } else { + log.error("Lost context for a native call", it) + 0 + } + return@onErrorResume BlockchainOuterClass.CallBlockchainReplyItem.newBuilder() + .setSucceed(false) + .setId(id) + .build() + .toMono() + } + .doOnComplete { + responseObserver.onCompleted() + } + .subscribe { + responseObserver.onNext(it) + } + } + + private fun extractParams(jsonParams: String): List { + val req = objectMapper.readValue(jsonParams, List::class.java) + return req as List + } + + private class CallContext(val id: Int, val payload: T) { + fun withPayload(payload: X): CallContext { + return CallContext(id, payload) + } + } + + class CallFailure(val id: Int, val reason: Throwable): Exception("Failed to call $id: ${reason.message}") +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/EthereumUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/EthereumUpstream.kt new file mode 100644 index 00000000..3ce29426 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/EthereumUpstream.kt @@ -0,0 +1,24 @@ +package io.emeraldpay.dshackle.upstream + +import com.fasterxml.jackson.databind.ObjectMapper +import io.infinitape.etherjar.rpc.json.ResponseJson +import io.infinitape.etherjar.rpc.transport.RpcTransport +import reactor.core.publisher.Mono + +class EthereumUpstream( + private val rpcTransport: RpcTransport, + private val objectMapper: ObjectMapper +) { + + fun execute(id: Int, method: String, params: List): Mono { + return Mono + .fromCompletionStage(rpcTransport.execute(method, params, Any::class.java)) + .map { + val resp = ResponseJson() + resp.id = id + resp.result = it + objectMapper.writer().writeValueAsBytes(resp) + } + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstreams.kt new file mode 100644 index 00000000..26c23e58 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstreams.kt @@ -0,0 +1,49 @@ +package io.emeraldpay.dshackle.upstream + +import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.grpc.Chain +import io.infinitape.etherjar.rpc.transport.DefaultRpcTransport +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.core.env.Environment +import org.springframework.stereotype.Repository +import java.net.URI +import javax.annotation.PostConstruct + +@Repository +class Upstreams( + @Autowired val env: Environment, + @Autowired private val objectMapper: ObjectMapper +) { + + private var seq = 0 + private val chainMapping = HashMap>() + + @PostConstruct + fun start() { + env.getProperty("upstream.ethereum")?.let { + chainMapping[Chain.ETHEREUM] = listOf(buildClient(it)) + } + env.getProperty("upstream.ethereumclassic")?.let { + chainMapping[Chain.ETHEREUM_CLASSIC] = listOf(buildClient(it)) + } + env.getProperty("upstream.morden")?.let { + chainMapping[Chain.MORDEN] = listOf(buildClient(it)) + } + } + + private fun buildClient(url: String): EthereumUpstream { + return EthereumUpstream( + DefaultRpcTransport(URI(url)), + objectMapper + ) + } + + fun validateUpstream(upstream: EthereumUpstream): Boolean { + return true + } + + fun ethereumUpstream(chain: Chain): EthereumUpstream? { + val all = chainMapping[chain] ?: return null + return all[seq++ % all.size] + } +} \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 00000000..5607da14 --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,2 @@ +upstream.ethereumclassic=http://localhost:8545 +upstream.ethereum=http://localhost:8546 \ No newline at end of file