From e8fc025966c704e49d63d8c9f134d9aece1462e1 Mon Sep 17 00:00:00 2001 From: Igor Artamonov Date: Mon, 19 Jul 2021 23:15:31 -0400 Subject: [PATCH] solution: docs for access logging --- docs/01-architecture-intro.adoc | 4 +- docs/02-quick-start.adoc | 2 +- docs/03-server-config.adoc | 6 +- docs/06-monitoring.adoc | 74 +++++++++++++++++++ docs/{06-methods.adoc => 07-methods.adoc} | 2 +- ...entication.adoc => 08-authentication.adoc} | 0 ...tors.adoc => 09-quorum-and-selectors.adoc} | 0 docs/{09-caching.adoc => 10-caching.adoc} | 0 ...ibraries.adoc => 11-client-libraries.adoc} | 0 docs/README.adoc | 13 ++-- docs/reference-configuration.adoc | 65 ++++++++++++---- 11 files changed, 141 insertions(+), 25 deletions(-) create mode 100644 docs/06-monitoring.adoc rename docs/{06-methods.adoc => 07-methods.adoc} (98%) rename docs/{07-authentication.adoc => 08-authentication.adoc} (100%) rename docs/{08-quorum-and-selectors.adoc => 09-quorum-and-selectors.adoc} (100%) rename docs/{09-caching.adoc => 10-caching.adoc} (100%) rename docs/{10-client-libraries.adoc => 11-client-libraries.adoc} (100%) diff --git a/docs/01-architecture-intro.adoc b/docs/01-architecture-intro.adoc index cf0c334e..e5c8cbf0 100644 --- a/docs/01-architecture-intro.adoc +++ b/docs/01-architecture-intro.adoc @@ -29,7 +29,7 @@ And for a request: - Is it for concrete data (_block #100_) or the latest (_get balance_)? - Is result a static value (_just block_) or may vary depending on network and node (_latest nonce_)? - Does it need to be repeated over multiple nodes (_broadcast transaction_)? -- Request can also specify which subset of nodes should be able to execute the request by selecting node _Labels_ (see "link:08-quorum-and-selectors.adoc[Quorum and Selectors]") +- Request can also specify which subset of nodes should be able to execute the request by selecting node _Labels_ (see "link:09-quorum-and-selectors.adoc[Quorum and Selectors]") Based on these factors, Dshackle executes the request on a most optimal node. For most of the simple requests, it just gets a node that is synchronized to the point that must have a response for that particular request. @@ -43,7 +43,7 @@ It provides: - Routes only realy/alive upstreams, i.e., synchronized and with enough peers - Load Balancing - Request Retry on upstream errors -- Local Caching (memory and Redis, see link:09-caching.adoc[Caching]) +- Local Caching (memory and Redis, see link:10-caching.adoc[Caching]) - Broadcasting and Quorum for requests === gRPC protocol diff --git a/docs/02-quick-start.adoc b/docs/02-quick-start.adoc index 40cde176..78b87c60 100644 --- a/docs/02-quick-start.adoc +++ b/docs/02-quick-start.adoc @@ -174,4 +174,4 @@ grpcurl -import-path ./proto/ -proto blockchain.proto -d '{"asset": {"chain": "1 ... ---- -See other enhanced methods in the link:06-methods.adoc[Documentation for Enhanced Methods] +See other enhanced methods in the link:07-methods.adoc[Documentation for Enhanced Methods] diff --git a/docs/03-server-config.adoc b/docs/03-server-config.adoc index e2ed9b4d..b1e45fc9 100644 --- a/docs/03-server-config.adoc +++ b/docs/03-server-config.adoc @@ -48,7 +48,8 @@ a| `port: 12449` a| `tls` | -| TLS configuration for gRPC. See link:07-authentication.adoc[Authentication] for details +| TLS configuration for gRPC. +See link:08-authentication.adoc[Authentication] for details a| `proxy` | @@ -121,7 +122,8 @@ a| `enabled: true` a| `tls` | -| TLS configuration for proxy. See link:07-authentication.adoc[Authentication] for details +| TLS configuration for proxy. +See link:08-authentication.adoc[Authentication] for details a| `routes` | diff --git a/docs/06-monitoring.adoc b/docs/06-monitoring.adoc new file mode 100644 index 00000000..9ef6d8b1 --- /dev/null +++ b/docs/06-monitoring.adoc @@ -0,0 +1,74 @@ += Logging & Monitoring + +== Access / Request Log + +Dshackle can log all requests to a file in JSON format. +Or https://jsonlines.org/[JSON Lines] to be more precise, i.e., a test file where each line is a JSON. + +NOTE: By default, the access log is disabled. + +To enable access log add following configuration: + +[source,yaml] +---- +accessLog: + enabled: true + filename: /var/log/dshackle/access_log.jsonl +---- + +`filename` is optional, and the default value is `access_log.jsonl` (i.e., in the current directory). + +Since a single request may contain multiple replies (ex., a batch call, or subscribe to the head blocks) the Dshackle logging is based on replies. +The access log file contains details per each response send from the server, and each of them refers to original request details. + +The access log contains the JSON lines similar to: + +[source,json] +---- +{ + "version":"accesslog/v1beta", + "ts":"2021-07-20T01:53:33.174645Z", + "id":"578d83db-cf53-4ef8-b73e-3f1cc0a67e96", + "method":"NativeCall", + "blockchain":"ETHEREUM", + "total":2, + "index":0, + "succeed":true, + "request":{ + "id":"513b9b49-b472-4c83-b4b7-58dd2aabe9f6", + "start":"2021-07-20T01:53:33.086946Z", + "remote":{ + "ips":["127.0.0.1", "10.0.5.102", "172.217.8.78"], + "ip":"172.217.8.78", + "userAgent":"grpc-node-js/1.1.8" + } + }, + "nativeCall":{ + "method":"eth_blockNumber", + "id":2, + "payloadSizeBytes":2 + } +} +---- + +.Where: +- `ts` timestamp of the reply +- `id` uniq id of the reply +- `method` Dshackle method which was called (i.e., not a Blockchain API method, see `nativeCall` details) +- `blockchain` blockchain code +- `total` how many requests in the batch (available only for a `NativeCall` call) +- `index` current index (i.e. count) of the reply to the original request +- `succeed` if call succeeded, in terms of Blockchain API +- `request` original request details +** `id` uniq id of the request; all replied to the same request have same id +** `start` when request was received +** `remote` remote details +*** `ips` list of all recognized IPs (including headers such as `X-Real-IP` and `X-Forwarded-For`) +*** `ip` a single ip, that likely represent a real IP of the remote +*** `userAgent` user agent +- `nativeCall` details of the individual Native Call request +** `method` method name terms of Blockchain API +** `id` request id provided in the original request +** `payloadSizeBytes` size of the original _individual_ request (for JSON RPC it's size of the `params` value) + + diff --git a/docs/06-methods.adoc b/docs/07-methods.adoc similarity index 98% rename from docs/06-methods.adoc rename to docs/07-methods.adoc index a5936743..8ffdae78 100644 --- a/docs/06-methods.adoc +++ b/docs/07-methods.adoc @@ -170,4 +170,4 @@ message TxStatus { === gRPC Client Libraries -See link:10-client-libraries.adoc[Client Libraries] documentation. +See link:11-client-libraries.adoc[Client Libraries] documentation. diff --git a/docs/07-authentication.adoc b/docs/08-authentication.adoc similarity index 100% rename from docs/07-authentication.adoc rename to docs/08-authentication.adoc diff --git a/docs/08-quorum-and-selectors.adoc b/docs/09-quorum-and-selectors.adoc similarity index 100% rename from docs/08-quorum-and-selectors.adoc rename to docs/09-quorum-and-selectors.adoc diff --git a/docs/09-caching.adoc b/docs/10-caching.adoc similarity index 100% rename from docs/09-caching.adoc rename to docs/10-caching.adoc diff --git a/docs/10-client-libraries.adoc b/docs/11-client-libraries.adoc similarity index 100% rename from docs/10-client-libraries.adoc rename to docs/11-client-libraries.adoc diff --git a/docs/README.adoc b/docs/README.adoc index 5aa5cdae..9c452b2f 100644 --- a/docs/README.adoc +++ b/docs/README.adoc @@ -3,7 +3,7 @@ == What is Dshackle Dshackle is a L7 Load Balancer for Blockchain APIs with automatic discovery, health checking, secure access, TLS with -client authentication, and many other features. It can be configured as an edge proxy, middle proxy or API gateway. +client authentication, and many other features.It can be configured as an edge proxy, middle proxy or API gateway. Dshackle provided a high level aggregated API on top of several underlying upstreams (blockchain nodes or providers, such as Geth, Parity, Infura, etc), automatically verifies their availability and the current status of the network, @@ -38,11 +38,12 @@ Main goals: . link:03-server-config.adoc[Server Configuration] . link:04-upstream-config.adoc[Upstreams Configuration] . link:05-start.adoc[How to launch a server] -. link:06-methods.adoc[API methods] -. link:07-authentication.adoc[Authentication] -. link:08-quorum-and-selectors.adoc[Quorum and Selectors] -. link:09-caching.adoc[Caching] -. link:10-client-libraries.adoc[Client Libraries] +. link:06-monitoring.adoc[Logging & Monitoring] +. link:07-methods.adoc[API methods] +. link:08-authentication.adoc[Authentication] +. link:09-quorum-and-selectors.adoc[Quorum and Selectors] +. link:10-caching.adoc[Caching] +. link:11-client-libraries.adoc[Client Libraries] == Reference diff --git a/docs/reference-configuration.adoc b/docs/reference-configuration.adoc index 3d992060..d8d615c9 100644 --- a/docs/reference-configuration.adoc +++ b/docs/reference-configuration.adoc @@ -58,6 +58,10 @@ tokens: type: ERC-20 address: 0xdac17f958d2ee523a2206206994597c13d831ec7 +accessLog: + enabled: true + filename: /var/log/dshackle/access_log.jsonl + cluster: defaults: - chains: @@ -142,24 +146,33 @@ cluster: | `tls` | -| Setup TLS configuration for the gRPC server. See <> section +| Setup TLS configuration for the gRPC server. +See <> section | `proxy` | -| Setup HTTP proxy that emulates all standard JSON RPC requests. See <> section +| Setup HTTP proxy that emulates all standard JSON RPC requests. +See <> section + +| `accessLog` +| +| Configure access logging. +See <> section + | `tokens` | -| Configure tokens for tracking balance. See <> section - +| Configure tokens for tracking balance. +See <> section | `cache` | -| Caching configuration. See <> section. +| Caching configuration. +See <> section. | `cluster` | -| Setup connection to remote nodes. See <> section +| Setup connection to remote nodes.See <> section |=== @@ -247,7 +260,7 @@ proxy: | `routes` | -a| Routing paths for Proxy. The proxy will handle requests as `https://${HOST}:${PORT}/${ROUTE_ID}` (or `http://` if TLS is not enabled) +a| Routing paths for Proxy.The proxy will handle requests as `https://${HOST}:${PORT}/${ROUTE_ID}` (or `http://` if TLS is not enabled) |=== .Route config @@ -265,6 +278,31 @@ a| Routing paths for Proxy. The proxy will handle requests as `https://${HOST}:$ |=== +[#accessLog] +== Access Log config + +[source,yaml] +---- +accessLog: + enabled: true + filename: /var/log/dshackle/access_log.jsonl +---- + +.Access Log config +[cols="2a,3a,7"] +|=== +| Option | Default | Description + +| `enabled` +| `false` +| Enable/Disable Access logging + +| `filename` +| `access_log.jsonl` +| Path to the access log file + +|=== + [#tokens] == Tokens config @@ -284,7 +322,7 @@ tokens: ---- Tokens config enables tracking of a balance amount in the configured tokens. -After making the configuration above you can request balance (`GetBalance`), or subscribe to balance changes (`SubscribeBalance`), using link:06-methods.adoc[enhanced protocol] +After making the configuration above you can request balance (`GetBalance`), or subscribe to balance changes (`SubscribeBalance`), using link:07-methods.adoc[enhanced protocol] .Token config [cols="2a,7"] @@ -301,7 +339,7 @@ After making the configuration above you can request balance (`GetBalance`), or | Name of the token, used for balance response as asset code (as converted to UPPERCASE) | `type` -| Type of token. Only `ERC-20` is supported at this moment +| Type of token.Only `ERC-20` is supported at this moment | `address` | Address of the deployed contract @@ -446,7 +484,7 @@ Accepted types: `bitcoin`, `bitcoin-testnet`, `ethereum`, `ethereum-classic`, `k | no | Key-Value pairs that are assigned to the upstream. Used to select an upstream per-request. -See link:08-quorum-and-selectors.adoc[Quorum and Selectors] +See link:09-quorum-and-selectors.adoc[Quorum and Selectors] | `methods` | no @@ -538,9 +576,10 @@ It's more effective, easier to secure connection, and allows to build a distribu | Address to connect to | `tls` -a| TLC configuration for the connection. In general it's an optional configuration, but it's strongly recommended. Also -HTTP2 + gRPC is designed to be used with TLS, and some of the related software is unable to use it without TLS. + - See link:07-authentication.adoc[Authentication] docs and <>. +a| TLC configuration for the connection. +In general it's an optional configuration, but it's strongly recommended. +Also HTTP2 + gRPC is designed to be used with TLS, and some of the related software is unable to use it without TLS. + +See link:08-authentication.adoc[Authentication] docs and <>. | `tls.ca` | Path to x509 certificate to verify remote server