From c10e871069d1bae046dacbf366c4438a8e9376b5 Mon Sep 17 00:00:00 2001 From: SpontaneousOverthrow Date: Thu, 17 Mar 2022 21:19:04 +0300 Subject: [PATCH] solution: reports for request errors/failures (#151) --- dashboard/alerts/dshackle.yml | 61 +++ dashboard/dshackle.json | 905 ++++++++++++++++++++++------------ 2 files changed, 655 insertions(+), 311 deletions(-) create mode 100644 dashboard/alerts/dshackle.yml diff --git a/dashboard/alerts/dshackle.yml b/dashboard/alerts/dshackle.yml new file mode 100644 index 00000000..02be3142 --- /dev/null +++ b/dashboard/alerts/dshackle.yml @@ -0,0 +1,61 @@ +groups: +- name: Dshackle + rules: + - alert: JsonRpcDshackleErrorRateIsHigh + expr: rate(dshackle_request_jsonrpc_err_total[1m]) > 0.1 + for: 1m + labels: + severity: critical + annotations: + summary: "Json RPC erorr rate is high for (instance {{ $labels.instance }})" + + - alert: JsonRpcDshackleFailRateIsHigh + expr: rate(dshackle_request_jsonrpc_fail_total[1m]) > 0.1 + for: 1m + labels: + severity: critical + annotations: + summary: "Json RPC erorr rate is high for (instance {{ $labels.instance }})" + + + - alert: JsonRpcDshackleRateToHigh + expr: rate(dshackle_request_jsonrpc_request_total[1m]) > 1000 + for: 1m + labels: + severity: critical + annotations: + summary: "Json RPC rate is high for (instance {{ $labels.instance }})" + + + - alert: WsRpcDshackleErrorRateIsHigh + expr: rate(dshackle_upstream_ws_err_total[1m]) > 0.1 + for: 1m + labels: + severity: critical + annotations: + summary: "WS RPC erorr rate is high for (instance {{ $labels.instance }})" + + - alert: UpstreamUnavailable + expr: dshackle_upstreams_availability{status!~"(ok|lagging)"} > 0 + for: 1m + labels: + severity: warning + annotations: + summary: "One of the eth node is broken (instance {{ $labels.instance }})" + + - alert: NoEthNodesForNetwork + expr: dshackle_upstreams_connected < 1 + for: 1m + labels: + severity: critical + annotations: + summary: "No nodes for network on (instance {{ $labels.instance }})" + + - alert: UpstreamIsNotSyncing + expr: rate(dshackle_upstreams_lag[5m]) < 0 + for: 1m + labels: + severity: critical + annotations: + summary: "ETH Node is not syncing on host (instance {{ $labels.instance }})" + diff --git a/dashboard/dshackle.json b/dashboard/dshackle.json index 51748df3..52dd2d44 100644 --- a/dashboard/dshackle.json +++ b/dashboard/dshackle.json @@ -8,18 +8,25 @@ "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, "type": "dashboard" } ] }, "editable": true, - "gnetId": null, + "fiscalYearStartMonth": 0, "graphTooltip": 0, - "iteration": 1638439560440, + "id": 4, + "iteration": 1647500182961, "links": [], + "liveNow": false, "panels": [ { - "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -29,18 +36,10 @@ "id": 4, "panels": [], "repeat": "chain", - "scopedVars": { - "chain": { - "selected": true, - "text": "ETH", - "value": "ETH" - } - }, "title": "CHAIN: $chain", "type": "row" }, { - "datasource": null, "fieldConfig": { "defaults": { "color": { @@ -208,14 +207,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "7.5.10", - "scopedVars": { - "chain": { - "selected": true, - "text": "ETH", - "value": "ETH" - } - }, + "pluginVersion": "8.3.5", "targets": [ { "exemplar": true, @@ -227,13 +219,10 @@ "refId": "A" } ], - "timeFrom": null, - "timeShift": null, "title": "Upstreams Availability", "type": "stat" }, { - "datasource": null, "fieldConfig": { "defaults": { "color": { @@ -241,6 +230,7 @@ }, "custom": { "align": "center", + "displayMode": "auto", "filterable": false }, "mappings": [], @@ -268,16 +258,16 @@ }, "id": 7, "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, "showHeader": true }, - "pluginVersion": "7.5.10", - "scopedVars": { - "chain": { - "selected": true, - "text": "ETH", - "value": "ETH" - } - }, + "pluginVersion": "8.3.5", "targets": [ { "exemplar": true, @@ -289,8 +279,6 @@ "refId": "A" } ], - "timeFrom": null, - "timeShift": null, "title": "Upstreams Lag", "transformations": [ { @@ -319,7 +307,6 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, "fieldConfig": { "defaults": { "unit": "reqps" @@ -353,17 +340,10 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.10", + "pluginVersion": "8.3.5", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "chain": { - "selected": true, - "text": "ETH", - "value": "ETH" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -386,9 +366,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "JSON RPC total request / failed requests", "tooltip": { "shared": true, @@ -397,9 +375,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -407,25 +383,20 @@ { "$$hashKey": "object:124", "format": "reqps", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true }, { "$$hashKey": "object:125", "format": "short", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": false } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -433,7 +404,6 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, "fieldConfig": { "defaults": { "unit": "reqps" @@ -467,17 +437,10 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.10", + "pluginVersion": "8.3.5", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "chain": { - "selected": true, - "text": "ETH", - "value": "ETH" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -500,9 +463,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "GRPC total request / failed requests", "tooltip": { "shared": true, @@ -511,9 +472,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -521,25 +480,20 @@ { "$$hashKey": "object:124", "format": "reqps", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true }, { "$$hashKey": "object:125", "format": "short", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -547,18 +501,288 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, "fieldConfig": { - "defaults": {}, + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 137, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.3.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (method) (rate(dshackle_request_jsonrpc_request_total{chain=~\"$chain\", instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "{{ method }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "JSON RPC OK requests by method", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:124", + "format": "reqps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:125", + "format": "short", + "logBase": 1, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, "overrides": [] }, "fill": 1, "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 139, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.3.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lQ3xLXW7k" + }, + "exemplar": true, + "expr": "sum by (method) (rate(dshackle_request_jsonrpc_err_total{chain=~\"$chain\", instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "{{ method }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "JSON RPC Error requests by method", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:124", + "format": "reqps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:125", + "format": "short", + "logBase": 1, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 28 + }, + "hiddenSeries": false, + "id": 141, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.3.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lQ3xLXW7k" + }, + "exemplar": true, + "expr": "sum by (method) (rate(dshackle_request_jsonrpc_fail_total{chain=~\"$chain\", instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "{{ method }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "JSON RPC Failed requests by method", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:124", + "format": "reqps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:125", + "format": "short", + "logBase": 1, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 24, "x": 0, - "y": 18 + "y": 37 }, "hiddenSeries": false, "id": 43, @@ -579,17 +803,10 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.10", + "pluginVersion": "8.3.5", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "chain": { - "selected": true, - "text": "ETH", - "value": "ETH" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -604,9 +821,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "JSON RPC Response time", "tooltip": { "shared": true, @@ -615,9 +830,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -625,25 +838,20 @@ { "$$hashKey": "object:124", "format": "s", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true }, { "$$hashKey": "object:125", "format": "short", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -651,7 +859,6 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, "fieldConfig": { "defaults": { "unit": "reqps" @@ -664,7 +871,7 @@ "h": 10, "w": 24, "x": 0, - "y": 27 + "y": 46 }, "hiddenSeries": false, "id": 133, @@ -685,7 +892,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.10", + "pluginVersion": "8.3.5", "pointradius": 2, "points": false, "renderer": "flot", @@ -703,9 +910,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Upstreams Errors", "tooltip": { "shared": true, @@ -714,9 +919,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -724,25 +927,20 @@ { "$$hashKey": "object:124", "format": "reqps", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true }, { "$$hashKey": "object:125", "format": "short", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": false } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -750,18 +948,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, - "y": 37 + "y": 56 }, "hiddenSeries": false, "id": 97, @@ -782,17 +975,10 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.10", + "pluginVersion": "8.3.5", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "chain": { - "selected": true, - "text": "ETH", - "value": "ETH" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -807,9 +993,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "JSON RPC upstream conn seconds 50 percentile", "tooltip": { "shared": true, @@ -818,9 +1002,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -828,25 +1010,20 @@ { "$$hashKey": "object:124", "format": "s", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true }, { "$$hashKey": "object:125", "format": "short", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -854,18 +1031,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, - "y": 37 + "y": 56 }, "hiddenSeries": false, "id": 24, @@ -886,17 +1058,10 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.10", + "pluginVersion": "8.3.5", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "chain": { - "selected": true, - "text": "ETH", - "value": "ETH" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -911,9 +1076,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "JSON RPC upstream conn seconds 75 percentile", "tooltip": { "shared": true, @@ -922,9 +1085,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -932,25 +1093,20 @@ { "$$hashKey": "object:124", "format": "s", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true }, { "$$hashKey": "object:125", "format": "short", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -958,18 +1114,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, - "y": 46 + "y": 65 }, "hiddenSeries": false, "id": 41, @@ -990,17 +1141,10 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.10", + "pluginVersion": "8.3.5", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "chain": { - "selected": true, - "text": "ETH", - "value": "ETH" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -1015,9 +1159,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "JSON RPC upstream conn seconds 90 percentile", "tooltip": { "shared": true, @@ -1026,9 +1168,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1036,25 +1176,20 @@ { "$$hashKey": "object:124", "format": "s", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true }, { "$$hashKey": "object:125", "format": "short", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -1062,18 +1197,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, - "y": 46 + "y": 65 }, "hiddenSeries": false, "id": 98, @@ -1094,17 +1224,10 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.10", + "pluginVersion": "8.3.5", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "chain": { - "selected": true, - "text": "ETH", - "value": "ETH" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -1119,9 +1242,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "JSON RPC upstream conn seconds 99 percentile", "tooltip": { "shared": true, @@ -1130,9 +1251,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1140,25 +1259,20 @@ { "$$hashKey": "object:124", "format": "s", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true }, { "$$hashKey": "object:125", "format": "short", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -1166,7 +1280,6 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, "fieldConfig": { "defaults": { "unit": "reqps" @@ -1179,7 +1292,7 @@ "h": 10, "w": 24, "x": 0, - "y": 55 + "y": 74 }, "hiddenSeries": false, "id": 132, @@ -1200,17 +1313,10 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.10", + "pluginVersion": "8.3.5", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "chain": { - "selected": true, - "text": "ETH", - "value": "ETH" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -1225,9 +1331,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "WS Upstreams Errors", "tooltip": { "shared": true, @@ -1236,9 +1340,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1246,25 +1348,20 @@ { "$$hashKey": "object:124", "format": "reqps", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true }, { "$$hashKey": "object:125", "format": "short", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": false } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -1272,18 +1369,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, - "y": 65 + "y": 84 }, "hiddenSeries": false, "id": 134, @@ -1304,7 +1396,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.10", + "pluginVersion": "8.3.5", "pointradius": 2, "points": false, "renderer": "flot", @@ -1322,9 +1414,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "WS upstream conn seconds 50 percentile", "tooltip": { "shared": true, @@ -1333,9 +1423,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1343,25 +1431,20 @@ { "$$hashKey": "object:124", "format": "s", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true }, { "$$hashKey": "object:125", "format": "short", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -1369,18 +1452,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, - "y": 65 + "y": 84 }, "hiddenSeries": false, "id": 135, @@ -1401,7 +1479,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.10", + "pluginVersion": "8.3.5", "pointradius": 2, "points": false, "renderer": "flot", @@ -1419,9 +1497,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "WS upstream conn seconds 75 percentile", "tooltip": { "shared": true, @@ -1430,9 +1506,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1440,25 +1514,20 @@ { "$$hashKey": "object:124", "format": "s", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true }, { "$$hashKey": "object:125", "format": "short", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -1466,18 +1535,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 24, "x": 0, - "y": 74 + "y": 93 }, "hiddenSeries": false, "id": 71, @@ -1499,17 +1563,10 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.10", + "pluginVersion": "8.3.5", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "chain": { - "selected": true, - "text": "ETH", - "value": "ETH" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -1524,9 +1581,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Upstreams Lag", "tooltip": { "shared": true, @@ -1535,9 +1590,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1545,30 +1598,285 @@ { "$$hashKey": "object:124", "format": "string", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true }, { "$$hashKey": "object:125", "format": "short", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 102 + }, + "id": 145, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lQ3xLXW7k" + }, + "exemplar": false, + "expr": "sum by (method) (dshackle_request_jsonrpc_err_total{chain=~\"$chain\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{ upstream }}", + "refId": "A" + } + ], + "title": "Error Requests Total Count", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "chain": true, + "instance": true, + "job": true + }, + "indexByName": {}, + "renameByName": { + "Value": "Number of Error requests", + "Value #A": "Blocks Behind", + "upstream": "Upstream" + } + } + } + ], + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 12, + "y": 102 + }, + "id": 147, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lQ3xLXW7k" + }, + "exemplar": false, + "expr": "sum by (method) (dshackle_request_jsonrpc_fail_total{chain=~\"$chain\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{ upstream }}", + "refId": "A" + } + ], + "title": "Failed Requests Total Count", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "chain": true, + "instance": true, + "job": true + }, + "indexByName": {}, + "renameByName": { + "Value": "Number of Error requests", + "Value #A": "Blocks Behind", + "upstream": "Upstream" + } + } + } + ], + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 117 + }, + "id": 143, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Number of OK requests" + } + ] + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lQ3xLXW7k" + }, + "exemplar": false, + "expr": "sum by (method) (dshackle_request_jsonrpc_request_total{chain=~\"$chain\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{ upstream }}", + "refId": "A" + } + ], + "title": "OK Requests Total Count", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "chain": true, + "instance": true, + "job": true + }, + "indexByName": {}, + "renameByName": { + "Value": "Number of OK requests", + "Value #A": "Blocks Behind", + "upstream": "Upstream" + } + } + } + ], + "type": "table" } ], "refresh": "10s", - "schemaVersion": 27, + "schemaVersion": 34, "style": "dark", "tags": [], "templating": { @@ -1579,11 +1887,8 @@ "text": "Prometheus", "value": "Prometheus" }, - "description": null, - "error": null, "hide": 0, "includeAll": false, - "label": null, "multi": false, "name": "datasource", "options": [], @@ -1595,51 +1900,33 @@ "type": "datasource" }, { - "allValue": null, "current": { "selected": false, "text": "dshackle-01", "value": "dshackle-01" }, - "datasource": null, "definition": "label_values(dshackle_request_jsonrpc_err_total{},instance)", - "description": null, - "error": null, "hide": 0, "includeAll": false, - "label": null, "multi": false, "name": "instance", - "options": [ - { - "selected": true, - "text": "dshackle-01", - "value": "dshackle-01" - }, - { - "selected": false, - "text": "dshackle-02", - "value": "dshackle-02" - } - ], + "options": [], "query": { "query": "label_values(dshackle_request_jsonrpc_err_total{},instance)", "refId": "StandardVariableQuery" }, - "refresh": 0, + "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", - "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { - "allValue": null, "current": { - "selected": false, + "selected": true, "text": [ "ETH" ], @@ -1647,13 +1934,9 @@ "ETH" ] }, - "datasource": null, "definition": "label_values(dshackle_upstreams_availability,chain)", - "description": null, - "error": null, "hide": 0, "includeAll": true, - "label": null, "multi": true, "name": "chain", "options": [], @@ -1666,7 +1949,6 @@ "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", - "tags": [], "tagsQuery": "", "type": "query", "useTags": false @@ -1679,7 +1961,8 @@ }, "timepicker": {}, "timezone": "", - "title": "Dshackle Stats", - "uid": "qkG2pqWnz", - "version": 5 + "title": "Dshackle Stats 2", + "uid": "qkG2pqWnp", + "version": 1, + "weekStart": "" }