diff --git a/metrics/grafana/client_java_summary.json b/metrics/grafana/client_java_summary.json index 160c013f0b4..2146682760a 100644 --- a/metrics/grafana/client_java_summary.json +++ b/metrics/grafana/client_java_summary.json @@ -49,6 +49,20 @@ "iteration": 1636696687563, "links": [], "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 23, + "panels": [], + "title": "Service Quality", + "type": "row" + }, { "aliasColors": {}, "bars": false, @@ -65,7 +79,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "hiddenSeries": false, "id": 2, @@ -99,7 +113,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(client_java_raw_requests_latency_count{instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(client_java_raw_requests_latency_count{instance=~\"$instance\"}[$__interval])) by (type)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -127,6 +141,7 @@ }, "yaxes": [ { + "$$hashKey": "object:2785", "format": "short", "label": null, "logBase": 1, @@ -135,6 +150,7 @@ "show": true }, { + "$$hashKey": "object:2786", "format": "short", "label": null, "logBase": 1, @@ -164,7 +180,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 1 }, "hiddenSeries": false, "id": 8, @@ -197,8 +213,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(client_java_raw_requests_failure_total[1m])) by (type)", + "exemplar": true, + "expr": "sum(increase(client_java_raw_requests_failure_total[$__interval])) by (type)", "format": "time_series", + "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "refId": "A" @@ -208,7 +226,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Client Failures", + "title": "Request Failure OPS", "tooltip": { "shared": true, "sort": 0, @@ -224,14 +242,16 @@ }, "yaxes": [ { + "$$hashKey": "object:2863", "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { + "$$hashKey": "object:2864", "format": "short", "label": null, "logBase": 1, @@ -261,10 +281,10 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 9 }, "hiddenSeries": false, - "id": 15, + "id": 4, "legend": { "alignAsTable": true, "avg": false, @@ -295,19 +315,37 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(client_java_raw_requests_latency_sum{instance=~\"$instance\"}[1m])) by (type) / sum(rate(client_java_raw_requests_latency_count{instance=~\"$instance\"}[1m])) by (type)", + "expr": "histogram_quantile(0.99, sum(rate(client_java_raw_requests_latency_bucket{instance=~\"$instance\"}[$__interval])) by (le, type))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(1, sum(rate(client_java_raw_requests_latency_bucket{instance=~\"$instance\"}[$__interval])) by (le, type))", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [ + { + "$$hashKey": "object:151", + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.85, + "yaxis": "left" } ], - "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Raw Request Avg Latency", + "title": "Request Duration 99th", "tooltip": { "shared": true, "sort": 0, @@ -323,20 +361,22 @@ }, "yaxes": [ { + "$$hashKey": "object:3019", "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true }, { + "$$hashKey": "object:3020", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -360,10 +400,10 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 9 }, "hiddenSeries": false, - "id": 4, + "id": 15, "legend": { "alignAsTable": true, "avg": false, @@ -394,19 +434,30 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(client_java_raw_requests_latency_bucket{instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "sum(rate(client_java_raw_requests_latency_sum{instance=~\"$instance\"}[$__interval])) by (type) / sum(rate(client_java_raw_requests_latency_count{instance=~\"$instance\"}[$__interval])) by (type)", "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "refId": "A" } ], - "thresholds": [], + "thresholds": [ + { + "$$hashKey": "object:186", + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.85, + "yaxis": "left" + } + ], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Raw Request 99th Latency", + "title": "Request Duration Avg", "tooltip": { "shared": true, "sort": 0, @@ -422,20 +473,22 @@ }, "yaxes": [ { + "$$hashKey": "object:2941", "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true }, { + "$$hashKey": "object:2942", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -443,14 +496,31 @@ "alignLevel": null } }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 25, + "panels": [], + "title": "Causes of TiKV Request Latency", + "type": "row" + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "", "fieldConfig": { - "defaults": {}, + "defaults": { + "unit": "s" + }, "overrides": [] }, "fill": 1, @@ -459,14 +529,15 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 18 }, "hiddenSeries": false, - "id": 7, + "id": 10, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": false, "max": true, "min": false, "rightSide": true, @@ -476,7 +547,6 @@ }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true @@ -493,19 +563,31 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(client_java_grpc_raw_requests_latency_sum{instance=~\"$instance\"}[1m])) by (type) / sum(rate(client_java_grpc_raw_requests_latency_count{instance=~\"$instance\"}[1m])) by (type)", + "expr": "histogram_quantile(0.99, sum(rate(client_java_grpc_raw_requests_latency_bucket{instance=~\"$instance\"}[$__interval])) by (le, type))", "format": "time_series", + "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{type}}", + "legendFormat": "{{ type }}", + "queryType": "randomWalk", "refId": "A" } ], - "thresholds": [], + "thresholds": [ + { + "$$hashKey": "object:413", + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.85, + "yaxis": "left" + } + ], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "gRPC Raw Requests Avg Latency", + "title": "Request Duration + Backoff (Single Run) 99th", "tooltip": { "shared": true, "sort": 0, @@ -521,20 +603,22 @@ }, "yaxes": [ { + "$$hashKey": "object:3179", "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true }, { + "$$hashKey": "object:3180", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -548,11 +632,8 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", "fieldConfig": { - "defaults": { - "unit": "s" - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -561,15 +642,14 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 18 }, "hiddenSeries": false, - "id": 10, + "id": 7, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, "max": true, "min": false, "rightSide": true, @@ -579,6 +659,7 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true @@ -595,21 +676,29 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(client_java_grpc_raw_requests_latency_bucket{instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "sum(rate(client_java_grpc_raw_requests_latency_sum{instance=~\"$instance\"}[$__interval])) by (type) / sum(rate(client_java_grpc_raw_requests_latency_count{instance=~\"$instance\"}[$__interval])) by (type)", "format": "time_series", - "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{ type }}", - "queryType": "randomWalk", + "legendFormat": "{{type}}", "refId": "A" } ], - "thresholds": [], + "thresholds": [ + { + "$$hashKey": "object:396", + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.85, + "yaxis": "left" + } + ], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "gRPC Raw Requests 99th latency", + "title": "Request Duration + Backoff (Single Run) Avg", "tooltip": { "shared": true, "sort": 0, @@ -625,20 +714,22 @@ }, "yaxes": [ { + "$$hashKey": "object:3101", "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true }, { + "$$hashKey": "object:3102", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -662,10 +753,10 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 26 }, "hiddenSeries": false, - "id": 12, + "id": 13, "legend": { "alignAsTable": true, "avg": false, @@ -695,18 +786,28 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(client_java_grpc_single_requests_latency_sum{instance=~\"$instance\"}[1m])) by (type) / sum(rate(client_java_grpc_single_requests_latency_count{instance=~\"$instance\"}[1m])) by (type)", + "expr": "histogram_quantile(0.99,sum(rate(client_java_grpc_single_requests_latency_bucket{instance=~\"$instance\", type!=\"pdpb.PD/GetRegion\", type!=\"pdpb.PD/GetStore\"}[$__interval])) by (le, type))", "interval": "", "legendFormat": "{{ type }}", "queryType": "randomWalk", "refId": "A" } ], - "thresholds": [], + "thresholds": [ + { + "$$hashKey": "object:427", + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.85, + "yaxis": "left" + } + ], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "gRPC Single Request Avg Latency", + "title": "TiKV Request Duration (Single Run) 99th", "tooltip": { "shared": true, "sort": 0, @@ -722,20 +823,22 @@ }, "yaxes": [ { + "$$hashKey": "object:3335", "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true }, { + "$$hashKey": "object:3336", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -759,10 +862,10 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 26 }, "hiddenSeries": false, - "id": 13, + "id": 12, "legend": { "alignAsTable": true, "avg": false, @@ -792,18 +895,28 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(client_java_grpc_single_requests_latency_bucket{instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "sum(rate(client_java_grpc_single_requests_latency_sum{instance=~\"$instance\", type!=\"pdpb.PD/GetRegion\", type!=\"pdpb.PD/GetStore\"}[$__interval])) by (type) / sum(rate(client_java_grpc_single_requests_latency_count{instance=~\"$instance\", type!=\"pdpb.PD/GetRegion\", type!=\"pdpb.PD/GetStore\"}[$__interval])) by (type)", "interval": "", "legendFormat": "{{ type }}", "queryType": "randomWalk", "refId": "A" } ], - "thresholds": [], + "thresholds": [ + { + "$$hashKey": "object:382", + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.85, + "yaxis": "left" + } + ], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "gRPC Single Request 99th Latency", + "title": "TiKV Request Duration (Single Run) Avg", "tooltip": { "shared": true, "sort": 0, @@ -819,20 +932,22 @@ }, "yaxes": [ { + "$$hashKey": "object:3257", "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true }, { + "$$hashKey": "object:3258", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -856,10 +971,10 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 34 }, "hiddenSeries": false, - "id": 14, + "id": 21, "legend": { "alignAsTable": true, "avg": false, @@ -889,26 +1004,153 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(client_java_get_region_by_requests_latency_sum{instance=~\"$instance\"}[1m])) / sum(rate(client_java_get_region_by_requests_latency_count{instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(client_java_seek_leader_store_duration_sum{instance=~\"$instance\"}[$__interval])) by (le) / sum(rate(client_java_seek_leader_store_duration_count{instance=~\"$instance\"}[$__interval])) by (le)", "interval": "", - "legendFormat": "avg", + "legendFormat": "seek-leader-store-avg", "queryType": "randomWalk", "refId": "A" }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(client_java_get_region_by_requests_latency_bucket{instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99,sum(rate(client_java_seek_leader_store_duration_bucket{instance=~\"$instance\"}[$__interval])) by (le, type))", "hide": false, "interval": "", - "legendFormat": "99th", - "refId": "B" + "legendFormat": "seek-leader-store-99", + "refId": "C" + } + ], + "thresholds": [ + { + "$$hashKey": "object:441", + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.3, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Backoff: Seek Leader Store Duration (Single Run)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:252", + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:253", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(client_java_seek_proxy_store_duration_sum{instance=~\"$instance\"}[$__interval])) by (le) / sum(rate(client_java_seek_proxy_store_duration_count{instance=~\"$instance\"}[$__interval])) by (le)", + "interval": "", + "legendFormat": "seek-proxy-store-avg", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99,sum(rate(client_java_seek_proxy_store_duration_bucket{instance=~\"$instance\"}[$__interval])) by (le, type))", + "hide": false, + "interval": "", + "legendFormat": "seek-proxy-store-99", + "refId": "C" + } + ], + "thresholds": [ + { + "$$hashKey": "object:441", + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.2, + "yaxis": "left" } ], - "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Get Region Latency", + "title": "Backoff: Seek Proxy Store Duration (Single Run)", "tooltip": { "shared": true, "sort": 0, @@ -924,14 +1166,1090 @@ }, "yaxes": [ { + "$$hashKey": "object:252", "format": "s", "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:253", + "format": "short", + "label": null, "logBase": 1, "max": null, "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(client_java_backoff_duration_sum{instance=~\"$instance\", type!=\"BoPDRPC\"}[$__interval])) by (le, type)", + "hide": false, + "interval": "", + "legendFormat": "{{type}}-total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Backoff Duration (Single Run + Inter-Run) Total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:221", + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:222", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(client_java_backoff_duration_count{instance=~\"$instance\"}[$__interval])) by (le, type)", + "interval": "", + "legendFormat": "{{type}}-count", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Backoff OPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:82", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:83", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "hiddenSeries": false, + "id": 46, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(client_java_backoff_duration_sum{instance=~\"$instance\", type!=\"BoPDRPC\"}[$__interval])) by (le, type) / sum(rate(client_java_backoff_duration_count{instance=~\"$instance\", type!=\"BoPDRPC\"}[$__interval])) by (le, type)", + "interval": "", + "legendFormat": "{{type}}-avg", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Backoff Duration (Single Run + Inter-Run) Avg", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:221", + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:222", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 40, + "panels": [], + "title": "Causes of PD Request Latency", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(client_java_get_region_by_requests_latency_sum{instance=~\"$instance\"}[$__interval])) / sum(rate(client_java_get_region_by_requests_latency_count{instance=~\"$instance\"}[$__interval]))", + "interval": "", + "legendFormat": "avg", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(client_java_get_region_by_requests_latency_bucket{instance=~\"$instance\"}[$__interval])) by (le))", + "hide": false, + "interval": "", + "legendFormat": "99th", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Get Region from Cache/PD Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3413", + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:3414", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "hiddenSeries": false, + "id": 37, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "1 - sum(rate(client_java_grpc_single_requests_latency_count{instance=~\"$instance\", type=\"pdpb.PD/GetRegion\"}[$__interval])) / sum(rate(client_java_get_region_by_requests_latency_count{instance=~\"$instance\"}[$__interval]))", + "interval": "", + "legendFormat": "hit ratio", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [ + { + "$$hashKey": "object:640", + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Region Cache Hit Ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:277", + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:278", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 67 + }, + "hiddenSeries": false, + "id": 38, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99,sum(rate(client_java_grpc_single_requests_latency_bucket{instance=~\"$instance\", type=\"pdpb.PD/GetStore\"}[$__interval])) by (le, type))", + "interval": "", + "legendFormat": "{{ type }}-99th", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99,sum(rate(client_java_grpc_single_requests_latency_bucket{instance=~\"$instance\", type=\"pdpb.PD/GetRegion\"}[$__interval])) by (le, type))", + "hide": false, + "interval": "", + "legendFormat": "{{ type }}-99th", + "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99,sum(rate(client_java_grpc_single_requests_latency_bucket{instance=~\"$instance\", type=\"pdpb.PD/GetMembers\"}[$__interval])) by (le, type))", + "hide": false, + "interval": "", + "legendFormat": "{{ type }}-99th", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "PD Request Duration 99th", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3335", + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:3336", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 67 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(client_java_grpc_single_requests_latency_count{instance=~\"$instance\", type=\"pdpb.PD/GetRegion\"}[$__interval])) by (type)", + "hide": false, + "interval": "", + "legendFormat": "{{type}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(client_java_grpc_single_requests_latency_count{instance=~\"$instance\", type=\"pdpb.PD/GetStore\"}[$__interval])) by (type)", + "hide": false, + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(client_java_grpc_single_requests_latency_count{instance=~\"$instance\", type=\"pdpb.PD/GetMembers\"}[$__interval])) by (type)", + "hide": false, + "interval": "", + "legendFormat": "{{type}}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "PD Request OPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2785", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2786", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "hiddenSeries": false, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(client_java_backoff_duration_sum{instance=~\"$instance\", type=\"BoPDRPC\"}[$__interval])) by (le, type) / sum(rate(client_java_backoff_duration_count{instance=~\"$instance\", type=\"BoPDRPC\"}[$__interval])) by (le, type)", + "interval": "", + "legendFormat": "{{type}}-avg", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(client_java_backoff_duration_sum{instance=~\"$instance\", type=\"BoPDRPC\"}[$__interval])) by (le, type)", + "hide": false, + "interval": "", + "legendFormat": "{{type}}-sum", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Backoff Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:221", + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:222", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 75 + }, + "hiddenSeries": false, + "id": 45, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(client_java_backoff_duration_count{instance=~\"$instance\", type=\"BoPDRPC\"}[$__interval])) by (le, type)", + "interval": "", + "legendFormat": "{{type}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Backoff OPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:82", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:83", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(client_java_pd_seek_leader_duration_bucket{instance=\"$instance\"}[5s])))", + "interval": "", + "legendFormat": "seek-pd-leader-99", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(client_java_pd_seek_proxy_duration_bucket{instance=\"$instance\"}[5s])))", + "hide": false, + "interval": "", + "legendFormat": "seek-pd-proxy-99", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "PD seek leader/proxy duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:226", + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, "show": true }, { + "$$hashKey": "object:227", "format": "short", "label": null, "logBase": 1, @@ -1032,6 +2350,6 @@ }, "timezone": "browser", "title": "Client-Java-Summary", - "uid": "000000911", - "version": 15 + "uid": "000000912", + "version": 16 } \ No newline at end of file