From 87afa5dc2209df1d594342e83e2e6565a8d09690 Mon Sep 17 00:00:00 2001 From: Jian Zhang Date: Wed, 16 Feb 2022 18:09:31 +0800 Subject: [PATCH 1/4] [to #421] add a python script to analyze slow log stats Signed-off-by: Jian Zhang --- .github/license-checker.yml | 1 - dev/README.md | 8 ++-- {scripts => dev}/proto.sh | 0 pom.xml | 2 +- scripts/rstats.py | 76 +++++++++++++++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 7 deletions(-) rename {scripts => dev}/proto.sh (100%) create mode 100755 scripts/rstats.py diff --git a/.github/license-checker.yml b/.github/license-checker.yml index 4389aa2ff95..f0087cc8eb8 100644 --- a/.github/license-checker.yml +++ b/.github/license-checker.yml @@ -7,7 +7,6 @@ header: - 'dev/' - 'docs/' - 'metrics/' - - 'scripts/' - 'LICENSE' - 'Makefile' - 'pom.xml' diff --git a/dev/README.md b/dev/README.md index b0eccd6057b..e924397398f 100644 --- a/dev/README.md +++ b/dev/README.md @@ -1,8 +1,6 @@ -# TiSpark Dev Tools Guide +# TiKV Java Client Dev Tools -## Formatting - -### Java Format +## Code Formatting TiKV Java Client formats its code using [Google-Java-Format Maven Plugin](https://github.com/coveooss/fmt-maven-plugin) which follows Google's code styleguide. It is also checked on CI before build. @@ -18,4 +16,4 @@ TiKV Java Client formats its code using [Google-Java-Format Maven Plugin](https: ```shell script ./dev/javafmt - ``` \ No newline at end of file + ``` diff --git a/scripts/proto.sh b/dev/proto.sh similarity index 100% rename from scripts/proto.sh rename to dev/proto.sh diff --git a/pom.xml b/pom.xml index a452e2891fd..4cab44b09ce 100644 --- a/pom.xml +++ b/pom.xml @@ -312,7 +312,7 @@ clone proto files - ${basedir}/scripts/proto.sh + ${basedir}/dev/proto.sh validate diff --git a/scripts/rstats.py b/scripts/rstats.py new file mode 100755 index 00000000000..6aeabb83436 --- /dev/null +++ b/scripts/rstats.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +#!coding:utf-8 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import re +import json +import argparse + +def main(): + pattern = r'.*SlowLog.*' + slowstr = "SlowLog:" + grpc_pattern = "gRPC tikvpb.Tikv" + backoff_pattern = "backoff " + + args = parse_args() + items = [] + with open(args.slowlog, encoding = 'utf-8') as f: + for line in f.readlines(): + matched = re.match(pattern, line, re.M|re.I) + if matched is not None: + log = json.loads(line[(line.index(slowstr) + len(slowstr)):]) + item = { + 'req': log['func'], + 'start': log['start'], + 'tot_lat': int(log['duration'][:len(log['duration'])-2]), + 'tot_grpc': 0, + 'tot_bo': 0, + } + items.append(item) + for span in log['spans']: + if grpc_pattern in span['name'] and span['duration'] != 'N/A': + item['tot_grpc'] += int(span['duration'][:len(span['duration'])-2]) + elif backoff_pattern in span['name'] and span['duration'] != 'N/A': + item['tot_bo'] += int(span['duration'][:len(span['duration'])-2]) + + if args.order == "total": + items = sorted(items, key=lambda d: d['tot_lat'], reverse=True) + elif args.order == "grpc": + items = sorted(items, key=lambda d: d['tot_grpc'], reverse=True) + elif args.order == "backoff": + items = sorted(items, key=lambda d: d['tot_bo'], reverse=True) + else: + print("unsupported order option, use default value: total") + items = sorted(items, key=lambda d: d['tot_lat'], reverse=True) + + fmtStr = "{:<12} {:<14} {:<14} {:<20} {:<20}" + print(fmtStr.format("Request", "Start", "Total Lat(ms)", "Total gRPC Lat(ms)", "Total Backoff Lat(ms)")) + for item in items: + print(fmtStr.format(item['req'], item['start'], item['tot_lat'], item['tot_grpc'], item['tot_bo'])) + +def parse_args(): + parser = argparse.ArgumentParser(description="rstats: A TiKV Java Client Request Stats Analyzer") + parser.add_argument("--order", dest="order", default="total", help="order the output, default: total. accepted value: total, grpc, backoff") + parser.add_argument("slowlog", help="slow log file") + return parser.parse_args() + +if __name__ == '__main__': + main() + From 7217fc1b6b1426f25d5bd9eb489dc7d981072033 Mon Sep 17 00:00:00 2001 From: Jian Zhang Date: Wed, 16 Feb 2022 18:15:41 +0800 Subject: [PATCH 2/4] fix license Signed-off-by: Jian Zhang --- scripts/rstats.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/scripts/rstats.py b/scripts/rstats.py index 6aeabb83436..e9ab5358931 100755 --- a/scripts/rstats.py +++ b/scripts/rstats.py @@ -1,22 +1,19 @@ #!/usr/bin/env python3 #!coding:utf-8 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Copyright 2022 TiKV Project Authors. # -# http://www.apache.org/licenses/LICENSE-2.0 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # import re From 41ba8fb84cfeea63d516eb7fd28fad070e548af4 Mon Sep 17 00:00:00 2001 From: Jian Zhang Date: Mon, 21 Feb 2022 21:13:38 +0800 Subject: [PATCH 3/4] address comment Signed-off-by: Jian Zhang --- scripts/rstats.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/rstats.py b/scripts/rstats.py index e9ab5358931..ec1974a1342 100755 --- a/scripts/rstats.py +++ b/scripts/rstats.py @@ -36,16 +36,16 @@ def main(): item = { 'req': log['func'], 'start': log['start'], - 'tot_lat': int(log['duration'][:len(log['duration'])-2]), + 'tot_lat': latency_ms(log), 'tot_grpc': 0, 'tot_bo': 0, } items.append(item) for span in log['spans']: if grpc_pattern in span['name'] and span['duration'] != 'N/A': - item['tot_grpc'] += int(span['duration'][:len(span['duration'])-2]) + item['tot_grpc'] += latency_ms(span) elif backoff_pattern in span['name'] and span['duration'] != 'N/A': - item['tot_bo'] += int(span['duration'][:len(span['duration'])-2]) + item['tot_bo'] += latency_ms(span) if args.order == "total": items = sorted(items, key=lambda d: d['tot_lat'], reverse=True) @@ -62,9 +62,13 @@ def main(): for item in items: print(fmtStr.format(item['req'], item['start'], item['tot_lat'], item['tot_grpc'], item['tot_bo'])) +def latency_ms(span): + return int(span['duration'][:len(span['duration'])-2]) + + def parse_args(): parser = argparse.ArgumentParser(description="rstats: A TiKV Java Client Request Stats Analyzer") - parser.add_argument("--order", dest="order", default="total", help="order the output, default: total. accepted value: total, grpc, backoff") + parser.add_argument("-o", dest="order", default="total", help="order the output, default: total. accepted value: total, grpc, backoff") parser.add_argument("slowlog", help="slow log file") return parser.parse_args() From 4f1785c61466a1f19aa2ce0ced65466afa59e36a Mon Sep 17 00:00:00 2001 From: Jian Zhang Date: Mon, 21 Feb 2022 21:16:53 +0800 Subject: [PATCH 4/4] address comment Signed-off-by: Jian Zhang --- scripts/rstats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/rstats.py b/scripts/rstats.py index ec1974a1342..4c483fb8ab6 100755 --- a/scripts/rstats.py +++ b/scripts/rstats.py @@ -21,7 +21,7 @@ import argparse def main(): - pattern = r'.*SlowLog.*' + pattern = r'.*SlowLog:.*' slowstr = "SlowLog:" grpc_pattern = "gRPC tikvpb.Tikv" backoff_pattern = "backoff "