diff --git a/.github/license-checker.yml b/.github/license-checker.yml index 4389aa2ff95..f0087cc8eb8 100644 --- a/.github/license-checker.yml +++ b/.github/license-checker.yml @@ -7,7 +7,6 @@ header: - 'dev/' - 'docs/' - 'metrics/' - - 'scripts/' - 'LICENSE' - 'Makefile' - 'pom.xml' diff --git a/dev/README.md b/dev/README.md index b0eccd6057b..e924397398f 100644 --- a/dev/README.md +++ b/dev/README.md @@ -1,8 +1,6 @@ -# TiSpark Dev Tools Guide +# TiKV Java Client Dev Tools -## Formatting - -### Java Format +## Code Formatting TiKV Java Client formats its code using [Google-Java-Format Maven Plugin](https://github.com/coveooss/fmt-maven-plugin) which follows Google's code styleguide. It is also checked on CI before build. @@ -18,4 +16,4 @@ TiKV Java Client formats its code using [Google-Java-Format Maven Plugin](https: ```shell script ./dev/javafmt - ``` \ No newline at end of file + ``` diff --git a/scripts/proto.sh b/dev/proto.sh similarity index 100% rename from scripts/proto.sh rename to dev/proto.sh diff --git a/pom.xml b/pom.xml index 23d6e934ce8..71930d3efc0 100644 --- a/pom.xml +++ b/pom.xml @@ -312,7 +312,7 @@ clone proto files - ${basedir}/scripts/proto.sh + ${basedir}/dev/proto.sh validate diff --git a/scripts/rstats.py b/scripts/rstats.py new file mode 100755 index 00000000000..4c483fb8ab6 --- /dev/null +++ b/scripts/rstats.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +#!coding:utf-8 + +# Copyright 2022 TiKV Project Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +import json +import argparse + +def main(): + pattern = r'.*SlowLog:.*' + slowstr = "SlowLog:" + grpc_pattern = "gRPC tikvpb.Tikv" + backoff_pattern = "backoff " + + args = parse_args() + items = [] + with open(args.slowlog, encoding = 'utf-8') as f: + for line in f.readlines(): + matched = re.match(pattern, line, re.M|re.I) + if matched is not None: + log = json.loads(line[(line.index(slowstr) + len(slowstr)):]) + item = { + 'req': log['func'], + 'start': log['start'], + 'tot_lat': latency_ms(log), + 'tot_grpc': 0, + 'tot_bo': 0, + } + items.append(item) + for span in log['spans']: + if grpc_pattern in span['name'] and span['duration'] != 'N/A': + item['tot_grpc'] += latency_ms(span) + elif backoff_pattern in span['name'] and span['duration'] != 'N/A': + item['tot_bo'] += latency_ms(span) + + if args.order == "total": + items = sorted(items, key=lambda d: d['tot_lat'], reverse=True) + elif args.order == "grpc": + items = sorted(items, key=lambda d: d['tot_grpc'], reverse=True) + elif args.order == "backoff": + items = sorted(items, key=lambda d: d['tot_bo'], reverse=True) + else: + print("unsupported order option, use default value: total") + items = sorted(items, key=lambda d: d['tot_lat'], reverse=True) + + fmtStr = "{:<12} {:<14} {:<14} {:<20} {:<20}" + print(fmtStr.format("Request", "Start", "Total Lat(ms)", "Total gRPC Lat(ms)", "Total Backoff Lat(ms)")) + for item in items: + print(fmtStr.format(item['req'], item['start'], item['tot_lat'], item['tot_grpc'], item['tot_bo'])) + +def latency_ms(span): + return int(span['duration'][:len(span['duration'])-2]) + + +def parse_args(): + parser = argparse.ArgumentParser(description="rstats: A TiKV Java Client Request Stats Analyzer") + parser.add_argument("-o", dest="order", default="total", help="order the output, default: total. accepted value: total, grpc, backoff") + parser.add_argument("slowlog", help="slow log file") + return parser.parse_args() + +if __name__ == '__main__': + main() +