From d18055ac200889321d26fd2d5ac5bd483ae022bb Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 13 Feb 2020 12:54:13 -0800 Subject: [PATCH 1/5] feat: live perf tests --- .kokoro/continuous/storage_perf_bench.cfg | 1 + tests/perf/benchwrapper.py | 16 +++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 .kokoro/continuous/storage_perf_bench.cfg diff --git a/.kokoro/continuous/storage_perf_bench.cfg b/.kokoro/continuous/storage_perf_bench.cfg new file mode 100644 index 000000000..8f43917d9 --- /dev/null +++ b/.kokoro/continuous/storage_perf_bench.cfg @@ -0,0 +1 @@ +# Format: //devtools/kokoro/config/proto/build.proto \ No newline at end of file diff --git a/tests/perf/benchwrapper.py b/tests/perf/benchwrapper.py index 9ebb3f455..551bf1c05 100644 --- a/tests/perf/benchwrapper.py +++ b/tests/perf/benchwrapper.py @@ -12,10 +12,10 @@ parser = argparse.ArgumentParser() -if os.environ.get("STORAGE_EMULATOR_HOST") is None: - sys.exit( - "This benchmarking server only works when connected to an emulator. Please set STORAGE_EMULATOR_HOST." - ) +# if os.environ.get("STORAGE_EMULATOR_HOST") is None: +# sys.exit( +# "This benchmarking server only works when connected to an emulator. Please set STORAGE_EMULATOR_HOST." +# ) parser.add_argument("--port", help="The port to run on.") @@ -24,8 +24,14 @@ if args.port is None: sys.exit("Usage: python3 main.py --port 8081") -client = storage.Client.create_anonymous_client() +import os +print("files at tmpfs") +files = [f for f in os.listdir('/tmpfs/src/gfile/') if os.path.isfile(f)] +for f in files: + print(f) +#client = storage.Client.create_anonymous_client() +client = storage.Client() class StorageBenchWrapperServicer(storage_pb2_grpc.StorageBenchWrapperServicer): def Write(self, request, context): From df5cf92a36fabdfea30f7a7315708be6fd135720 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 13 Feb 2020 18:25:20 -0800 Subject: [PATCH 2/5] fix: add logging to ensure running --- tests/perf/benchwrapper.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/perf/benchwrapper.py b/tests/perf/benchwrapper.py index 551bf1c05..9da2dabcf 100644 --- a/tests/perf/benchwrapper.py +++ b/tests/perf/benchwrapper.py @@ -24,12 +24,6 @@ if args.port is None: sys.exit("Usage: python3 main.py --port 8081") -import os -print("files at tmpfs") -files = [f for f in os.listdir('/tmpfs/src/gfile/') if os.path.isfile(f)] -for f in files: - print(f) - #client = storage.Client.create_anonymous_client() client = storage.Client() @@ -39,9 +33,11 @@ def Write(self, request, context): return storage_pb2.EmptyResponse() def Read(self, request, context): + print("Request:", request.bucketName, request.objectName) bucket = client.bucket(request.bucketName) blob = storage.Blob(request.objectName, bucket) - blob.download_as_string() + obj = blob.download_as_string() + print("Object Length:", len(obj)) return storage_pb2.EmptyResponse() From 54eef5c6857f4cda7bb0167f7c437fcb095b251d Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 14 Feb 2020 13:21:05 -0800 Subject: [PATCH 3/5] fix: blacken --- tests/perf/benchwrapper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/perf/benchwrapper.py b/tests/perf/benchwrapper.py index 9da2dabcf..1143cac70 100644 --- a/tests/perf/benchwrapper.py +++ b/tests/perf/benchwrapper.py @@ -24,9 +24,10 @@ if args.port is None: sys.exit("Usage: python3 main.py --port 8081") -#client = storage.Client.create_anonymous_client() +# client = storage.Client.create_anonymous_client() client = storage.Client() + class StorageBenchWrapperServicer(storage_pb2_grpc.StorageBenchWrapperServicer): def Write(self, request, context): # TODO(deklerk): implement this From e62fd9209bd74ab199c86b3b13b9223b9357694b Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 14 Feb 2020 13:22:18 -0800 Subject: [PATCH 4/5] fix: remove debug content --- tests/perf/benchwrapper.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/perf/benchwrapper.py b/tests/perf/benchwrapper.py index 1143cac70..456ded874 100644 --- a/tests/perf/benchwrapper.py +++ b/tests/perf/benchwrapper.py @@ -34,11 +34,9 @@ def Write(self, request, context): return storage_pb2.EmptyResponse() def Read(self, request, context): - print("Request:", request.bucketName, request.objectName) bucket = client.bucket(request.bucketName) blob = storage.Blob(request.objectName, bucket) - obj = blob.download_as_string() - print("Object Length:", len(obj)) + blob.download_as_string() return storage_pb2.EmptyResponse() From f3b17a8a039f9cde2f28d7eff55ba268adcc86f8 Mon Sep 17 00:00:00 2001 From: Christopher Wilcox Date: Fri, 14 Feb 2020 17:26:51 -0800 Subject: [PATCH 5/5] fix: remove unused import --- tests/perf/benchwrapper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/perf/benchwrapper.py b/tests/perf/benchwrapper.py index 456ded874..c81d6bb20 100644 --- a/tests/perf/benchwrapper.py +++ b/tests/perf/benchwrapper.py @@ -2,7 +2,6 @@ import sys import time import grpc -import os from concurrent import futures import storage_pb2_grpc import storage_pb2