From 05feb240cd7b89b23b15f6bd7c9bc0aa5fc0af20 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 17 Jun 2020 11:48:04 +0200 Subject: [PATCH 1/4] Allow customization of benchmark runs via CLI --- benchmark/benchmark.py | 45 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index c44d88a..e348b92 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -3,31 +3,59 @@ @author: gocht ''' +import argparse import sys import benchmark_helper import pickle +# Available tests tests = ["bm_baseline.py", "bm_simplefunc.py"] +# Available instrumenters instrumenters = ["profile", "trace", "dummy", "None"] if sys.version_info.major >= 3: instrumenters.extend(["cProfile", "cTrace"]) -# How many times the instrumented code is run during 1 test run +# Default values for: How many times the instrumented code is run during 1 test run reps_x = { "bm_baseline.py": ["1000000", "2000000", "3000000", "4000000", "5000000"], "bm_simplefunc.py": ["100000", "200000", "300000", "400000", "500000"], } -# How many times a test invocation is repeated (number of timings per test instance) -test_repetitions = 51 -bench = benchmark_helper.BenchmarkEnv(repetitions=test_repetitions) + +def str_to_int(s): + try: + return int(s) + except ValueError: + return int(float(s)) + + +parser = argparse.ArgumentParser(description='Benchmark the instrumenters.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--test', '-t', metavar='TEST', nargs='+', default=tests, + choices=tests, help='Which test(s) to run') +parser.add_argument('--repetitions', '-r', default=51, type=str_to_int, + help='How many times a test invocation is repeated (number of timings per test instance)') +parser.add_argument('--loop-count', '-l', type=str_to_int, nargs='+', + help=('How many times the instrumented code is run during 1 test run. ' + 'Can be repeated and will create 1 test instance per argument')) +parser.add_argument('--instrumenter', '-i', metavar='INST', nargs='+', default=instrumenters, + choices=instrumenters, help='The instrumenter(s) to use') +parser.add_argument('--output', '-o', default='results.pkl', help='Output file for the results') +parser.add_argument('--dry-run', action='store_true', help='Print parsed arguments and exit') +args = parser.parse_args() + +if args.dry_run: + print(args) + sys.exit(0) + +bench = benchmark_helper.BenchmarkEnv(repetitions=args.repetitions) results = {} -for test in tests: +for test in args.test: results[test] = {} - for instrumenter in instrumenters: + for instrumenter in args.instrumenter: results[test][instrumenter] = {} if instrumenter == "None": @@ -40,12 +68,13 @@ print("#########") print("{}: {}".format(test, scorep_settings)) print("#########") - for reps in reps_x[test]: + loop_counts = args.loop_count if args.loop_count else reps_x[test] + for reps in loop_counts: times = bench.call(test, [reps], enable_scorep, scorep_settings=scorep_settings) print("{:<8}: {}".format(reps, times)) results[test][instrumenter][reps] = times -with open("results.pkl", "wb") as f: +with open(args.output, "wb") as f: pickle.dump(results, f) From 6bfee8bf10e8a049feed31a6350f9e469af40ae3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 17 Jun 2020 11:48:53 +0200 Subject: [PATCH 2/4] Make benchmark.py executable --- benchmark/benchmark.py | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 benchmark/benchmark.py diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py old mode 100644 new mode 100755 index e348b92..6a2fdb4 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python ''' Created on 04.10.2019 From 06e9478c66dc1738dc9fa8921dc91833ff2e96af Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 18 Jun 2020 17:14:54 +0200 Subject: [PATCH 3/4] Fix: call needs string arguments not int (from CLI) --- benchmark/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 6a2fdb4..59cd980 100755 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -71,7 +71,7 @@ def str_to_int(s): print("#########") loop_counts = args.loop_count if args.loop_count else reps_x[test] for reps in loop_counts: - times = bench.call(test, [reps], + times = bench.call(test, [str(reps)], enable_scorep, scorep_settings=scorep_settings) print("{:<8}: {}".format(reps, times)) From f9dc96b8da310d026d7d56e50b2d57a68c1e357e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 5 Oct 2020 09:14:52 +0200 Subject: [PATCH 4/4] Convert always to int over float --- benchmark/benchmark.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 59cd980..5667e36 100755 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -25,10 +25,7 @@ def str_to_int(s): - try: - return int(s) - except ValueError: - return int(float(s)) + return int(float(s)) parser = argparse.ArgumentParser(description='Benchmark the instrumenters.',