From 8fe6d08c80f9d7c854ffd78365bb282c80bcb80d Mon Sep 17 00:00:00 2001 From: John Wiggins Date: Tue, 2 Mar 2021 11:30:54 +0100 Subject: [PATCH] Add a page for comparing backend outputs --- enable/gcbench/bench.py | 4 +- enable/gcbench/publish.py | 114 ++++++++++++++++++++++++++++++-------- 2 files changed, 92 insertions(+), 26 deletions(-) diff --git a/enable/gcbench/bench.py b/enable/gcbench/bench.py index 675d5bffe..b87d38374 100644 --- a/enable/gcbench/bench.py +++ b/enable/gcbench/bench.py @@ -122,9 +122,9 @@ def gen_timings(gc, func): times = np.array(times) return { - "max": times.max() * 1000, - "min": times.min() * 1000, "mean": times.mean() * 1000, + "min": times.min() * 1000, + "max": times.max() * 1000, "std": times.std() * 1000, "count": len(times), } diff --git a/enable/gcbench/publish.py b/enable/gcbench/publish.py index 29262b68e..bad6ed95d 100644 --- a/enable/gcbench/publish.py +++ b/enable/gcbench/publish.py @@ -7,10 +7,9 @@ # is also available online at http://www.enthought.com/licenses/BSD.txt # # Thanks for using Enthought open source! -import math import os -_DOC_TEMPLATE = """ +_INDEX_TEMPLATE = """ @@ -40,7 +39,30 @@ All results are shown relative to the kiva.agg backend. Numbers less than 1.0 indicate a slower result and numbers greater than 1.0 indicate a faster result.

-{content} +{comparison_table} + + +""" +_IMAGE_PAGE_TEMPLATE = """ + + + + +{benchmark_name} Outputs + + + +

+Results for the "{benchmark_name}" benchmark. All times are in milliseconds. +

+{image_table} """ @@ -61,28 +83,30 @@ def publish(results, outdir): functions = {} for bend in backends: for func, stats in results[bend].items(): - value = stats["mean"] if stats is not None else math.nan - functions.setdefault(func, {})[bend] = value - - # Scale timing values relative to a "baseline" backend implementation - for name in functions.keys(): - functions[name] = _format_stats(functions[name], name, "kiva.agg") + functions.setdefault(func, {})[bend] = stats - table = _build_table(backends, functions) + # Scale timing values relative to the "kiva.agg" backend implementation + comparisons = {} + for name, results in functions.items(): + _build_function_page(name, results, outdir) + comparisons[name] = _format_mean(results, "kiva.agg") + comparison_table = _build_comparison_table(backends, comparisons) path = os.path.join(outdir, "index.html") with open(path, "w") as fp: - fp.write(_DOC_TEMPLATE.format(content=table)) + fp.write(_INDEX_TEMPLATE.format(comparison_table=comparison_table)) -def _build_table(backends, functions): - """ Build some table data +def _build_comparison_table(backends, comparisons): + """ Build some table data for comparison of backend performance timings. """ # All the row data rows = [] - for name, stats in functions.items(): + for name, stats in comparisons.items(): # Start the row off with the name of the function - row = [f"{name}"] + # Link to the table of images created by each backend + link = f'' + row = [f"{link}{name}"] for bend in backends: # Each backend stat includes a "valid" flag stat, valid = stats[bend] @@ -102,18 +126,60 @@ def _build_table(backends, functions): return _TABLE_TEMPLATE.format(headers=headers, rows=rows) -def _format_stats(function, func_name, baseline): +def _build_function_page(benchmark_name, results, outdir): + """ Build a page which shows backend outputs next to each other. + """ + # Build the rows + backends = [] + img_tds, stat_tds = "", "" + for name, stats in results.items(): + if stats is None: + continue + + backends.append(name) + img_tds += f'' + stat_tds += f"{_format_stats(stats)}" + + rows = f"{img_tds}\n{stat_tds}" + + # Headers + headers = "\n".join(f"{name}" for name in backends) + + table = _TABLE_TEMPLATE.format(headers=headers, rows=rows) + content = _IMAGE_PAGE_TEMPLATE.format( + benchmark_name=benchmark_name, + image_table=table, + ) + path = os.path.join(outdir, f"{benchmark_name}.html") + with open(path, "w") as fp: + fp.write(content) + + +def _format_mean(results, baseline): """ Convert stats for individual benchmark runs into data for a table cell. """ - basevalue = function[baseline] + basestats = results[baseline] + if basestats is None: + return {name: ("invalid", False) for name in results} + + basevalue = basestats["mean"] formatted = {} - for name, value in function.items(): - if value is math.nan: - formatted[name] = ("n/a", False) + for name, stats in results.items(): + if stats is not None: + relvalue = basevalue / stats["mean"] + formatted[name] = (f"{relvalue:0.2f}", True) else: - relvalue = basevalue / value - # Link to the image created by the benchmark - link = f'' - formatted[name] = (f"{link}{relvalue:0.2f}", True) + formatted[name] = ("n/a", False) return formatted + + +def _format_stats(stats): + """ Convert stats for a single benchmark run into a table. + """ + rows = [ + f"{key.capitalize()}{value:0.4f}" + for key, value in stats.items() + ] + rows = "\n".join(rows) + return f"

Timings:

{rows}
"