Skip to content

Add a page for comparing backend outputs #661

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jwiggins merged 1 commit into from
Mar 2, 2021
Merged

Add a page for comparing backend outputs #661

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions enable/gcbench/bench.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,9 +122,9 @@ def gen_timings(gc, func):

times = np.array(times)
return {
"max": times.max() * 1000,
"min": times.min() * 1000,
"mean": times.mean() * 1000,
"min": times.min() * 1000,
"max": times.max() * 1000,
Copy link
Copy Markdown
Member Author

@jwiggins jwiggins Mar 2, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm using the order of this dictionary when turning it into an HTML table

Copy link
Copy Markdown
Contributor

@rahulporuri rahulporuri Mar 2, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should be fine now that we only support python >= 3.6.

"std": times.std() * 1000,
"count": len(times),
}
114 changes: 90 additions & 24 deletions enable/gcbench/publish.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,9 @@
# is also available online at http://www.enthought.com/licenses/BSD.txt
#
# Thanks for using Enthought open source!
import math
import os

_DOC_TEMPLATE = """
_INDEX_TEMPLATE = """
<!doctype html>
<html lang=en>
<head>
Expand Down Expand Up @@ -40,7 +39,30 @@
All results are shown relative to the kiva.agg backend. Numbers less than 1.0
indicate a slower result and numbers greater than 1.0 indicate a faster result.
</p>
{content}
{comparison_table}
</body>
</html>
"""
_IMAGE_PAGE_TEMPLATE = """
<!doctype html>
<html lang=en>
<head>
<meta charset=utf-8>
<title>{benchmark_name} Outputs</title>
</head>
<body>
<style>
table, th, td {{
padding: 4px;
border: 1px solid gray;
border-collapse: collapse;
text-align: left;
}}
</style>
<p>
Results for the "{benchmark_name}" benchmark. All times are in milliseconds.
</p>
{image_table}
</body>
</html>
"""
Expand All @@ -61,28 +83,30 @@ def publish(results, outdir):
functions = {}
for bend in backends:
for func, stats in results[bend].items():
value = stats["mean"] if stats is not None else math.nan
functions.setdefault(func, {})[bend] = value

# Scale timing values relative to a "baseline" backend implementation
for name in functions.keys():
functions[name] = _format_stats(functions[name], name, "kiva.agg")
functions.setdefault(func, {})[bend] = stats

table = _build_table(backends, functions)
# Scale timing values relative to the "kiva.agg" backend implementation
comparisons = {}
for name, results in functions.items():
_build_function_page(name, results, outdir)
comparisons[name] = _format_mean(results, "kiva.agg")

comparison_table = _build_comparison_table(backends, comparisons)
path = os.path.join(outdir, "index.html")
with open(path, "w") as fp:
fp.write(_DOC_TEMPLATE.format(content=table))
fp.write(_INDEX_TEMPLATE.format(comparison_table=comparison_table))


def _build_table(backends, functions):
""" Build some table data
def _build_comparison_table(backends, comparisons):
""" Build some table data for comparison of backend performance timings.
"""
# All the row data
rows = []
for name, stats in functions.items():
for name, stats in comparisons.items():
# Start the row off with the name of the function
row = [f"<td>{name}</td>"]
# Link to the table of images created by each backend
link = f'<a href="{name}.html">'
row = [f"<td>{link}{name}</a></td>"]
for bend in backends:
# Each backend stat includes a "valid" flag
stat, valid = stats[bend]
Expand All @@ -102,18 +126,60 @@ def _build_table(backends, functions):
return _TABLE_TEMPLATE.format(headers=headers, rows=rows)


def _format_stats(function, func_name, baseline):
def _build_function_page(benchmark_name, results, outdir):
""" Build a page which shows backend outputs next to each other.
"""
# Build the rows
backends = []
img_tds, stat_tds = "", ""
for name, stats in results.items():
if stats is None:
continue

backends.append(name)
img_tds += f'<td><img src="{name}.{benchmark_name}.png" /></td>'
stat_tds += f"<td>{_format_stats(stats)}</td>"

rows = f"<tr>{img_tds}</tr>\n<tr>{stat_tds}</tr>"

# Headers
headers = "\n".join(f"<th>{name}</th>" for name in backends)

table = _TABLE_TEMPLATE.format(headers=headers, rows=rows)
content = _IMAGE_PAGE_TEMPLATE.format(
benchmark_name=benchmark_name,
image_table=table,
)
path = os.path.join(outdir, f"{benchmark_name}.html")
with open(path, "w") as fp:
fp.write(content)


def _format_mean(results, baseline):
""" Convert stats for individual benchmark runs into data for a table cell.
"""
basevalue = function[baseline]
basestats = results[baseline]
if basestats is None:
return {name: ("invalid", False) for name in results}

basevalue = basestats["mean"]
formatted = {}
for name, value in function.items():
if value is math.nan:
formatted[name] = ("n/a", False)
for name, stats in results.items():
if stats is not None:
relvalue = basevalue / stats["mean"]
formatted[name] = (f"{relvalue:0.2f}", True)
else:
relvalue = basevalue / value
# Link to the image created by the benchmark
link = f'<a href="{name}.{func_name}.png">'
formatted[name] = (f"{link}{relvalue:0.2f}</a>", True)
formatted[name] = ("n/a", False)

return formatted


def _format_stats(stats):
""" Convert stats for a single benchmark run into a table.
"""
rows = [
f"<tr><td>{key.capitalize()}</td><td>{value:0.4f}</td></tr>"
for key, value in stats.items()
]
rows = "\n".join(rows)
return f"<p>Timings:</p><table>{rows}</table>"