From af4892d8fc203c826b9ab20b66dcfd517995aa5b Mon Sep 17 00:00:00 2001 From: Viktor Kronvall Date: Sat, 7 Nov 2020 03:30:15 +0900 Subject: [PATCH] Rework reports using chart.js 3249df7 add chart.js thing a806dfd Output table 424d0f7 WIP 4b72393 Add colophon and styling ec94eba Properly format scatter tooltip 60fdba8 Add outliers d51ffeb Add grokularation 9a8b08e Minimum bar width 0ca856b Remove stray console logs 73d573b Implement animating error bars 80da94a Replace annotation plugin with custom function 9be269c Fix groups and regression table 1c1b66a Make overview ticks robust 56a9375 Render 3 significant digits f4afc21 Update overview tick rendering 2886337 Add endpoint tick on logarithmic scale if within one order efd3b6b Fix hover on y-axis c543d8d Split chart.html into separate JS and CSS files 3f6d9a7 Update to use js-chart c14274c Remove backup file 0674754 Remove unused jQuery plugin 930da09 Update www/ with new reports d29acdd Flatten templates/js/ 21bebd8 Add groups field to report fda1377 Make drawErrorBar scope b65e7db Comment on criterion.js functions 9e719c8 Re-render report.html 2ad7928 cleanup 414911b Expand hover hit radius for points 543778c Add explanatory note about interactivity d6d5ce9 Clip error bars to chart area 0547c42 Use chart.js helper to get chart position ed2a680 Add legend that hides groups 7142270 Constrain legend box width 2eec914 Add titles and axes labels 64d9076 Don't display labels 910573b Fix "a" in outlier information 5c26f2c Fix wording ce402c4 Cater explanation depending on screen or print 8e501fe Make legend toggle-able 8da2146 Controls explanation and legend chevron cd2dd68 Avoid page breaks inside single report 7fc20ac Update grokularation with no-prints and scatter plot CI explanation fe49774 Colophon link color in printed version fc8690e Re-generate report.html and fibber.html 2d00dcd Render overview labels over bars on small screens 1ba261e Base error bars on bar index 271bbcf Remove `report` from JS context 446e2a6 Link to Wikipedia for (co)lexical order 198f1ff Update changelog for new HTML reports --- Criterion/Analysis.hs | 6 +- Criterion/EmbeddedData.hs | 18 +- Criterion/Report.hs | 93 +- changelog.md | 20 + criterion.cabal | 5 +- stack.yaml | 1 + templates/criterion.css | 183 ++- templates/criterion.js | 870 ++++++++++ templates/default.tpl | 490 ++---- templates/js/jquery.criterion.js | 106 -- www/fibber-screenshot.png | Bin 30573 -> 42451 bytes www/fibber.html | 1761 ++++++++++++-------- www/report.html | 2573 +++++++++++++----------------- 13 files changed, 3339 insertions(+), 2787 deletions(-) create mode 100644 templates/criterion.js delete mode 100644 templates/js/jquery.criterion.js diff --git a/Criterion/Analysis.hs b/Criterion/Analysis.hs index cbc0454f..8f45716d 100644 --- a/Criterion/Analysis.hs +++ b/Criterion/Analysis.hs @@ -89,9 +89,9 @@ outlierVariance outlierVariance µ σ a = OutlierVariance effect desc varOutMin where ( effect, desc ) | varOutMin < 0.01 = (Unaffected, "no") - | varOutMin < 0.1 = (Slight, "slight") - | varOutMin < 0.5 = (Moderate, "moderate") - | otherwise = (Severe, "severe") + | varOutMin < 0.1 = (Slight, "a slight") + | varOutMin < 0.5 = (Moderate, "a moderate") + | otherwise = (Severe, "a severe") varOutMin = (minBy varOut 1 (minBy cMax 0 µgMin)) / σb2 varOut c = (ac / a) * (σb2 - ac * σg2) where ac = a - c σb = B.estPoint σ diff --git a/Criterion/EmbeddedData.hs b/Criterion/EmbeddedData.hs index b6e85822..1a4dfa5c 100644 --- a/Criterion/EmbeddedData.hs +++ b/Criterion/EmbeddedData.hs @@ -11,27 +11,19 @@ -- -- When the @embed-data-files@ @Cabal@ flag is enabled, this module exports -- the contents of various files (the @data-files@ from @criterion.cabal@, as --- well as minimized versions of jQuery and Flot) embedded as 'ByteString's. +-- well as a minimized version of Chart.js) embedded as a 'ByteString'. module Criterion.EmbeddedData ( dataFiles - , jQueryContents - , flotContents - , flotErrorbarsContents - , flotNavigateContents + , chartContents ) where import Data.ByteString (ByteString) import Data.FileEmbed (embedDir, embedFile) import Language.Haskell.TH.Syntax (runIO) -import qualified Language.Javascript.Flot as Flot -import qualified Language.Javascript.JQuery as JQuery +import qualified Language.Javascript.Chart as Chart dataFiles :: [(FilePath, ByteString)] dataFiles = $(embedDir "templates") -jQueryContents, flotContents, - flotErrorbarsContents, flotNavigateContents :: ByteString -jQueryContents = $(embedFile =<< runIO JQuery.file) -flotContents = $(embedFile =<< runIO (Flot.file Flot.Flot)) -flotErrorbarsContents = $(embedFile =<< runIO (Flot.file Flot.FlotErrorbars)) -flotNavigateContents = $(embedFile =<< runIO (Flot.file Flot.FlotNavigate)) +chartContents :: ByteString +chartContents = $(embedFile =<< runIO (Chart.file Chart.Chart)) diff --git a/Criterion/Report.hs b/Criterion/Report.hs index d85775d7..833bca78 100644 --- a/Criterion/Report.hs +++ b/Criterion/Report.hs @@ -37,13 +37,12 @@ import Control.Monad.IO.Class (MonadIO(liftIO)) import Control.Monad.Reader (ask) import Criterion.Monad (Criterion) import Criterion.Types -import Data.Aeson (ToJSON (..), Value(..), object, (.=), Value, encode) +import Data.Aeson (ToJSON (..), Value(..), object, (.=), Value) import Data.Data (Data, Typeable) import Data.Foldable (forM_) import GHC.Generics (Generic) import Paths_criterion (getDataFileName) import Statistics.Function (minMax) -import Statistics.Types (confidenceInterval, confidenceLevel, confIntCL, estError) import System.Directory (doesFileExist) import System.FilePath ((), (<.>), isPathSeparator) import System.IO (hPutStrLn, stderr) @@ -53,7 +52,9 @@ import Prelude () import Prelude.Compat import qualified Control.Exception as E import qualified Data.Text as T +#if defined(EMBED) import qualified Data.Text.Lazy.Encoding as TLE +#endif import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.IO as TL @@ -61,13 +62,11 @@ import qualified Data.Vector.Generic as G import qualified Data.Vector.Unboxed as U #if defined(EMBED) -import Criterion.EmbeddedData (dataFiles, jQueryContents, flotContents, - flotErrorbarsContents, flotNavigateContents) +import Criterion.EmbeddedData (dataFiles, chartContents) import qualified Data.ByteString.Lazy as BL import qualified Data.Text.Encoding as TE #else -import qualified Language.Javascript.Flot as Flot -import qualified Language.Javascript.JQuery as JQuery +import qualified Language.Javascript.Chart as Chart #endif -- | Trim long flat tails from a KDE plot. @@ -113,26 +112,18 @@ formatReport reports templateName = do Left err -> fail (show err) -- TODO: throw a template exception? Right x -> return x - jQuery <- jQueryFileContents - flot <- flotFileContents - flotErrorbars <- flotErrorbarsFileContents - flotNavigate <- flotNavigateFileContents - jQueryCriterionJS <- readDataFile ("js" "jquery.criterion.js") - criterionCSS <- readDataFile "criterion.css" + criterionJS <- readDataFile "criterion.js" + criterionCSS <- readDataFile "criterion.css" + chartJS <- chartFileContents -- includes, only top level templates <- getTemplateDir template <- includeTemplate (includeFile [templates]) template0 - reports' <- mapM inner reports let context = object [ "json" .= reports - , "report" .= reports' - , "js-jquery" .= jQuery - , "js-flot" .= flot - , "js-flot-errorbars" .= flotErrorbars - , "js-flot-navigate" .= flotNavigate - , "jquery-criterion-js" .= jQueryCriterionJS + , "js-criterion" .= criterionJS + , "js-chart" .= chartJS , "criterion-css" .= criterionCSS ] @@ -152,17 +143,11 @@ formatReport reports templateName = do criterionWarning $ displayMustacheWarning warning return formatted where - jQueryFileContents, flotFileContents :: IO T.Text + chartFileContents :: IO T.Text #if defined(EMBED) - jQueryFileContents = pure $ TE.decodeUtf8 jQueryContents - flotFileContents = pure $ TE.decodeUtf8 flotContents - flotErrorbarsFileContents = pure $ TE.decodeUtf8 flotErrorbarsContents - flotNavigateFileContents = pure $ TE.decodeUtf8 flotNavigateContents + chartFileContents = pure $ TE.decodeUtf8 chartContents #else - jQueryFileContents = T.readFile =<< JQuery.file - flotFileContents = T.readFile =<< Flot.file Flot.Flot - flotErrorbarsFileContents = T.readFile =<< Flot.file Flot.FlotErrorbars - flotNavigateFileContents = T.readFile =<< Flot.file Flot.FlotNavigate + chartFileContents = T.readFile =<< Chart.file Chart.Chart #endif readDataFile :: FilePath -> IO T.Text @@ -185,58 +170,6 @@ formatReport reports templateName = do fmap TextBlock (f (T.unpack fp)) includeNode _ n = return n - -- Merge Report with it's analysis and outliers - merge :: ToJSON a => a -> Value -> Value - merge x y = case toJSON x of - Object x' -> case y of - Object y' -> Object (x' <> y') - _ -> y - _ -> y - - inner :: Report -> IO Value - inner r@Report {..} = do - reportName' <- sanitizeJSString $ T.pack reportName - return $ merge reportAnalysis $ merge reportOutliers $ object - [ "name" .= reportName' - , "json" .= TLE.decodeUtf8 (encode r) - , "number" .= reportNumber - , "iters" .= vector "x" iters - , "times" .= vector "x" times - , "cycles" .= vector "x" cycles - , "kdetimes" .= vector "x" kdeValues - , "kdepdf" .= vector "x" kdePDF - , "kde" .= vector2 "time" "pdf" kdeValues kdePDF - , "anMeanConfidenceLevel" .= anMeanConfidenceLevel - , "anMeanLowerBound" .= anMeanLowerBound - , "anMeanUpperBound" .= anMeanUpperBound - , "anStdDevLowerBound" .= anStdDevLowerBound - , "anStdDevUpperBound" .= anStdDevUpperBound - ] - where - [KDE{..}] = reportKDEs - SampleAnalysis{..} = reportAnalysis - - iters = measure measIters reportMeasured - times = measure measTime reportMeasured - cycles = measure measCycles reportMeasured - anMeanConfidenceLevel - = confidenceLevel $ confIntCL $ estError anMean - (anMeanLowerBound, anMeanUpperBound) - = confidenceInterval anMean - (anStdDevLowerBound, anStdDevUpperBound) - = confidenceInterval anStdDev - - sanitizeJSString :: T.Text -> IO T.Text - sanitizeJSString str = do - let pieces = T.splitOn "\n" str - case pieces of - (_word1:_word2:_) -> do - criterionWarning $ - "Report name " ++ show str ++ " contains newlines, which " ++ - "will be replaced with spaces in the HTML report." - return $ T.unwords pieces - _ -> return str - criterionWarning :: String -> IO () criterionWarning msg = hPutStrLn stderr $ unlines diff --git a/changelog.md b/changelog.md index 24f7c296..38553797 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,23 @@ +Unreleased + +* The HTML reports have been reworked. + + * The `flot` plotting library (`js-flot` on Hackage) has been replaced by + `Chart.js` (`js-chart`). + * Most practical changes focus on improving the functionality of the overview + chart: + * It now supports logarithmic scale (#213). The scale can be toggled by + clicking the x-axis. + * Manual zooming has been replaced by clicking to focus a single bar. + * It now supports a variety of sort orders. + * The legend can now be toggled on/off and is hidden by default. + * Clicking the name of a group in the legend shows/hides all bars in that + group. + * The regression line on the scatter plot shows confidence interval. + * Better support for mobile and print. + * JSON escaping has been made more robust by no longer directly injecting + reports as JavaScript code. + 1.5.7.0 * Warn if an HTML report name contains newlines, and replace newlines with diff --git a/criterion.cabal b/criterion.cabal index be571ff9..3f993999 100644 --- a/criterion.cabal +++ b/criterion.cabal @@ -33,7 +33,7 @@ tested-with: data-files: templates/*.css templates/*.tpl - templates/js/jquery.criterion.js + templates/*.js description: This library provides a powerful but simple way to measure software @@ -97,8 +97,7 @@ library filepath, Glob >= 0.7.2, microstache >= 1.0.1 && < 1.1, - js-flot, - js-jquery, + js-chart >= 2.9.4 && < 3, mtl >= 2, mwc-random >= 0.8.0.3, optparse-applicative >= 0.13, diff --git a/stack.yaml b/stack.yaml index 7bb40178..ce432e9d 100644 --- a/stack.yaml +++ b/stack.yaml @@ -7,5 +7,6 @@ packages: extra-deps: - microstache-1.0.1.1 - statistics-0.14.0.0 +- js-chart-2.9.4.1 flags: {} diff --git a/templates/criterion.css b/templates/criterion.css index 11e22cab..705af318 100644 --- a/templates/criterion.css +++ b/templates/criterion.css @@ -1,102 +1,143 @@ -html, body { - height: 100%; - margin: 0; +html,body { + padding: 0; margin: 0; + font-family: sans-serif; } - -#wrap { - min-height: 100%; +* { + -webkit-tap-highlight-color: transparent; } - -#main { - overflow: auto; - padding-bottom: 180px; /* must be same height as the footer */ +div.scatter, div.kde { + position: relative; + display: inline-block; + box-sizing: border-box; + width: 50%; + padding: 0 2em; +} +.content, .explanation { + margin: auto; + max-width: 1000px; + padding: 0 20px; } -#footer { - position: relative; - margin-top: -180px; /* negative value of footer height */ - height: 180px; - clear: both; - background: #888; - margin: 40px 0 0; - color: white; - font-size: larger; - font-weight: 300; +#legend-toggle { + cursor: pointer; } -body:before { - /* Opera fix */ - content: ""; - height: 100%; - float: left; - width: 0; - margin-top: -32767px; +.overview-info { + float:right; } -body { - font: 14px Helvetica Neue; - text-rendering: optimizeLegibility; - margin-top: 1em; +.overview-info a { + display: inline-block; + margin-left: 10px; +} +.overview-info .info { + font-size: 120%; + font-weight: 400; + vertical-align: middle; + line-height: 1em; +} +.chevron { + position:relative; + color: black; + display:block; + transition-property: transform; + transition-duration: 400ms; + line-height: 1em; + font-size: 180%; +} +.chevron.right { + transform: scale(-1,1); +} +.chevron::before { + vertical-align: middle; + content:"\2039"; } -a:link { - color: steelblue; - text-decoration: none; +select { + outline: none; + border:none; + background: transparent; } -a:visited { - color: #4a743b; - text-decoration: none; +footer .content { + padding: 0; } -#footer a { - color: white; - text-decoration: underline; +span#explain-interactivity { + display-block: inline; + float: right; + color: #444; + font-size: 0.7em; } -.hover { - color: steelblue; +@media screen and (max-width: 800px) { + div.scatter, div.kde { + width: 100%; + display: block; + } + .report-details .outliers { + margin: auto; + } + .report-details table { + margin: auto; + } +} +table.analysis .low, table.analysis .high { + opacity: 0.5; +} +.report-details { + margin: 2em 0; + page-break-inside: avoid; +} +a, a:hover, a:visited, a:active { text-decoration: none; + color: #309ef2; } - -.body { - width: 960px; - margin: auto; +h1.title { + font-weight: 600; } - -.footfirst { - position: relative; - top: 30px; +h1 { + font-weight: 400; } - -th { - font-weight: 500; - opacity: 0.8; +#overview-chart { + width: 100%; /*height is determined by number of rows in JavaScript */ +} +footer { + background: #777777; + color: #ffffff; + padding: 20px; +} +footer a, footer a:hover, footer a:visited, footer a:active { + text-decoration: underline; + color: #fff; } -th.cibound { - opacity: 0.4; +.explanation { + margin-top: 3em; } -.confinterval { - opacity: 0.5; +.explanation h1 { + font-size: 2.6em; } -h1 { - font-size: 36px; - font-weight: 300; - margin-bottom: .3em; +#grokularation.explanation li { + margin: 1em 0; } -h2 { - font-size: 30px; - font-weight: 300; - margin-bottom: .3em; +#controls-explanation.explanation em { + font-weight: 600; + font-style: normal; } -.meanlegend { - color: #404040; - background-color: #ffffff; - opacity: 0.6; - font-size: smaller; +@media print { + footer { + background: transparent; + color: black; + } + footer a, footer a:hover, footer a:visited, footer a:active { + color: #309ef2; + } + .no-print { + display: none; + } } diff --git a/templates/criterion.js b/templates/criterion.js new file mode 100644 index 00000000..b127edee --- /dev/null +++ b/templates/criterion.js @@ -0,0 +1,870 @@ +(function() { + 'use strict'; + window.addEventListener('beforeprint', function() { + for (var id in Chart.instances) { + Chart.instances[id].resize(); + } + }, false); + + var errorBarPlugin = (function () { + function drawErrorBar(chart, ctx, low, high, y, height, color) { + ctx.save(); + ctx.lineWidth = 3; + ctx.strokeStyle = color; + var area = chart.chartArea; + ctx.rect(area.left, area.top, area.right - area.left, area.bottom - area.top); + ctx.clip(); + ctx.beginPath(); + ctx.moveTo(low, y - height); + ctx.lineTo(low, y + height); + ctx.moveTo(low, y); + ctx.lineTo(high, y); + ctx.moveTo(high, y - height); + ctx.lineTo(high, y + height); + ctx.stroke(); + ctx.restore(); + } + // Avoid sudden jumps in error bars when switching + // between linear and logarithmic scale + function conservativeError(vx, mx, now, final, scale) { + var finalDiff = Math.abs(mx - final); + var diff = Math.abs(vx - now); + return (diff > finalDiff) ? vx + scale * finalDiff : now; + } + return { + afterDatasetDraw: function(chart, easingOptions) { + var ctx = chart.ctx; + var easing = easingOptions.easingValue; + chart.data.datasets.forEach(function(d, i) { + var bars = chart.getDatasetMeta(i).data; + var axis = chart.scales[chart.options.scales.xAxes[0].id]; + bars.forEach(function(b, j) { + var value = axis.getValueForPixel(b._view.x); + var final = axis.getValueForPixel(b._model.x); + var errorBar = d.errorBars[j]; + var low = axis.getPixelForValue(value - errorBar.minus); + var high = axis.getPixelForValue(value + errorBar.plus); + var finalLow = axis.getPixelForValue(final - errorBar.minus); + var finalHigh = axis.getPixelForValue(final + errorBar.plus); + var l = easing === 1 ? finalLow : + conservativeError(b._view.x, b._model.x, low, + finalLow, -1.0); + var h = easing === 1 ? finalHigh : + conservativeError(b._view.x, b._model.x, + high, finalHigh, 1.0); + drawErrorBar(chart, ctx, l, h, b._view.y, 4, errorBar.color); + }); + }); + }, + }; + })(); + + // Formats the ticks on the X-axis on the scatter plot + var iterFormatter = function() { + var denom = 0; + return function(iters, index, values) { + if (iters == 0) { + return ''; + } + if (index == values.length - 1) { + return ''; + } + var power; + if (iters >= 1e9) { + denom = 1e9; + power = '⁹'; + } else if (iters >= 1e6) { + denom = 1e6; + power = '⁶'; + } else if (iters >= 1e3) { + denom = 1e3; + power = '³'; + } else { + denom = 1; + } + if (denom > 1) { + var value = (iters / denom).toFixed(); + return String(value) + '×10' + power; + } else { + return String(iters); + } + }; + }; + + var colors = ["#edc240", "#afd8f8", "#cb4b4b", "#4da74d", "#9440ed"]; + var errorColors = ["#cda220", "#8fb8d8", "#ab2b2b", "#2d872d", "#7420cd"]; + + + // Positions tooltips at cursor. Required for overview since the bars may + // extend past the canvas width. + Chart.Tooltip.positioners.cursor = function(_elems, position) { + return position; + } + + function axisType(logaxis) { + return logaxis ? 'logarithmic' : 'linear'; + } + + function reportSort(a, b) { + return a.reportNumber - b.reportNumber; + } + + // adds groupNumber and group fields to reports; + // returns list of list of reports, grouped by group + function groupReports(reports) { + + function reportGroup(report) { + var parts = report.groups.slice(); + parts.pop(); + return parts.join('/'); + } + + var groups = []; + reports.forEach(function(report) { + report.group = reportGroup(report); + if (groups.length === 0) { + groups.push([report]); + } else { + var prevGroup = groups[groups.length - 1]; + var prevGroupName = prevGroup[0].group; + if (prevGroupName === report.group) { + prevGroup.push(report); + } else { + groups.push([report]); + } + } + report.groupNumber = groups.length - 1; + }); + return groups; + } + + // compares 2 arrays lexicographically + function lex(aParts, bParts) { + for(var i = 0; i < aParts.length && i < bParts.length; i++) { + var x = aParts[i]; + var y = bParts[i]; + if (x < y) { + return -1; + } + if (y < x) { + return 1; + } + } + return aParts.length - bParts.length; + } + function lexicalSort(a, b) { + return lex(a.groups, b.groups); + } + + function reverseLexicalSort(a, b) { + return lex(a.groups.slice().reverse(), b.groups.slice().reverse()); + } + + function durationSort(a, b) { + return a.reportAnalysis.anMean.estPoint - b.reportAnalysis.anMean.estPoint; + } + function reverseDurationSort(a,b) { + return -durationSort(a,b); + } + + function timeUnits(secs) { + if (secs < 0) + return timeUnits(-secs); + else if (secs >= 1e9) + return [1e-9, "Gs"]; + else if (secs >= 1e6) + return [1e-6, "Ms"]; + else if (secs >= 1) + return [1, "s"]; + else if (secs >= 1e-3) + return [1e3, "ms"]; + else if (secs >= 1e-6) + return [1e6, "\u03bcs"]; + else if (secs >= 1e-9) + return [1e9, "ns"]; + else if (secs >= 1e-12) + return [1e12, "ps"]; + return [1, "s"]; + } + + function formatUnit(raw, unit, precision) { + var v = precision ? raw.toPrecision(precision) : Math.round(raw); + var label = String(v) + ' ' + unit; + return label; + } + + function formatTime(value, precision) { + var units = timeUnits(value); + var scale = units[0]; + return formatUnit(value * scale, units[1], precision); + } + + // pure function that produces the 'data' object of the overview chart + function overviewData(state, reports) { + var order = state.order; + var sorter = order === 'report-index' ? reportSort + : order === 'lex' ? lexicalSort + : order === 'colex' ? reverseLexicalSort + : order === 'duration' ? durationSort + : order === 'rev-duration' ? reverseDurationSort + : reportSort; + var sortedReports = reports.filter(function(report) { + return !state.hidden[report.groupNumber]; + }).slice().sort(sorter); + var data = sortedReports.map(function(report) { + return report.reportAnalysis.anMean.estPoint; + }); + var labels = sortedReports.map(function(report) { + return report.groups.join(' / '); + }); + var upperBound = function(report) { + var est = report.reportAnalysis.anMean; + return est.estPoint + est.estError.confIntUDX; + }; + var errorBars = sortedReports.map(function(report) { + var est = report.reportAnalysis.anMean; + return { + minus: est.estError.confIntLDX, + plus: est.estError.confIntUDX, + color: errorColors[report.groupNumber % errorColors.length] + }; + }); + var top = sortedReports.map(upperBound).reduce(function(a, b) { + return Math.max(a, b); + }, 0); + var scale = top; + if(state.activeReport !== null) { + reports.forEach(function(report) { + if(report.reportNumber === state.activeReport) { + scale = upperBound(report); + } + }); + } + + return { + labels: labels, + top: top, + max: scale * 1.1, + reports: sortedReports, + datasets: [{ + borderWidth: 1, + backgroundColor: sortedReports.map(function(report) { + var active = report.reportNumber === state.activeReport; + var alpha = active ? 'ff' : 'a0'; + var color = colors[report.groupNumber % colors.length] + alpha; + if (active) { + return Chart.helpers.getHoverColor(color); + } else { + return color; + } + }), + barThickness: 16, + barPercentage: 0.8, + data: data, + errorBars: errorBars, + minBarLength: 2, + }] + }; + } + + function inside(box, point) { + return (point.x >= box.left && point.x <= box.right && point.y >= box.top && + point.y <= box.bottom); + } + + function overviewHover(event, elems) { + var chart = this; + var xAxis = chart.scales[chart.options.scales.xAxes[0].id]; + var yAxis = chart.scales[chart.options.scales.yAxes[0].id]; + var point = Chart.helpers.getRelativePosition(event, chart); + var over = + (inside(xAxis, point) || inside(yAxis, point) || elems.length > 0); + if (over) { + chart.canvas.style.cursor = "pointer"; + } else { + chart.canvas.style.cursor = "default"; + } + } + + // Re-renders the overview after clicking/sorting + function renderOverview(state, reports, chart) { + var data = overviewData(state, reports); + var xaxis = chart.options.scales.xAxes[0]; + xaxis.ticks.max = data.max; + chart.config.data.datasets[0].backgroundColor = data.datasets[0].backgroundColor; + chart.config.data.datasets[0].errorBars = data.datasets[0].errorBars; + chart.config.data.datasets[0].data = data.datasets[0].data; + chart.options.scales.xAxes[0].type = axisType(state.logaxis); + chart.options.legend.display = state.legend; + chart.data.labels = data.labels; + chart.update(); + } + + function overviewClick(state, reports) { + return function(event, elems) { + var chart = this; + var xAxis = chart.scales[chart.options.scales.xAxes[0].id]; + var yAxis = chart.scales[chart.options.scales.yAxes[0].id]; + var point = Chart.helpers.getRelativePosition(event, chart); + var sorted = overviewData(state, reports).reports; + + function activateBar(index) { + // Trying to activate active bar disables instead + if (sorted[index].reportNumber === state.activeReport) { + state.activeReport = null; + } else { + state.activeReport = sorted[index].reportNumber; + } + } + + if (inside(xAxis, point)) { + state.activeReport = null; + state.logaxis = !state.logaxis; + renderOverview(state, reports, chart); + } else if (inside(yAxis, point)) { + var index = yAxis.getValueForPixel(point.y); + activateBar(index); + renderOverview(state, reports, chart); + } else if (elems.length > 0) { + var elem = elems[0]; + var index = elem._index; + activateBar(index); + state.logaxis = false; + renderOverview(state, reports, chart); + } else if(inside(chart.chartArea, point)) { + state.activeReport = null; + renderOverview(state, reports, chart); + } + }; + } + + // listener for sort drop-down + function overviewSort(state, reports, chart) { + return function(event) { + state.order = event.currentTarget.value; + renderOverview(state, reports, chart); + }; + } + + // Returns a formatter for the ticks on the X-axis of the overview + function overviewTick(state) { + return function(value, index, values) { + var label = formatTime(value); + if (state.logaxis) { + const remain = Math.round(value / + (Math.pow(10, Math.floor(Chart.helpers.log10(value))))); + if (index === values.length - 1) { + // Draw endpoint if we don't span a full order of magnitude + if (values[index] / values[1] < 10) { + return label; + } else { + return ''; + } + } + if (remain === 1) { + return label; + } + return ''; + } else { + // Don't show the right endpoint + if (index === values.length - 1) { + return ''; + } + return label; + } + } + } + + function mkOverview(reports) { + var canvas = document.createElement('canvas'); + + var state = { + logaxis: false, + activeReport: null, + order: 'index', + hidden: {}, + legend: false, + }; + + + var data = overviewData(state, reports); + var chart = new Chart(canvas.getContext('2d'), { + type: 'horizontalBar', + data: data, + plugins: [errorBarPlugin], + options: { + onHover: overviewHover, + onClick: overviewClick(state, reports), + onResize: function(chart, size) { + if (size.width < 800) { + chart.options.scales.yAxes[0].ticks.mirror = true; + chart.options.scales.yAxes[0].ticks.padding = -10; + chart.options.scales.yAxes[0].ticks.fontColor = '#000'; + } else { + chart.options.scales.yAxes[0].ticks.fontColor = '#666'; + chart.options.scales.yAxes[0].ticks.mirror = false; + chart.options.scales.yAxes[0].ticks.padding = 0; + } + }, + elements: { + rectangle: { + borderWidth: 2, + }, + }, + scales: { + yAxes: [{ + ticks: { + // make sure we draw the ticks above the error bars + z: 2, + } + }], + xAxes: [{ + display: true, + type: axisType(state.logaxis), + ticks: { + autoSkip: false, + min: 0, + max: data.top * 1.1, + minRotation: 0, + maxRotation: 0, + callback: overviewTick(state), + } + }] + }, + responsive: true, + maintainAspectRatio: false, + legend: { + display: state.legend, + position: 'right', + onLeave: function() { + chart.canvas.style.cursor = 'default'; + }, + onHover: function() { + chart.canvas.style.cursor = 'pointer'; + }, + onClick: function(_event, item) { + // toggle hidden + state.hidden[item.groupNumber] = !state.hidden[item.groupNumber]; + renderOverview(state, reports, chart); + }, + labels: { + boxWidth: 12, + generateLabels: function() { + var groups = []; + var groupNames = []; + reports.forEach(function(report) { + var index = groups.indexOf(report.groupNumber); + if (index === -1) { + groups.push(report.groupNumber); + var groupName = report.groups.slice(0,report.groups.length-1).join(' / '); + groupNames.push(groupName); + } + }); + return groups.map(function(groupNumber, index) { + var color = colors[groupNumber % colors.length]; + return { + text: groupNames[index], + fillStyle: color, + hidden: state.hidden[groupNumber], + groupNumber: groupNumber, + }; + }); + }, + }, + }, + tooltips: { + position: 'cursor', + callbacks: { + label: function(item) { + return formatTime(item.xLabel, 3); + }, + }, + }, + title: { + display: false, + text: 'Chart.js Horizontal Bar Chart' + } + } + }); + document.getElementById('sort-overview') + .addEventListener('change', overviewSort(state, reports, chart)); + var toggle = document.getElementById('legend-toggle'); + toggle.addEventListener('mouseup', function () { + state.legend = !state.legend; + if(state.legend) { + toggle.classList.add('right'); + } else { + toggle.classList.remove('right'); + } + renderOverview(state, reports, chart); + }) + return canvas; + } + + function mkKDE(report) { + var canvas = document.createElement('canvas'); + var mean = report.reportAnalysis.anMean.estPoint; + var units = timeUnits(mean); + var scale = units[0]; + var reportKDE = report.reportKDEs[0]; + var data = reportKDE.kdeValues.map(function(time, index) { + var pdf = reportKDE.kdePDF[index]; + return { + x: time * scale, + y: pdf + }; + }); + var chart = new Chart(canvas.getContext('2d'), { + type: 'line', + data: { + datasets: [{ + label: 'KDE', + borderColor: colors[0], + borderWidth: 2, + backgroundColor: '#00000000', + data: data, + hoverBorderWidth: 1, + pointHitRadius: 8, + }, + { + label: 'mean' + } + ], + }, + plugins: [{ + afterDraw: function(chart) { + var ctx = chart.ctx; + var area = chart.chartArea; + var axis = chart.scales[chart.options.scales.xAxes[0].id]; + var value = axis.getPixelForValue(mean * scale); + ctx.save(); + ctx.strokeStyle = colors[1]; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.moveTo(value, area.top); + ctx.lineTo(value, area.bottom); + ctx.stroke(); + ctx.restore(); + }, + }], + options: { + title: { + display: true, + text: report.groups.join(' / ') + ' — time densities', + }, + elements: { + point: { + radius: 0, + hitRadius: 0 + } + }, + scales: { + xAxes: [{ + display: true, + type: 'linear', + scaleLabel: { + display: false, + labelString: 'Time' + }, + ticks: { + min: reportKDE.kdeValues[0] * scale, + max: reportKDE.kdeValues[reportKDE.kdeValues.length - 1] * scale, + callback: function(value, index, values) { + // Don't show endpoints + if (index === 0 || index === values.length - 1) { + return ''; + } + var str = String(value) + ' ' + units[1]; + return str; + }, + } + }], + yAxes: [{ + display: true, + type: 'linear', + ticks: { + min: 0, + callback: function() { + return ''; + }, + }, + }] + }, + responsive: true, + legend: { + display: false, + position: 'right', + }, + tooltips: { + mode: 'nearest', + callbacks: { + title: function() { + return ''; + }, + label: function( + item) { + return formatUnit(item.xLabel, units[1], 3); + }, + }, + }, + hover: { + intersect: false + }, + } + }); + return canvas; + } + + function mkScatter(report) { + + // collect the measured value for a given regression + function getMeasured(key) { + var ix = report.reportKeys.indexOf(key); + return report.reportMeasured.map(function(x) { + return x[ix]; + }); + } + + var canvas = document.createElement('canvas'); + var times = getMeasured("time"); + var iters = getMeasured("iters"); + var lastIter = iters[iters.length - 1]; + var olsTime = report.reportAnalysis.anRegress[0].regCoeffs.iters; + var dataPoints = times.map(function(time, i) { + return { + x: iters[i], + y: time + } + }); + var formatter = iterFormatter(); + var chart = new Chart(canvas.getContext('2d'), { + type: 'scatter', + data: { + datasets: [{ + data: dataPoints, + label: 'scatter', + borderWidth: 2, + pointHitRadius: 8, + borderColor: colors[1], + backgroundColor: '#fff', + }, + { + data: [ + {x: 0, y: 0 }, + { x: lastIter, y: olsTime.estPoint * lastIter } + ], + label: 'regression', + type: 'line', + backgroundColor: "#00000000", + borderColor: colors[0], + pointRadius: 0, + }, + { + data: [{ + x: 0, + y: 0 + }, { + x: lastIter, + y: (olsTime.estPoint - olsTime.estError.confIntLDX) * lastIter, + }], + label: 'lower', + type: 'line', + fill: 1, + borderWidth: 0, + pointRadius: 0, + borderColor: '#00000000', + backgroundColor: colors[0] + '33', + }, + { + data: [{ + x: 0, + y: 0 + }, { + x: lastIter, + y: (olsTime.estPoint + olsTime.estError.confIntUDX) * lastIter, + }], + label: 'upper', + type: 'line', + fill: 1, + borderWidth: 0, + borderColor: '#00000000', + pointRadius: 0, + backgroundColor: colors[0] + '33', + }, + ], + }, + options: { + title: { + display: true, + text: report.groups.join(' / ') + ' — time per iteration', + }, + scales: { + yAxes: [{ + display: true, + type: 'linear', + scaleLabel: { + display: false, + labelString: 'Time' + }, + ticks: { + callback: function(value, index, values) { + return formatTime(value); + }, + } + }], + xAxes: [{ + display: true, + type: 'linear', + scaleLabel: { + display: false, + labelString: 'Iterations' + }, + ticks: { + callback: formatter, + max: lastIter, + } + }], + }, + legend: { + display: false, + }, + tooltips: { + callbacks: { + label: function(ttitem, ttdata) { + var iters = ttitem.xLabel; + var duration = ttitem.yLabel; + return formatTime(duration, 3) + ' / ' + + iters.toLocaleString() + ' iters'; + }, + }, + }, + } + }); + return canvas; + } + + // Create an HTML Element with attributes and child nodes + function elem(tag, props, children) { + var node = document.createElement(tag); + if (children) { + children.forEach(function(child) { + if (typeof child === 'string') { + var txt = document.createTextNode(child); + node.appendChild(txt); + } else { + node.appendChild(child); + } + }); + } + Object.assign(node, props); + return node; + } + + function bounds(analysis) { + var mean = analysis.estPoint; + return { + low: mean - analysis.estError.confIntLDX, + mean: mean, + high: mean + analysis.estError.confIntUDX + }; + } + + function confidence(level) { + return String(1 - level) + ' confidence level'; + } + + function mkOutliers(report) { + var outliers = report.reportAnalysis.anOutlierVar; + return elem('div', {className: 'outliers'}, [ + elem('p', {}, [ + 'Outlying measurements have ', + outliers.ovDesc, + ' (', String((outliers.ovFraction * 100).toPrecision(3)), '%)', + ' effect on estimated standard deviation.' + ]) + ]); + } + + function mkTable(report) { + var analysis = report.reportAnalysis; + var timep4 = function(t) { + return formatTime(t, 3) + }; + var idformatter = function(t) { + return t.toPrecision(3) + }; + var rows = [ + Object.assign({ + label: 'OLS regression', + formatter: timep4 + }, + bounds(analysis.anRegress[0].regCoeffs.iters)), + Object.assign({ + label: 'R² goodness-of-fit', + formatter: idformatter + }, + bounds(analysis.anRegress[0].regRSquare)), + Object.assign({ + label: 'Mean execution time', + formatter: timep4 + }, + bounds(analysis.anMean)), + Object.assign({ + label: 'Standard deviation', + formatter: timep4 + }, + bounds(analysis.anStdDev)), + ]; + return elem('table', { + className: 'analysis' + }, [ + elem('thead', {}, [ + elem('tr', {}, [ + elem('th'), + elem('th', { + className: 'low', + title: confidence(analysis.anRegress[0].regCoeffs.iters.estError.confIntCL) + }, ['lower bound']), + elem('th', {}, ['estimate']), + elem('th', { + className: 'high', + title: confidence(analysis.anRegress[0].regCoeffs.iters.estError.confIntCL) + }, ['upper bound']), + ]) + ]), + elem('tbody', {}, rows.map(function(row) { + return elem('tr', {}, [ + elem('td', {}, [row.label]), + elem('td', {className: 'low'}, [row.formatter(row.low, 4)]), + elem('td', {}, [row.formatter(row.mean)]), + elem('td', {className: 'high'}, [row.formatter(row.high, 4)]), + ]); + })) + ]); + } + document.addEventListener('DOMContentLoaded', function() { + var reportData = JSON.parse(document.getElementById('report-data') + .getAttribute('data-report-json')) + .map(function(report) { + report.groups = report.reportName.split('/'); + return report; + }); + groupReports(reportData); + var overview = document.getElementById('overview-chart'); + var overviewLineHeight = 16 * 1.25; + overview.style.height = + String(overviewLineHeight * reportData.length + 36) + 'px'; + overview.appendChild(mkOverview(reportData.slice())); + var reports = document.getElementById('reports'); + reportData.forEach(function(report, i) { + var id = 'report_' + String(i); + reports.appendChild( + elem('div', {id: id, className: 'report-details'}, [ + elem('h1', {}, [elem('a', {href: '#' + id}, [report.groups.join(' / ')])]), + elem('div', {className: 'kde'}, [mkKDE(report)]), + elem('div', {className: 'scatter'}, [mkScatter(report)]), + mkTable(report), mkOutliers(report) + ])); + }); + }, false); +})(); diff --git a/templates/default.tpl b/templates/default.tpl index 9d6dcb3f..8cd6ec62 100644 --- a/templates/default.tpl +++ b/templates/default.tpl @@ -1,362 +1,138 @@ - + - - - criterion report - - - - - - - - - -
-
-

criterion performance measurements

- -

overview

- -

want to understand this report?

- -
- -{{#report}} -

{{name}}

- - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
lower boundestimateupper bound
OLS regressionxxxxxxxxx
R² goodness-of-fitxxxxxxxxx
Mean execution time{{anMeanLowerBound}}{{anMean.estPoint}}{{anMeanUpperBound}}
Standard deviation{{anStdDevLowerBound}}{{anStdDev.estPoint}}{{anStdDevUpperBound}}
- - -

Outlying measurements have {{anOutlierVar.ovDesc}} - ({{anOutlierVar.ovFraction}}%) - effect on estimated standard deviation.

- -{{/report}} - -

understanding this report

- -

In this report, each function benchmarked by criterion is assigned - a section of its own. The charts in each section are active; if - you hover your mouse over data points and annotations, you will see - more details.

- -
    -
  • The chart on the left is a - kernel - density estimate (also known as a KDE) of time - measurements. This graphs the probability of any given time - measurement occurring. A spike indicates that a measurement of a - particular time occurred; its height indicates how often that - measurement was repeated.
    • - -
  • The chart on the right is the raw data from which the kernel - density estimate is built. The x axis indicates the - number of loop iterations, while the y axis shows measured - execution time for the given number of loop iterations. The - line behind the values is the linear regression prediction of - execution time for a given number of iterations. Ideally, all - measurements will be on (or very near) this line.
    • -
- -

Under the charts is a small table. - The first two rows are the results of a linear regression run - on the measurements displayed in the right-hand chart.

- -
    -
  • OLS regression indicates the - time estimated for a single loop iteration using an ordinary - least-squares regression model. This number is more accurate - than the mean estimate below it, as it more effectively - eliminates measurement overhead and other constant factors.
    • -
  • R² goodness-of-fit is a measure of how - accurately the linear regression model fits the observed - measurements. If the measurements are not too noisy, R² - should lie between 0.99 and 1, indicating an excellent fit. If - the number is below 0.99, something is confounding the accuracy - of the linear model.
    • -
  • Mean execution time and standard deviation are - statistics calculated from execution time - divided by number of iterations.
    • -
- -

We use a statistical technique called - the bootstrap - to provide confidence intervals on our estimates. The - bootstrap-derived upper and lower bounds on estimates let you see - how accurate we believe those estimates to be. (Hover the mouse - over the table headers to see the confidence levels.)

- -

A noisy benchmarking environment can cause some or many - measurements to fall far from the mean. These outlying - measurements can have a significant inflationary effect on the - estimate of the standard deviation. We calculate and display an - estimate of the extent to which the standard deviation has been - inflated by outliers.

- - + + + criterion report + + + + + +
+

criterion performance measurements

+

want to understand this report?

+

overview

+
+ + + ⓘ + + +
+ +
-
+
-