From 844ca8b6b2931a7c6911dcbc1a1a29a76572b0ac Mon Sep 17 00:00:00 2001 From: Ben Alpert Date: Wed, 18 Nov 2015 16:16:34 -0800 Subject: [PATCH] benchmarking: measure and analyze scripts This uses wall-clock time (for now) so it's noisier than alternatives (cachegrind, CPU perf-counters), but it's still valuable. In a future diff we can make it use those. `measure.py` outputs something that `analyze.py` can understand, but you can use `analyze.py` without `measure.py` too. The file format is simple: ``` $ cat measurements.txt factory_ms_jsc_jit 13.580322265625 factory_ms_jsc_jit 13.659912109375 factory_ms_jsc_jit 13.67919921875 factory_ms_jsc_nojit 12.827880859375 factory_ms_jsc_nojit 13.105224609375 factory_ms_jsc_nojit 13.195068359375 factory_ms_node 40.4891400039196 factory_ms_node 40.6669420003891 factory_ms_node 43.52413299679756 ssr_pe_cold_ms_jsc_jit 43.06005859375 ... ``` (The lines do not need to be sorted.) Comparing 0.14.0 vs master: ``` $ ./measure.py react-0.14.0.min.js >014.txt Measuring SSR for PE benchmark (30 trials) .............................. Measuring SSR for PE with warm JIT (3 slow trials) ... $ ./measure.py react.min.js >master.txt Measuring SSR for PE benchmark (30 trials) .............................. Measuring SSR for PE with warm JIT (3 slow trials) ... $ ./analyze.py 014.txt master.txt Comparing 014.txt (control) vs master.txt (test) Significant differences marked by *** % change from control to test, with 99% CIs: * factory_ms_jsc_jit % change: -0.56% [ -2.51%, +1.39%] means: 14.037 (control), 13.9593 (test) * factory_ms_jsc_nojit % change: +1.23% [ -1.18%, +3.64%] means: 13.2586 (control), 13.4223 (test) * factory_ms_node % change: +3.53% [ +0.29%, +6.77%] *** means: 42.0529 (control), 43.54 (test) * ssr_pe_cold_ms_jsc_jit % change: -6.84% [ -9.04%, -4.65%] *** means: 44.2444 (control), 41.2187 (test) * ssr_pe_cold_ms_jsc_nojit % change: -11.81% [-14.66%, -8.96%] *** means: 52.9449 (control), 46.6953 (test) * ssr_pe_cold_ms_node % change: -2.70% [ -4.52%, -0.88%] *** means: 96.8909 (control), 94.2741 (test) * ssr_pe_warm_ms_jsc_jit % change: -17.60% [-22.04%, -13.16%] *** means: 13.763 (control), 11.3439 (test) * ssr_pe_warm_ms_jsc_nojit % change: -20.65% [-22.62%, -18.68%] *** means: 30.8829 (control), 24.5074 (test) * ssr_pe_warm_ms_node % change: -8.76% [-13.48%, -4.03%] *** means: 30.0193 (control), 27.3964 (test) $ ``` --- scripts/bench/README.md | 8 +++ scripts/bench/analyze.py | 111 ++++++++++++++++++++++++++++ scripts/bench/measure.py | 151 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 270 insertions(+) create mode 100755 scripts/bench/analyze.py create mode 100755 scripts/bench/measure.py diff --git a/scripts/bench/README.md b/scripts/bench/README.md index e0f2f44159a..7fa8003d5a2 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -1,5 +1,13 @@ Work-in-progress benchmarks. +## Running the suite + +``` +$ ./measure.py react-a.min.js >a.txt +$ ./measure.py react-b.min.js >b.txt +$ ./analyze.py a.txt b.txt +``` + ## Running one One thing you can do with them is benchmark initial render time for a realistic hierarchy: diff --git a/scripts/bench/analyze.py b/scripts/bench/analyze.py new file mode 100755 index 00000000000..f6eccbe015a --- /dev/null +++ b/scripts/bench/analyze.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# Copyright 2015, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +import math +import sys + +import numpy as np +import numpy.random as npr +import scipy.stats + + +def _bootstrap_mean_sem(samples): + """Return the estimated standard error for a distribution's mean.""" + samples = np.array(samples) + n = len(samples) + indices = npr.randint(0, n, (10000, n)) + samples = samples[indices] + means = np.sort(np.mean(samples, axis=1)) + return np.std(means, ddof=1) + + +def _read_measurements(f): + """Read measurements from a file. + + Returns {'a': [1.0, 2.0, 3.0], 'b': [5.0, 5.0, 5.0]} for a file containing + the six lines: ['a 1', 'a 2', 'a 3', 'b 5', 'b 5', 'b 5']. + """ + measurements = {} + for line in f: + label, value = line.split(None, 1) + measurements.setdefault(label, []).append(float(value)) + return measurements + + +def _compute_mean_and_sd_of_ratio_from_delta_method( + mean_test, + sem_test, + mean_control, + sem_control +): + mean = ( + ((mean_test - mean_control) / mean_control) - + (pow(sem_control, 2) * mean_test / pow(mean_control, 3)) + ) + var = ( + pow(sem_test / mean_control, 2) + + (pow(sem_control * mean_test, 2) / pow(mean_control, 4)) + ) + return (mean, math.sqrt(var)) + + +def _main(): + if len(sys.argv) != 3: + sys.stderr.write("usage: analyze.py control.txt test.txt\n") + return 1 + + ci_size = 0.99 + p_value = scipy.stats.norm.ppf(0.5 * (1 + ci_size)) + + control, test = sys.argv[1:] + with open(control) as f: + control_measurements = _read_measurements(f) + with open(test) as f: + test_measurements = _read_measurements(f) + keys = set() + keys.update(control_measurements.iterkeys()) + keys.update(test_measurements.iterkeys()) + + print "Comparing %s (control) vs %s (test)" % (control, test) + print "Significant differences marked by ***" + print "%% change from control to test, with %g%% CIs:" % (ci_size * 100,) + print + + any_sig = False + for key in sorted(keys): + print "* %s" % (key,) + control_nums = control_measurements.get(key, []) + test_nums = test_measurements.get(key, []) + if not control_nums or not test_nums: + print " skipping..." + continue + + mean_control = np.mean(control_nums) + mean_test = np.mean(test_nums) + sem_control = _bootstrap_mean_sem(control_nums) + sem_test = _bootstrap_mean_sem(test_nums) + + rat_mean, rat_sem = _compute_mean_and_sd_of_ratio_from_delta_method( + mean_test, sem_test, mean_control, sem_control + ) + rat_low = rat_mean - p_value * rat_sem + rat_high = rat_mean + p_value * rat_sem + + sig = rat_high < 0 or rat_low > 0 + any_sig = any_sig or sig + + print " %% change: %+6.2f%% [%+6.2f%%, %+6.2f%%]%s" % ( + 100 * rat_mean, + 100 * rat_low, + 100 * rat_high, + ' ***' if sig else '' + ) + print " means: %g (control), %g (test)" % (mean_control, mean_test) + +if __name__ == '__main__': + sys.exit(_main()) diff --git a/scripts/bench/measure.py b/scripts/bench/measure.py new file mode 100755 index 00000000000..4cedf47c78e --- /dev/null +++ b/scripts/bench/measure.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +# Copyright 2015, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +import functools +import json +import os +import subprocess +import sys + + +def _run_js_in_jsc(jit, js, env): + return subprocess.check_call( + ['jsc', '-e', """ + function now() { + return preciseTime() * 1000; + } + function globalEval(code) { + (0, eval)(code); + } + function report(label, time) { + print(label + '_' + %(engine)s, time); + } + + this.ENV = %(env)s; + %(js)s + """ % { + 'env': json.dumps(env), + 'js': js, + 'engine': json.dumps('jsc_' + ('jit' if jit else 'nojit')), + }], + env=dict(os.environ, JSC_useJIT='yes' if jit else 'no'), + ) + +_run_js_in_jsc_jit = functools.partial(_run_js_in_jsc, True) +_run_js_in_jsc_nojit = functools.partial(_run_js_in_jsc, False) + + +def _run_js_in_node(js, env): + return subprocess.check_call( + ['node', '-e', """ + function now() { + var hrTime = process.hrtime(); + return hrTime[0] * 1e3 + hrTime[1] * 1e-6; + } + function globalEval(code) { + var vm = require('vm'); + // Hide "module" so UMD wrappers use the global + vm.runInThisContext('(function(module){' + code + '\\n})()'); + } + function readFile(filename) { + var fs = require('fs'); + return fs.readFileSync(filename); + } + function report(label, time) { + console.log(label + '_node', time); + } + + global.ENV = %(env)s; + %(js)s + """ % { + 'env': json.dumps(env), + 'js': js + }] + ) + + +def _measure_ssr_ms(engine, react_path, bench_name, bench_path, measure_warm): + engine( + """ + var reactCode = readFile(ENV.react_path); + var START = now(); + globalEval(reactCode); + var END = now(); + if (typeof React !== 'object') throw new Error('React not laoded'); + report('factory_ms', END - START); + + globalEval(readFile(ENV.bench_path)); + if (typeof Benchmark !== 'function') { + throw new Error('benchmark not loaded'); + } + var START = now(); + var html = React.renderToString(React.createElement(Benchmark)); + html.charCodeAt(0); // flatten ropes + var END = now(); + report('ssr_' + ENV.bench_name + '_cold_ms', END - START); + + var warmup = ENV.measure_warm ? 80 : 0; + var trials = ENV.measure_warm ? 40 : 0; + + for (var i = 0; i < warmup; i++) { + React.renderToString(React.createElement(Benchmark)); + } + + for (var i = 0; i < trials; i++) { + var START = now(); + var html = React.renderToString(React.createElement(Benchmark)); + html.charCodeAt(0); // flatten ropes + var END = now(); + report('ssr_' + ENV.bench_name + '_warm_ms', END - START); + } + """, + { + 'bench_name': bench_name, + 'bench_path': bench_path, + 'measure_warm': measure_warm, + 'react_path': react_path, + }, + ) + + +def _main(): + if len(sys.argv) != 2: + sys.stderr.write("usage: measure.py react.min.js >out.txt\n") + return 1 + react_path = sys.argv[1] + + trials = 30 + sys.stderr.write("Measuring SSR for PE benchmark (%d trials)\n" % trials) + for i in range(trials): + for engine in [ + _run_js_in_jsc_jit, + _run_js_in_jsc_nojit, + _run_js_in_node + ]: + _measure_ssr_ms(engine, react_path, 'pe', 'bench-pe-es5.js', False) + sys.stderr.write(".") + sys.stderr.flush() + sys.stderr.write("\n") + + trials = 3 + sys.stderr.write("Measuring SSR for PE with warm JIT (%d slow trials)\n" % trials) + for i in range(trials): + for engine in [ + _run_js_in_jsc_jit, + _run_js_in_jsc_nojit, + _run_js_in_node + ]: + _measure_ssr_ms(engine, react_path, 'pe', 'bench-pe-es5.js', True) + sys.stderr.write(".") + sys.stderr.flush() + sys.stderr.write("\n") + + +if __name__ == '__main__': + sys.exit(_main()) +