diff --git a/packages/devtools_app/test_benchmarks/README.md b/packages/devtools_app/benchmark/README.md similarity index 100% rename from packages/devtools_app/test_benchmarks/README.md rename to packages/devtools_app/benchmark/README.md diff --git a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart new file mode 100644 index 00000000000..a64ebd3c7c5 --- /dev/null +++ b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart @@ -0,0 +1,86 @@ +// Copyright 2023 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Note: this test was modeled after the example test from Flutter Gallery: +// https://github.com/flutter/gallery/blob/master/test_benchmarks/benchmarks_test.dart + +import 'dart:convert' show JsonEncoder; +import 'dart:io'; + +import 'package:test/test.dart'; +import 'package:web_benchmarks/server.dart'; + +import 'test_infra/common.dart'; +import 'test_infra/project_root_directory.dart'; + +final metricList = [ + 'preroll_frame', + 'apply_frame', + 'drawFrameDuration', +]; + +final valueList = [ + 'average', + 'outlierAverage', + 'outlierRatio', + 'noise', +]; + +/// Tests that the DevTools web benchmarks are run and reported correctly. +void main() { + test( + 'Can run a web benchmark', + () async { + stdout.writeln('Starting web benchmark tests ...'); + + final taskResult = await serveWebBenchmark( + benchmarkAppDirectory: projectRootDirectory(), + entryPoint: 'benchmark/test_infra/client.dart', + useCanvasKit: true, + treeShakeIcons: false, + initialPage: benchmarkInitialPage, + ); + + stdout.writeln('Web benchmark tests finished.'); + + expect( + taskResult.scores.keys, + hasLength(DevToolsBenchmark.values.length), + ); + + for (final benchmarkName in DevToolsBenchmark.values.map((e) => e.id)) { + expect( + taskResult.scores[benchmarkName], + hasLength(metricList.length * valueList.length + 1), + ); + + for (final metricName in metricList) { + for (final valueName in valueList) { + expect( + taskResult.scores[benchmarkName]?.where( + (score) => score.metric == '$metricName.$valueName', + ), + hasLength(1), + ); + } + } + + expect( + taskResult.scores[benchmarkName]?.where( + (score) => score.metric == 'totalUiFrame.average', + ), + hasLength(1), + ); + } + + expect( + const JsonEncoder.withIndent(' ').convert(taskResult.toJson()), + isA(), + ); + }, + timeout: Timeout.none, + ); + + // TODO(kenz): add tests that verify performance meets some expected threshold +} diff --git a/packages/devtools_app/benchmark/test_infra/client.dart b/packages/devtools_app/benchmark/test_infra/client.dart new file mode 100644 index 00000000000..8d6b7b71e36 --- /dev/null +++ b/packages/devtools_app/benchmark/test_infra/client.dart @@ -0,0 +1,25 @@ +// Copyright 2023 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:web_benchmarks/client.dart'; + +import 'common.dart'; +import 'devtools_recorder.dart'; + +typedef RecorderFactory = Recorder Function(); + +final Map benchmarks = { + DevToolsBenchmark.navigateThroughOfflineScreens.id: () => DevToolsRecorder( + benchmark: DevToolsBenchmark.navigateThroughOfflineScreens, + ), +}; + +/// Runs the client of the DevTools web benchmarks. +/// +/// When the DevTools web benchmarks are run, the server builds an app with this +/// file as the entry point (see `run_benchmarks.dart`). The app automates +/// the DevTools web app, records some performance data, and reports them. +Future main() async { + await runBenchmarks(benchmarks, initialPage: benchmarkInitialPage); +} diff --git a/packages/devtools_app/benchmark/test_infra/common.dart b/packages/devtools_app/benchmark/test_infra/common.dart new file mode 100644 index 00000000000..5c9078b3652 --- /dev/null +++ b/packages/devtools_app/benchmark/test_infra/common.dart @@ -0,0 +1,19 @@ +// Copyright 2023 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// The initial page to load upon opening the DevTools benchmark app or +/// reloading it in Chrome. +// +// We use an empty initial page so that the benchmark server does not attempt +// to load the default page 'index.html', which will show up as "page not +// found" in DevTools. +const String benchmarkInitialPage = ''; + +const String devtoolsBenchmarkPrefix = 'devtools'; + +enum DevToolsBenchmark { + navigateThroughOfflineScreens; + + String get id => '${devtoolsBenchmarkPrefix}_$name'; +} diff --git a/packages/devtools_app/benchmark/test_infra/devtools_automator.dart b/packages/devtools_app/benchmark/test_infra/devtools_automator.dart new file mode 100644 index 00000000000..08f98bbba3c --- /dev/null +++ b/packages/devtools_app/benchmark/test_infra/devtools_automator.dart @@ -0,0 +1,110 @@ +// Copyright 2023 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:devtools_app/devtools_app.dart'; +import 'package:devtools_test/helpers.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import 'common.dart'; + +/// A class that automates the DevTools web app. +class DevToolsAutomater { + DevToolsAutomater({ + required this.benchmark, + required this.stopWarmingUpCallback, + }); + + /// The current benchmark. + final DevToolsBenchmark benchmark; + + /// A function to call when warm-up is finished. + /// + /// This function is intended to ask `Recorder` to mark the warm-up phase + /// as over. + final void Function() stopWarmingUpCallback; + + /// Whether the automation has ended. + bool finished = false; + + /// A widget controller for automation. + late LiveWidgetController controller; + + /// The [DevToolsApp] widget with automation. + Widget createWidget() { + // There is no `catchError` here, because all errors are caught by + // the zone set up in `lib/web_benchmarks.dart` in `flutter/flutter`. + Future.delayed(safePumpDuration, automateDevToolsGestures); + return DevToolsApp( + defaultScreens(), + AnalyticsController(enabled: false, firstRun: false), + ); + } + + Future automateDevToolsGestures() async { + await warmUp(); + + switch (benchmark) { + case DevToolsBenchmark.navigateThroughOfflineScreens: + await _handleNavigateThroughOfflineScreens(); + } + + // At the end of the test, mark as finished. + finished = true; + } + + /// Warm up the animation. + Future warmUp() async { + _logStatus('Warming up.'); + + // Let animation stop. + await animationStops(); + + // Set controller. + controller = LiveWidgetController(WidgetsBinding.instance); + + await controller.pumpAndSettle(); + + // TODO(kenz): investigate if we need to do something like the Flutter + // Gallery benchmark tests to warn up the Flutter engine. + + // When warm-up finishes, inform the recorder. + stopWarmingUpCallback(); + + _logStatus('Warm-up finished.'); + } + + Future _handleNavigateThroughOfflineScreens() async { + _logStatus('Navigate through offline DevTools tabs'); + await navigateThroughDevToolsScreens( + controller, + runWithExpectations: false, + ); + _logStatus('==== End navigate through offline DevTools tabs ===='); + } +} + +void _logStatus(String log) { + // ignore: avoid_print, intentional test logging. + print('==== $log ===='); +} + +const Duration _animationCheckingInterval = Duration(milliseconds: 50); + +Future animationStops() async { + if (!WidgetsBinding.instance.hasScheduledFrame) return; + + final Completer stopped = Completer(); + + Timer.periodic(_animationCheckingInterval, (timer) { + if (!WidgetsBinding.instance.hasScheduledFrame) { + stopped.complete(); + timer.cancel(); + } + }); + + await stopped.future; +} diff --git a/packages/devtools_app/benchmark/test_infra/devtools_recorder.dart b/packages/devtools_app/benchmark/test_infra/devtools_recorder.dart new file mode 100644 index 00000000000..f4ecc3a637f --- /dev/null +++ b/packages/devtools_app/benchmark/test_infra/devtools_recorder.dart @@ -0,0 +1,45 @@ +// Copyright 2023 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:devtools_app/initialization.dart'; +import 'package:flutter/material.dart'; +import 'package:web_benchmarks/client.dart'; + +import 'common.dart'; +import 'devtools_automator.dart'; + +/// A recorder that measures frame building durations for the DevTools. +class DevToolsRecorder extends WidgetRecorder { + DevToolsRecorder({required this.benchmark}) + : super(name: benchmark.id, useCustomWarmUp: true); + + /// The name of the DevTools benchmark to be run. + /// + /// See `common.dart` for the list of the names of all benchmarks. + final DevToolsBenchmark benchmark; + + DevToolsAutomater? _devToolsAutomator; + bool get _finished => _devToolsAutomator?.finished ?? false; + + /// Whether we should continue recording. + @override + bool shouldContinue() => !_finished || profile.shouldContinue(); + + /// Creates the [DevToolsAutomater] widget. + @override + Widget createWidget() { + _devToolsAutomator = DevToolsAutomater( + benchmark: benchmark, + stopWarmingUpCallback: profile.stopWarmingUp, + ); + return _devToolsAutomator!.createWidget(); + } + + @override + Future run() async { + // ignore: invalid_use_of_visible_for_testing_member, valid use for benchmark tests. + await initializeDevTools(); + return super.run(); + } +} diff --git a/packages/devtools_app/benchmark/test_infra/project_root_directory.dart b/packages/devtools_app/benchmark/test_infra/project_root_directory.dart new file mode 100644 index 00000000000..968cd23ed02 --- /dev/null +++ b/packages/devtools_app/benchmark/test_infra/project_root_directory.dart @@ -0,0 +1,31 @@ +// Copyright 2023 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Note: this code was copied from Flutter gallery +// https://github.com/flutter/gallery/blob/main/test_benchmarks/benchmarks/project_root_directory.dart + +import 'dart:io'; +import 'package:path/path.dart' as path; + +bool _hasPubspec(Directory directory) { + return directory.listSync().any( + (entity) => + FileSystemEntity.isFileSync(entity.path) && + path.basename(entity.path) == 'pubspec.yaml', + ); +} + +Directory projectRootDirectory() { + var current = Directory.current.absolute; + + while (!_hasPubspec(current)) { + if (current.path == current.parent.path) { + throw Exception('Reached file system root when seeking project root.'); + } + + current = current.parent; + } + + return current; +} diff --git a/packages/devtools_app/test_benchmarks/web_bundle_size_test.dart b/packages/devtools_app/benchmark/web_bundle_size_test.dart similarity index 100% rename from packages/devtools_app/test_benchmarks/web_bundle_size_test.dart rename to packages/devtools_app/benchmark/web_bundle_size_test.dart diff --git a/packages/devtools_app/pubspec.yaml b/packages/devtools_app/pubspec.yaml index 306737acdac..4d8762c98b2 100644 --- a/packages/devtools_app/pubspec.yaml +++ b/packages/devtools_app/pubspec.yaml @@ -79,6 +79,7 @@ dev_dependencies: mockito: ^5.4.1 stager: ^1.0.1 test: ^1.21.1 + web_benchmarks: ^0.1.0+10 webkit_inspection_protocol: ">=0.5.0 <2.0.0" flutter: diff --git a/tool/ci/benchmark_tests.sh b/tool/ci/benchmark_tests.sh index e41e278cb6f..c23ba3eb29e 100755 --- a/tool/ci/benchmark_tests.sh +++ b/tool/ci/benchmark_tests.sh @@ -10,5 +10,5 @@ set -ex source ./tool/ci/setup.sh pushd $DEVTOOLS_DIR/packages/devtools_app -flutter test test_benchmarks/ +flutter test test/benchmarks/ popd