From b4392a2ff229fb5bceaf1aa9034c32af0bc88de4 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Tue, 28 Nov 2023 08:35:15 -0800 Subject: [PATCH 01/19] Refactor DevTools test to share more helpers fixes remove import Add a basic performance benchmark test for DevTools. --- packages/devtools_app/pubspec.yaml | 3 + .../devtools_benchmarks_test.dart | 81 +++++++++ .../test_benchmarks/run_benchmarks.dart | 34 ++++ .../test_benchmarks/test_infra/client.dart | 23 +++ .../test_benchmarks/test_infra/common.dart | 11 ++ .../test_infra/devtools_automator.dart | 81 +++++++++ .../test_infra/devtools_recorder.dart | 44 +++++ .../test_infra/project_root_directory.dart | 28 +++ .../test_benchmarks/test_infra/scroll.dart | 166 ++++++++++++++++++ 9 files changed, 471 insertions(+) create mode 100644 packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart create mode 100644 packages/devtools_app/test_benchmarks/run_benchmarks.dart create mode 100644 packages/devtools_app/test_benchmarks/test_infra/client.dart create mode 100644 packages/devtools_app/test_benchmarks/test_infra/common.dart create mode 100644 packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart create mode 100644 packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart create mode 100644 packages/devtools_app/test_benchmarks/test_infra/project_root_directory.dart create mode 100644 packages/devtools_app/test_benchmarks/test_infra/scroll.dart diff --git a/packages/devtools_app/pubspec.yaml b/packages/devtools_app/pubspec.yaml index 5cdbb55b2c2..9efbbf2f8b7 100644 --- a/packages/devtools_app/pubspec.yaml +++ b/packages/devtools_app/pubspec.yaml @@ -83,6 +83,7 @@ dev_dependencies: mockito: ^5.4.1 stager: ^1.0.1 test: ^1.21.1 + web_benchmarks: ^0.1.0+8 webkit_inspection_protocol: ">=0.5.0 <2.0.0" flutter: @@ -176,3 +177,5 @@ dependency_overrides: path: ../devtools_test devtools_extensions: path: ../devtools_extensions + web_benchmarks: + path: ../../../packages/packages/web_benchmarks diff --git a/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart b/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart new file mode 100644 index 00000000000..a89393331d5 --- /dev/null +++ b/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart @@ -0,0 +1,81 @@ +// Copyright 2023 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Note: this test was modeled after the example test from Flutter Gallery: +// https://github.com/flutter/gallery/blob/master/test_benchmarks/benchmarks_test.dart + +import 'dart:convert' show JsonEncoder; +import 'dart:io'; + +import 'package:test/test.dart'; +import 'package:web_benchmarks/server.dart'; + +import 'test_infra/common.dart'; +import 'test_infra/project_root_directory.dart'; + +final metricList = [ + 'preroll_frame', + 'apply_frame', + 'drawFrameDuration', +]; + +final valueList = [ + 'average', + 'outlierAverage', + 'outlierRatio', + 'noise', +]; + +/// Tests that the DevTools web benchmarks are run and reported correctly. +Future main() async { + test( + 'Can run a web benchmark', + () async { + stdout.writeln('Starting web benchmark tests ...'); + + final taskResult = await serveWebBenchmark( + benchmarkAppDirectory: projectRootDirectory(), + entryPoint: 'test_benchmarks/test_infra/client.dart', + useCanvasKit: true, + treeShakeIcons: false, + initialPage: '', + ); + + stdout.writeln('Web benchmark tests finished.'); + + expect(taskResult.scores.keys, hasLength(benchmarkList.length)); + + for (final benchmarkName in benchmarkList) { + expect( + taskResult.scores[benchmarkName], + hasLength(metricList.length * valueList.length + 1), + ); + + for (final metricName in metricList) { + for (final valueName in valueList) { + expect( + taskResult.scores[benchmarkName]?.where( + (score) => score.metric == '$metricName.$valueName', + ), + hasLength(1), + ); + } + } + + expect( + taskResult.scores[benchmarkName]?.where( + (score) => score.metric == 'totalUiFrame.average', + ), + hasLength(1), + ); + } + + expect( + const JsonEncoder.withIndent(' ').convert(taskResult.toJson()), + isA(), + ); + }, + timeout: Timeout.none, + ); +} diff --git a/packages/devtools_app/test_benchmarks/run_benchmarks.dart b/packages/devtools_app/test_benchmarks/run_benchmarks.dart new file mode 100644 index 00000000000..f26a7ed3546 --- /dev/null +++ b/packages/devtools_app/test_benchmarks/run_benchmarks.dart @@ -0,0 +1,34 @@ +// Copyright 2023 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert' show JsonEncoder; +import 'dart:io'; + +import 'package:web_benchmarks/server.dart'; + +import 'test_infra/project_root_directory.dart'; + +/// Runs the DevTools web benchmarks and reports the benchmark data. +Future main() async { + stdout.writeln('Starting web benchmark tests ...'); + + final taskResult = await serveWebBenchmark( + benchmarkAppDirectory: projectRootDirectory(), + entryPoint: 'test_benchmarks/test_infra/client.dart', + useCanvasKit: true, + treeShakeIcons: false, + headless: false, + initialPage: '', + ); + + stdout.writeln('Web benchmark tests finished.'); + + stdout.writeln('==== Results ===='); + + stdout.writeln( + const JsonEncoder.withIndent(' ').convert(taskResult.toJson()), + ); + + stdout.writeln('==== End of results ===='); +} diff --git a/packages/devtools_app/test_benchmarks/test_infra/client.dart b/packages/devtools_app/test_benchmarks/test_infra/client.dart new file mode 100644 index 00000000000..791b7445630 --- /dev/null +++ b/packages/devtools_app/test_benchmarks/test_infra/client.dart @@ -0,0 +1,23 @@ +// Copyright 2022 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:web_benchmarks/client.dart'; + +import 'common.dart'; +import 'devtools_recorder.dart'; + +typedef RecorderFactory = Recorder Function(); + +final Map benchmarks = { + devtoolsPageLoadPerf: () => DevToolsRecorder(benchmarkName: devtoolsPageLoadPerf), +}; + +/// Runs the client of the DevTools web benchmarks. +/// +/// When the DevTools web benchmarks are run, the server builds an app with this +/// file as the entry point (see `run_benchmarks.dart`). The app automates +/// the DevTools web app, records some performance data, and reports them. +Future main() async { + await runBenchmarks(benchmarks); +} diff --git a/packages/devtools_app/test_benchmarks/test_infra/common.dart b/packages/devtools_app/test_benchmarks/test_infra/common.dart new file mode 100644 index 00000000000..824e465066a --- /dev/null +++ b/packages/devtools_app/test_benchmarks/test_infra/common.dart @@ -0,0 +1,11 @@ +// Copyright 2023 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +const String devtoolsBenchmarkPrefix = 'devtools'; + +const String devtoolsPageLoadPerf = '${devtoolsBenchmarkPrefix}_page_load_perf'; + +const benchmarkList = [ + devtoolsPageLoadPerf, +]; diff --git a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart new file mode 100644 index 00000000000..33b7f204b23 --- /dev/null +++ b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart @@ -0,0 +1,81 @@ +// Copyright 2023 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// ignore_for_file:avoid_print + +import 'package:devtools_app/devtools_app.dart'; +import 'package:devtools_test/helpers.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import 'scroll.dart'; + +/// A class that automates the DevTools web app. +class DevToolsAutomater { + DevToolsAutomater({ + required this.benchmarkName, + required this.stopWarmingUpCallback, + }); + + /// The name of the current benchmark. + final String benchmarkName; + + /// A function to call when warm-up is finished. + /// + /// This function is intended to ask `Recorder` to mark the warm-up phase + /// as over. + final void Function() stopWarmingUpCallback; + + /// Whether the automation has ended. + bool finished = false; + + /// A widget controller for automation. + late LiveWidgetController controller; + + /// The [DevToolsApp] widget with automation. + Widget createWidget() { + // There is no `catchError` here, because all errors are caught by + // the zone set up in `lib/web_benchmarks.dart` in `flutter/flutter`. + Future.delayed(safePumpDuration, automateDevToolsGestures); + return DevToolsApp( + defaultScreens(), + AnalyticsController(enabled: false, firstRun: false), + ); + } + + Future automateDevToolsGestures() async { + await warmUp(); + + print('==== Navigate through DevTools tabs ===='); + await navigateThroughDevToolsScreens( + controller, + runWithExpectations: false, + ); + print('==== End of navigate through DevTools tabs ===='); + + // At the end of the test, mark as finished. + finished = true; + } + + /// Warm up the animation. + Future warmUp() async { + print('Warming up.'); + + // Let animation stop. + await animationStops(); + + // Set controller. + controller = LiveWidgetController(WidgetsBinding.instance); + + await controller.pumpAndSettle(); + + // TODO(kenz): investigate if we need to do something like the Flutter + // Gallery benchmark tests to warn up the Flutter engine. + + // When warm-up finishes, inform the recorder. + stopWarmingUpCallback(); + + print('Warm-up finished.'); + } +} diff --git a/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart b/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart new file mode 100644 index 00000000000..259c571a1f7 --- /dev/null +++ b/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart @@ -0,0 +1,44 @@ +// Copyright 2023 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:devtools_app/initialization.dart'; +import 'package:flutter/material.dart'; +import 'package:web_benchmarks/client.dart'; + +import 'devtools_automator.dart'; + +/// A recorder that measures frame building durations for the DevTools. +class DevToolsRecorder extends WidgetRecorder { + DevToolsRecorder({required this.benchmarkName}) + : super(name: benchmarkName, useCustomWarmUp: true); + + /// The name of the DevTools benchmark to be run. + /// + /// See `common.dart` for the list of the names of all benchmarks. + final String benchmarkName; + + DevToolsAutomater? _devToolsAutomator; + bool get _finished => _devToolsAutomator?.finished ?? false; + + /// Whether we should continue recording. + @override + bool shouldContinue() => !_finished || profile.shouldContinue(); + + /// Creates the [DevToolsAutomater] widget. + @override + Widget createWidget() { + _devToolsAutomator = DevToolsAutomater( + benchmarkName: benchmarkName, + stopWarmingUpCallback: profile.stopWarmingUp, + ); + return _devToolsAutomator!.createWidget(); + } + + @override + Future run() async { + // ignore: invalid_use_of_visible_for_testing_member, valid use for benchmark tests. + await initializeDevTools(); + return super.run(); + } +} diff --git a/packages/devtools_app/test_benchmarks/test_infra/project_root_directory.dart b/packages/devtools_app/test_benchmarks/test_infra/project_root_directory.dart new file mode 100644 index 00000000000..b80dc1eb104 --- /dev/null +++ b/packages/devtools_app/test_benchmarks/test_infra/project_root_directory.dart @@ -0,0 +1,28 @@ +// Copyright 2023 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; +import 'package:path/path.dart' as path; + +bool _hasPubspec(Directory directory) { + return directory.listSync().any( + (entity) => + FileSystemEntity.isFileSync(entity.path) && + path.basename(entity.path) == 'pubspec.yaml', + ); +} + +Directory projectRootDirectory() { + var current = Directory.current.absolute; + + while (!_hasPubspec(current)) { + if (current.path == current.parent.path) { + throw Exception('Reached file system root when seeking project root.'); + } + + current = current.parent; + } + + return current; +} \ No newline at end of file diff --git a/packages/devtools_app/test_benchmarks/test_infra/scroll.dart b/packages/devtools_app/test_benchmarks/test_infra/scroll.dart new file mode 100644 index 00000000000..4b9d865afb3 --- /dev/null +++ b/packages/devtools_app/test_benchmarks/test_infra/scroll.dart @@ -0,0 +1,166 @@ +// Copyright 2023 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter/material.dart'; +import 'package:flutter/rendering.dart'; +import 'package:flutter_test/flutter_test.dart'; + +const Duration _animationCheckingInterval = Duration(milliseconds: 50); + +const Duration _scrollAnimationLength = Duration(milliseconds: 300); + +Offset _absoluteTopLeft(RenderObject renderObject) => + (renderObject as RenderBox).localToGlobal(Offset.zero); + +Rect _absoluteRect(RenderObject renderObject) => + _absoluteTopLeft(renderObject) & renderObject.paintBounds.size; + +Size _windowSize(BuildContext context) => MediaQuery.of(context).size; + +Rect _windowRect(BuildContext context) => Offset.zero & _windowSize(context); + +bool _isSuperset({required Rect large, required Rect small}) => + large.top <= small.top && + large.left <= small.left && + large.bottom >= small.bottom && + large.right >= small.right; + +const _minFreeRoomRequirement = 5.0; + +/// Whether [small] is a subset of [large] and has sufficient room +/// inside [large], at the end of [large] specified by [axisDirection]. +bool _hasSufficientFreeRoom({ + required Rect large, + required Rect small, + required AxisDirection axisDirection, +}) { + if (!_isSuperset(large: large, small: small)) { + return false; + } else { + bool result; + + switch (axisDirection) { + case AxisDirection.down: + result = large.bottom - small.bottom >= _minFreeRoomRequirement; + break; + case AxisDirection.right: + result = large.right - small.right >= _minFreeRoomRequirement; + break; + case AxisDirection.left: + result = small.left - large.left >= _minFreeRoomRequirement; + break; + case AxisDirection.up: + result = small.top - large.top >= _minFreeRoomRequirement; + break; + } + + return result; + } +} + +Future animationStops() async { + if (!WidgetsBinding.instance.hasScheduledFrame) return; + + final Completer stopped = Completer(); + + Timer.periodic(_animationCheckingInterval, (timer) { + if (!WidgetsBinding.instance.hasScheduledFrame) { + stopped.complete(); + timer.cancel(); + } + }); + + await stopped.future; +} + +Future scrollUntilVisible({ + required Element element, + bool strict = false, + bool animated = true, +}) async { + final elementRenderObject = element.renderObject!; + final elementRect = _absoluteRect(elementRenderObject); + + final scrollable = Scrollable.of(element); + final viewport = RenderAbstractViewport.of(elementRenderObject); + + final visibleWindow = _absoluteRect(viewport).intersect(_windowRect(element)); + + // If there is free room between this demo button and the end of + // the scrollable, the next demo button is visible and can be tapped. + if (!strict && + _hasSufficientFreeRoom( + large: visibleWindow, + small: elementRect, + axisDirection: scrollable.axisDirection, + )) { + return; + } + + late double pixelsToBeMoved; + switch (scrollable.axisDirection) { + case AxisDirection.down: + pixelsToBeMoved = elementRect.top - visibleWindow.top; + break; + case AxisDirection.right: + pixelsToBeMoved = elementRect.left - visibleWindow.left; + break; + case AxisDirection.left: + pixelsToBeMoved = visibleWindow.right - elementRect.right; + break; + case AxisDirection.up: + pixelsToBeMoved = visibleWindow.bottom - elementRect.bottom; + break; + } + + final targetPixels = scrollable.position.pixels + pixelsToBeMoved; + final restrictedTargetPixels = targetPixels + .clamp( + scrollable.position.minScrollExtent, + scrollable.position.maxScrollExtent, + ) + .toDouble(); + + await scrollToPosition( + scrollable: scrollable, + pixels: restrictedTargetPixels, + animated: animated, + ); +} + +Future scrollToExtreme({ + required ScrollableState? scrollable, + bool toEnd = false, + bool animated = true, +}) async { + final targetPixels = toEnd + ? scrollable!.position.maxScrollExtent + : scrollable!.position.minScrollExtent; + + await scrollToPosition( + scrollable: scrollable, + pixels: targetPixels, + animated: animated, + ); +} + +Future scrollToPosition({ + required ScrollableState? scrollable, + required double pixels, + bool animated = true, +}) async { + if (animated) { + await scrollable!.position.animateTo( + pixels, + duration: _scrollAnimationLength, + curve: Curves.easeInOut, + ); + } else { + scrollable!.position.jumpTo(pixels); + } + + await animationStops(); +} \ No newline at end of file From 17ba32bef8ff775569c57d70383a07de3df4a7ac Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Mon, 4 Dec 2023 09:59:20 -0800 Subject: [PATCH 02/19] upgrade web_benchmarks --- packages/devtools_app/pubspec.yaml | 4 +--- .../test_benchmarks/devtools_benchmarks_test.dart | 3 +++ packages/devtools_app/test_benchmarks/run_benchmarks.dart | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/devtools_app/pubspec.yaml b/packages/devtools_app/pubspec.yaml index 9efbbf2f8b7..e00f97ba5f5 100644 --- a/packages/devtools_app/pubspec.yaml +++ b/packages/devtools_app/pubspec.yaml @@ -83,7 +83,7 @@ dev_dependencies: mockito: ^5.4.1 stager: ^1.0.1 test: ^1.21.1 - web_benchmarks: ^0.1.0+8 + web_benchmarks: ^0.1.0+9 webkit_inspection_protocol: ">=0.5.0 <2.0.0" flutter: @@ -177,5 +177,3 @@ dependency_overrides: path: ../devtools_test devtools_extensions: path: ../devtools_extensions - web_benchmarks: - path: ../../../packages/packages/web_benchmarks diff --git a/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart b/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart index a89393331d5..81261214368 100644 --- a/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart +++ b/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart @@ -39,6 +39,9 @@ Future main() async { entryPoint: 'test_benchmarks/test_infra/client.dart', useCanvasKit: true, treeShakeIcons: false, + // Pass an empty initial page so that the benchmark server does not + // attempt to load the default page 'index.html', which will show up as + // "page not found" in DevTools. initialPage: '', ); diff --git a/packages/devtools_app/test_benchmarks/run_benchmarks.dart b/packages/devtools_app/test_benchmarks/run_benchmarks.dart index f26a7ed3546..e1d2ac0e2fa 100644 --- a/packages/devtools_app/test_benchmarks/run_benchmarks.dart +++ b/packages/devtools_app/test_benchmarks/run_benchmarks.dart @@ -19,6 +19,9 @@ Future main() async { useCanvasKit: true, treeShakeIcons: false, headless: false, + // Pass an empty initial page so that the benchmark server does not attempt + // to load the default page 'index.html', which will show up as "page not + // found" in DevTools. initialPage: '', ); From 6fe43f514bb938271c9ab1f506490957c083bbb9 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Mon, 4 Dec 2023 10:01:05 -0800 Subject: [PATCH 03/19] copyright year --- packages/devtools_app/test_benchmarks/test_infra/client.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/devtools_app/test_benchmarks/test_infra/client.dart b/packages/devtools_app/test_benchmarks/test_infra/client.dart index 791b7445630..1fc2d5606b8 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/client.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/client.dart @@ -1,4 +1,4 @@ -// Copyright 2022 The Chromium Authors. All rights reserved. +// Copyright 2023 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. From 4c7aa5f0e2ec19eaa9094b3d8ad21418bcf1a5d6 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Tue, 5 Dec 2023 10:59:08 -0800 Subject: [PATCH 04/19] Add initial page to runBenchmarks --- packages/devtools_app/test_benchmarks/run_benchmarks.dart | 7 ++----- .../devtools_app/test_benchmarks/test_infra/client.dart | 2 +- .../devtools_app/test_benchmarks/test_infra/common.dart | 8 ++++++++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/devtools_app/test_benchmarks/run_benchmarks.dart b/packages/devtools_app/test_benchmarks/run_benchmarks.dart index e1d2ac0e2fa..7aac370b3f6 100644 --- a/packages/devtools_app/test_benchmarks/run_benchmarks.dart +++ b/packages/devtools_app/test_benchmarks/run_benchmarks.dart @@ -7,6 +7,7 @@ import 'dart:io'; import 'package:web_benchmarks/server.dart'; +import 'test_infra/common.dart'; import 'test_infra/project_root_directory.dart'; /// Runs the DevTools web benchmarks and reports the benchmark data. @@ -18,11 +19,7 @@ Future main() async { entryPoint: 'test_benchmarks/test_infra/client.dart', useCanvasKit: true, treeShakeIcons: false, - headless: false, - // Pass an empty initial page so that the benchmark server does not attempt - // to load the default page 'index.html', which will show up as "page not - // found" in DevTools. - initialPage: '', + initialPage: benchmarkInitialPage, ); stdout.writeln('Web benchmark tests finished.'); diff --git a/packages/devtools_app/test_benchmarks/test_infra/client.dart b/packages/devtools_app/test_benchmarks/test_infra/client.dart index 1fc2d5606b8..f546acba963 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/client.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/client.dart @@ -19,5 +19,5 @@ final Map benchmarks = { /// file as the entry point (see `run_benchmarks.dart`). The app automates /// the DevTools web app, records some performance data, and reports them. Future main() async { - await runBenchmarks(benchmarks); + await runBenchmarks(benchmarks, initialPage: benchmarkInitialPage); } diff --git a/packages/devtools_app/test_benchmarks/test_infra/common.dart b/packages/devtools_app/test_benchmarks/test_infra/common.dart index 824e465066a..a72f9dfc830 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/common.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/common.dart @@ -6,6 +6,14 @@ const String devtoolsBenchmarkPrefix = 'devtools'; const String devtoolsPageLoadPerf = '${devtoolsBenchmarkPrefix}_page_load_perf'; +/// The initial page to load upon opening the DevTools benchmark app or +/// reloading it in Chrome. +// +// We use an empty initial page so that the benchmark server does not attempt +// to load the default page 'index.html', which will show up as "page not +// found" in DevTools. +const String benchmarkInitialPage = ''; + const benchmarkList = [ devtoolsPageLoadPerf, ]; From cf3af46e1178d0f48b05a217f29549da94731227 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Tue, 5 Dec 2023 12:49:28 -0800 Subject: [PATCH 05/19] cleanup --- packages/devtools_app/pubspec.yaml | 2 +- .../test_infra/devtools_automator.dart | 15 +++++++++------ .../test_infra/project_root_directory.dart | 3 +++ .../test_benchmarks/test_infra/scroll.dart | 3 +++ 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/packages/devtools_app/pubspec.yaml b/packages/devtools_app/pubspec.yaml index e00f97ba5f5..7be4f3307e0 100644 --- a/packages/devtools_app/pubspec.yaml +++ b/packages/devtools_app/pubspec.yaml @@ -83,7 +83,7 @@ dev_dependencies: mockito: ^5.4.1 stager: ^1.0.1 test: ^1.21.1 - web_benchmarks: ^0.1.0+9 + web_benchmarks: ^0.1.0+10 webkit_inspection_protocol: ">=0.5.0 <2.0.0" flutter: diff --git a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart index 33b7f204b23..1def560da1c 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// ignore_for_file:avoid_print - import 'package:devtools_app/devtools_app.dart'; import 'package:devtools_test/helpers.dart'; import 'package:flutter/material.dart'; @@ -47,12 +45,12 @@ class DevToolsAutomater { Future automateDevToolsGestures() async { await warmUp(); - print('==== Navigate through DevTools tabs ===='); + _logStatus('==== Navigate through DevTools tabs ===='); await navigateThroughDevToolsScreens( controller, runWithExpectations: false, ); - print('==== End of navigate through DevTools tabs ===='); + _logStatus('==== End of navigate through DevTools tabs ===='); // At the end of the test, mark as finished. finished = true; @@ -60,7 +58,7 @@ class DevToolsAutomater { /// Warm up the animation. Future warmUp() async { - print('Warming up.'); + _logStatus('Warming up.'); // Let animation stop. await animationStops(); @@ -76,6 +74,11 @@ class DevToolsAutomater { // When warm-up finishes, inform the recorder. stopWarmingUpCallback(); - print('Warm-up finished.'); + _logStatus('Warm-up finished.'); } } + +void _logStatus(String log) { + // ignore: avoid_print, intentional test logging. + print(log); +} diff --git a/packages/devtools_app/test_benchmarks/test_infra/project_root_directory.dart b/packages/devtools_app/test_benchmarks/test_infra/project_root_directory.dart index b80dc1eb104..d1910e14a5c 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/project_root_directory.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/project_root_directory.dart @@ -2,6 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// Note: this code was copied from Flutter gallery +// https://github.com/flutter/gallery/blob/main/test_benchmarks/benchmarks/project_root_directory.dart + import 'dart:io'; import 'package:path/path.dart' as path; diff --git a/packages/devtools_app/test_benchmarks/test_infra/scroll.dart b/packages/devtools_app/test_benchmarks/test_infra/scroll.dart index 4b9d865afb3..b189ae91d16 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/scroll.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/scroll.dart @@ -2,6 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// Note: this code was copied from Flutter gallery +// https://github.com/flutter/gallery/blob/main/test_benchmarks/benchmarks/scroll.dart + import 'dart:async'; import 'package:flutter/material.dart'; From ea22ce02679ec30a06c43da4dfa5318159b53a76 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Tue, 5 Dec 2023 13:26:08 -0800 Subject: [PATCH 06/19] Use enum --- packages/devtools_app/pubspec.yaml | 2 ++ .../devtools_benchmarks_test.dart | 7 ++++-- .../test_benchmarks/test_infra/client.dart | 4 ++- .../test_benchmarks/test_infra/common.dart | 10 ++++---- .../test_infra/devtools_automator.dart | 25 ++++++++++++------- .../test_infra/devtools_recorder.dart | 9 ++++--- 6 files changed, 36 insertions(+), 21 deletions(-) diff --git a/packages/devtools_app/pubspec.yaml b/packages/devtools_app/pubspec.yaml index 7be4f3307e0..54bd80ee7de 100644 --- a/packages/devtools_app/pubspec.yaml +++ b/packages/devtools_app/pubspec.yaml @@ -177,3 +177,5 @@ dependency_overrides: path: ../devtools_test devtools_extensions: path: ../devtools_extensions + web_benchmarks: + path: ../../../packages/packages/web_benchmarks diff --git a/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart b/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart index 81261214368..32146dd49b7 100644 --- a/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart +++ b/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart @@ -47,9 +47,12 @@ Future main() async { stdout.writeln('Web benchmark tests finished.'); - expect(taskResult.scores.keys, hasLength(benchmarkList.length)); + expect( + taskResult.scores.keys, + hasLength(DevToolsBenchmark.values.length), + ); - for (final benchmarkName in benchmarkList) { + for (final benchmarkName in DevToolsBenchmark.values.map((e) => e.id)) { expect( taskResult.scores[benchmarkName], hasLength(metricList.length * valueList.length + 1), diff --git a/packages/devtools_app/test_benchmarks/test_infra/client.dart b/packages/devtools_app/test_benchmarks/test_infra/client.dart index f546acba963..8d6b7b71e36 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/client.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/client.dart @@ -10,7 +10,9 @@ import 'devtools_recorder.dart'; typedef RecorderFactory = Recorder Function(); final Map benchmarks = { - devtoolsPageLoadPerf: () => DevToolsRecorder(benchmarkName: devtoolsPageLoadPerf), + DevToolsBenchmark.navigateThroughOfflineScreens.id: () => DevToolsRecorder( + benchmark: DevToolsBenchmark.navigateThroughOfflineScreens, + ), }; /// Runs the client of the DevTools web benchmarks. diff --git a/packages/devtools_app/test_benchmarks/test_infra/common.dart b/packages/devtools_app/test_benchmarks/test_infra/common.dart index a72f9dfc830..63126dec542 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/common.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/common.dart @@ -4,8 +4,6 @@ const String devtoolsBenchmarkPrefix = 'devtools'; -const String devtoolsPageLoadPerf = '${devtoolsBenchmarkPrefix}_page_load_perf'; - /// The initial page to load upon opening the DevTools benchmark app or /// reloading it in Chrome. // @@ -14,6 +12,8 @@ const String devtoolsPageLoadPerf = '${devtoolsBenchmarkPrefix}_page_load_perf'; // found" in DevTools. const String benchmarkInitialPage = ''; -const benchmarkList = [ - devtoolsPageLoadPerf, -]; +enum DevToolsBenchmark { + navigateThroughOfflineScreens; + + String get id => '${devtoolsBenchmarkPrefix}_$name'; +} diff --git a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart index 1def560da1c..b839a115206 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart @@ -7,17 +7,18 @@ import 'package:devtools_test/helpers.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'common.dart'; import 'scroll.dart'; /// A class that automates the DevTools web app. class DevToolsAutomater { DevToolsAutomater({ - required this.benchmarkName, + required this.benchmark, required this.stopWarmingUpCallback, }); - /// The name of the current benchmark. - final String benchmarkName; + /// The current benchmark. + final DevToolsBenchmark benchmark; /// A function to call when warm-up is finished. /// @@ -45,12 +46,10 @@ class DevToolsAutomater { Future automateDevToolsGestures() async { await warmUp(); - _logStatus('==== Navigate through DevTools tabs ===='); - await navigateThroughDevToolsScreens( - controller, - runWithExpectations: false, - ); - _logStatus('==== End of navigate through DevTools tabs ===='); + switch (benchmark) { + case DevToolsBenchmark.navigateThroughOfflineScreens: + await _handleNavigateThroughOfflineScreens(); + } // At the end of the test, mark as finished. finished = true; @@ -76,6 +75,14 @@ class DevToolsAutomater { _logStatus('Warm-up finished.'); } + + Future _handleNavigateThroughOfflineScreens() async { + _logStatus('==== Navigate through offline DevTools tabs ===='); + await navigateThroughDevToolsScreens( + controller, + runWithExpectations: false, + ); + _logStatus('==== End navigate through offline DevTools tabs ===='); } void _logStatus(String log) { diff --git a/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart b/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart index 259c571a1f7..f4ecc3a637f 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart @@ -6,17 +6,18 @@ import 'package:devtools_app/initialization.dart'; import 'package:flutter/material.dart'; import 'package:web_benchmarks/client.dart'; +import 'common.dart'; import 'devtools_automator.dart'; /// A recorder that measures frame building durations for the DevTools. class DevToolsRecorder extends WidgetRecorder { - DevToolsRecorder({required this.benchmarkName}) - : super(name: benchmarkName, useCustomWarmUp: true); + DevToolsRecorder({required this.benchmark}) + : super(name: benchmark.id, useCustomWarmUp: true); /// The name of the DevTools benchmark to be run. /// /// See `common.dart` for the list of the names of all benchmarks. - final String benchmarkName; + final DevToolsBenchmark benchmark; DevToolsAutomater? _devToolsAutomator; bool get _finished => _devToolsAutomator?.finished ?? false; @@ -29,7 +30,7 @@ class DevToolsRecorder extends WidgetRecorder { @override Widget createWidget() { _devToolsAutomator = DevToolsAutomater( - benchmarkName: benchmarkName, + benchmark: benchmark, stopWarmingUpCallback: profile.stopWarmingUp, ); return _devToolsAutomator!.createWidget(); From 8df398eba52209b5a886132eb2a021a7f17846fd Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Tue, 5 Dec 2023 13:26:40 -0800 Subject: [PATCH 07/19] fix pubspec --- packages/devtools_app/pubspec.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/devtools_app/pubspec.yaml b/packages/devtools_app/pubspec.yaml index 54bd80ee7de..7be4f3307e0 100644 --- a/packages/devtools_app/pubspec.yaml +++ b/packages/devtools_app/pubspec.yaml @@ -177,5 +177,3 @@ dependency_overrides: path: ../devtools_test devtools_extensions: path: ../devtools_extensions - web_benchmarks: - path: ../../../packages/packages/web_benchmarks From 08522eb2c6a3521d491e231b3070b283c82ef47c Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Tue, 5 Dec 2023 13:29:20 -0800 Subject: [PATCH 08/19] more fixes --- .../test_benchmarks/devtools_benchmarks_test.dart | 5 +---- .../test_benchmarks/run_benchmarks.dart | 14 +++++--------- .../test_benchmarks/test_infra/common.dart | 4 ++-- .../test_infra/project_root_directory.dart | 2 +- .../test_benchmarks/test_infra/scroll.dart | 2 +- 5 files changed, 10 insertions(+), 17 deletions(-) diff --git a/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart b/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart index 32146dd49b7..5756d1b2463 100644 --- a/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart +++ b/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart @@ -39,10 +39,7 @@ Future main() async { entryPoint: 'test_benchmarks/test_infra/client.dart', useCanvasKit: true, treeShakeIcons: false, - // Pass an empty initial page so that the benchmark server does not - // attempt to load the default page 'index.html', which will show up as - // "page not found" in DevTools. - initialPage: '', + initialPage: benchmarkInitialPage, ); stdout.writeln('Web benchmark tests finished.'); diff --git a/packages/devtools_app/test_benchmarks/run_benchmarks.dart b/packages/devtools_app/test_benchmarks/run_benchmarks.dart index 7aac370b3f6..091ac493bf6 100644 --- a/packages/devtools_app/test_benchmarks/run_benchmarks.dart +++ b/packages/devtools_app/test_benchmarks/run_benchmarks.dart @@ -22,13 +22,9 @@ Future main() async { initialPage: benchmarkInitialPage, ); - stdout.writeln('Web benchmark tests finished.'); - - stdout.writeln('==== Results ===='); - - stdout.writeln( - const JsonEncoder.withIndent(' ').convert(taskResult.toJson()), - ); - - stdout.writeln('==== End of results ===='); + stdout + ..writeln('Web benchmark tests finished.') + ..writeln('==== Results ====') + ..writeln(const JsonEncoder.withIndent(' ').convert(taskResult.toJson())) + ..writeln('==== End of results ===='); } diff --git a/packages/devtools_app/test_benchmarks/test_infra/common.dart b/packages/devtools_app/test_benchmarks/test_infra/common.dart index 63126dec542..5c9078b3652 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/common.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/common.dart @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -const String devtoolsBenchmarkPrefix = 'devtools'; - /// The initial page to load upon opening the DevTools benchmark app or /// reloading it in Chrome. // @@ -12,6 +10,8 @@ const String devtoolsBenchmarkPrefix = 'devtools'; // found" in DevTools. const String benchmarkInitialPage = ''; +const String devtoolsBenchmarkPrefix = 'devtools'; + enum DevToolsBenchmark { navigateThroughOfflineScreens; diff --git a/packages/devtools_app/test_benchmarks/test_infra/project_root_directory.dart b/packages/devtools_app/test_benchmarks/test_infra/project_root_directory.dart index d1910e14a5c..968cd23ed02 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/project_root_directory.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/project_root_directory.dart @@ -28,4 +28,4 @@ Directory projectRootDirectory() { } return current; -} \ No newline at end of file +} diff --git a/packages/devtools_app/test_benchmarks/test_infra/scroll.dart b/packages/devtools_app/test_benchmarks/test_infra/scroll.dart index b189ae91d16..f49a1002dae 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/scroll.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/scroll.dart @@ -166,4 +166,4 @@ Future scrollToPosition({ } await animationStops(); -} \ No newline at end of file +} From 08a77d224c57066a1c7118ba005fc8bb892de32c Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Tue, 5 Dec 2023 14:52:35 -0800 Subject: [PATCH 09/19] cleanup --- .../devtools_benchmarks_test.dart | 2 +- .../test_infra/devtools_automator.dart | 23 ++- .../test_benchmarks/test_infra/scroll.dart | 169 ------------------ 3 files changed, 21 insertions(+), 173 deletions(-) delete mode 100644 packages/devtools_app/test_benchmarks/test_infra/scroll.dart diff --git a/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart b/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart index 5756d1b2463..e6a54a42334 100644 --- a/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart +++ b/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart @@ -28,7 +28,7 @@ final valueList = [ ]; /// Tests that the DevTools web benchmarks are run and reported correctly. -Future main() async { +void main() { test( 'Can run a web benchmark', () async { diff --git a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart index b839a115206..c229a80d717 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart @@ -2,13 +2,14 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:async'; + import 'package:devtools_app/devtools_app.dart'; import 'package:devtools_test/helpers.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; import 'common.dart'; -import 'scroll.dart'; /// A class that automates the DevTools web app. class DevToolsAutomater { @@ -77,7 +78,7 @@ class DevToolsAutomater { } Future _handleNavigateThroughOfflineScreens() async { - _logStatus('==== Navigate through offline DevTools tabs ===='); + _logStatus('Navigate through offline DevTools tabs'); await navigateThroughDevToolsScreens( controller, runWithExpectations: false, @@ -87,5 +88,21 @@ class DevToolsAutomater { void _logStatus(String log) { // ignore: avoid_print, intentional test logging. - print(log); + print('==== $log ===='); + +const Duration _animationCheckingInterval = Duration(milliseconds: 50); + +Future animationStops() async { + if (!WidgetsBinding.instance.hasScheduledFrame) return; + + final Completer stopped = Completer(); + + Timer.periodic(_animationCheckingInterval, (timer) { + if (!WidgetsBinding.instance.hasScheduledFrame) { + stopped.complete(); + timer.cancel(); + } + }); + + await stopped.future; } diff --git a/packages/devtools_app/test_benchmarks/test_infra/scroll.dart b/packages/devtools_app/test_benchmarks/test_infra/scroll.dart deleted file mode 100644 index f49a1002dae..00000000000 --- a/packages/devtools_app/test_benchmarks/test_infra/scroll.dart +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2023 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Note: this code was copied from Flutter gallery -// https://github.com/flutter/gallery/blob/main/test_benchmarks/benchmarks/scroll.dart - -import 'dart:async'; - -import 'package:flutter/material.dart'; -import 'package:flutter/rendering.dart'; -import 'package:flutter_test/flutter_test.dart'; - -const Duration _animationCheckingInterval = Duration(milliseconds: 50); - -const Duration _scrollAnimationLength = Duration(milliseconds: 300); - -Offset _absoluteTopLeft(RenderObject renderObject) => - (renderObject as RenderBox).localToGlobal(Offset.zero); - -Rect _absoluteRect(RenderObject renderObject) => - _absoluteTopLeft(renderObject) & renderObject.paintBounds.size; - -Size _windowSize(BuildContext context) => MediaQuery.of(context).size; - -Rect _windowRect(BuildContext context) => Offset.zero & _windowSize(context); - -bool _isSuperset({required Rect large, required Rect small}) => - large.top <= small.top && - large.left <= small.left && - large.bottom >= small.bottom && - large.right >= small.right; - -const _minFreeRoomRequirement = 5.0; - -/// Whether [small] is a subset of [large] and has sufficient room -/// inside [large], at the end of [large] specified by [axisDirection]. -bool _hasSufficientFreeRoom({ - required Rect large, - required Rect small, - required AxisDirection axisDirection, -}) { - if (!_isSuperset(large: large, small: small)) { - return false; - } else { - bool result; - - switch (axisDirection) { - case AxisDirection.down: - result = large.bottom - small.bottom >= _minFreeRoomRequirement; - break; - case AxisDirection.right: - result = large.right - small.right >= _minFreeRoomRequirement; - break; - case AxisDirection.left: - result = small.left - large.left >= _minFreeRoomRequirement; - break; - case AxisDirection.up: - result = small.top - large.top >= _minFreeRoomRequirement; - break; - } - - return result; - } -} - -Future animationStops() async { - if (!WidgetsBinding.instance.hasScheduledFrame) return; - - final Completer stopped = Completer(); - - Timer.periodic(_animationCheckingInterval, (timer) { - if (!WidgetsBinding.instance.hasScheduledFrame) { - stopped.complete(); - timer.cancel(); - } - }); - - await stopped.future; -} - -Future scrollUntilVisible({ - required Element element, - bool strict = false, - bool animated = true, -}) async { - final elementRenderObject = element.renderObject!; - final elementRect = _absoluteRect(elementRenderObject); - - final scrollable = Scrollable.of(element); - final viewport = RenderAbstractViewport.of(elementRenderObject); - - final visibleWindow = _absoluteRect(viewport).intersect(_windowRect(element)); - - // If there is free room between this demo button and the end of - // the scrollable, the next demo button is visible and can be tapped. - if (!strict && - _hasSufficientFreeRoom( - large: visibleWindow, - small: elementRect, - axisDirection: scrollable.axisDirection, - )) { - return; - } - - late double pixelsToBeMoved; - switch (scrollable.axisDirection) { - case AxisDirection.down: - pixelsToBeMoved = elementRect.top - visibleWindow.top; - break; - case AxisDirection.right: - pixelsToBeMoved = elementRect.left - visibleWindow.left; - break; - case AxisDirection.left: - pixelsToBeMoved = visibleWindow.right - elementRect.right; - break; - case AxisDirection.up: - pixelsToBeMoved = visibleWindow.bottom - elementRect.bottom; - break; - } - - final targetPixels = scrollable.position.pixels + pixelsToBeMoved; - final restrictedTargetPixels = targetPixels - .clamp( - scrollable.position.minScrollExtent, - scrollable.position.maxScrollExtent, - ) - .toDouble(); - - await scrollToPosition( - scrollable: scrollable, - pixels: restrictedTargetPixels, - animated: animated, - ); -} - -Future scrollToExtreme({ - required ScrollableState? scrollable, - bool toEnd = false, - bool animated = true, -}) async { - final targetPixels = toEnd - ? scrollable!.position.maxScrollExtent - : scrollable!.position.minScrollExtent; - - await scrollToPosition( - scrollable: scrollable, - pixels: targetPixels, - animated: animated, - ); -} - -Future scrollToPosition({ - required ScrollableState? scrollable, - required double pixels, - bool animated = true, -}) async { - if (animated) { - await scrollable!.position.animateTo( - pixels, - duration: _scrollAnimationLength, - curve: Curves.easeInOut, - ); - } else { - scrollable!.position.jumpTo(pixels); - } - - await animationStops(); -} From dabb35a6e44c205de32863425e5bba284d093db2 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Tue, 5 Dec 2023 15:00:08 -0800 Subject: [PATCH 10/19] add todo --- .../devtools_app/test_benchmarks/devtools_benchmarks_test.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart b/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart index e6a54a42334..767be97dbd1 100644 --- a/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart +++ b/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart @@ -81,4 +81,6 @@ void main() { }, timeout: Timeout.none, ); + + // TODO(kenz): add tests that verify performance meets some expected threshold } From 37121b74994d34b7608b4175e3354eb79e35831e Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Wed, 6 Dec 2023 08:35:52 -0800 Subject: [PATCH 11/19] bm chkpt --- packages/devtools_app/pubspec.yaml | 2 + .../test_benchmarks/test_infra/client.dart | 3 ++ .../test_benchmarks/test_infra/common.dart | 3 +- .../test_infra/devtools_automator.dart | 49 ++++++++++++++++++- .../test_infra/devtools_recorder.dart | 8 +++ 5 files changed, 62 insertions(+), 3 deletions(-) diff --git a/packages/devtools_app/pubspec.yaml b/packages/devtools_app/pubspec.yaml index 7be4f3307e0..54bd80ee7de 100644 --- a/packages/devtools_app/pubspec.yaml +++ b/packages/devtools_app/pubspec.yaml @@ -177,3 +177,5 @@ dependency_overrides: path: ../devtools_test devtools_extensions: path: ../devtools_extensions + web_benchmarks: + path: ../../../packages/packages/web_benchmarks diff --git a/packages/devtools_app/test_benchmarks/test_infra/client.dart b/packages/devtools_app/test_benchmarks/test_infra/client.dart index 8d6b7b71e36..655aaf153d5 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/client.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/client.dart @@ -13,6 +13,9 @@ final Map benchmarks = { DevToolsBenchmark.navigateThroughOfflineScreens.id: () => DevToolsRecorder( benchmark: DevToolsBenchmark.navigateThroughOfflineScreens, ), + DevToolsBenchmark.offlinePerformanceScreen.id: () => DevToolsRecorder( + benchmark: DevToolsBenchmark.offlinePerformanceScreen, + ), }; /// Runs the client of the DevTools web benchmarks. diff --git a/packages/devtools_app/test_benchmarks/test_infra/common.dart b/packages/devtools_app/test_benchmarks/test_infra/common.dart index 5c9078b3652..985cbc603cc 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/common.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/common.dart @@ -13,7 +13,8 @@ const String benchmarkInitialPage = ''; const String devtoolsBenchmarkPrefix = 'devtools'; enum DevToolsBenchmark { - navigateThroughOfflineScreens; + navigateThroughOfflineScreens, + offlinePerformanceScreen; String get id => '${devtoolsBenchmarkPrefix}_$name'; } diff --git a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart index c229a80d717..446cabbe9c4 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart @@ -5,7 +5,10 @@ import 'dart:async'; import 'package:devtools_app/devtools_app.dart'; +import 'package:devtools_app/src/screens/performance/panes/flutter_frames/flutter_frames_chart.dart'; +import 'package:devtools_app_shared/ui.dart'; import 'package:devtools_test/helpers.dart'; +import 'package:devtools_test/test_data.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -39,7 +42,7 @@ class DevToolsAutomater { // the zone set up in `lib/web_benchmarks.dart` in `flutter/flutter`. Future.delayed(safePumpDuration, automateDevToolsGestures); return DevToolsApp( - defaultScreens(), + defaultScreens(sampleData: sampleData), AnalyticsController(enabled: false, firstRun: false), ); } @@ -50,6 +53,8 @@ class DevToolsAutomater { switch (benchmark) { case DevToolsBenchmark.navigateThroughOfflineScreens: await _handleNavigateThroughOfflineScreens(); + case DevToolsBenchmark.offlinePerformanceScreen: + await _handleOfflinePerformanceScreen(); } // At the end of the test, mark as finished. @@ -83,12 +88,52 @@ class DevToolsAutomater { controller, runWithExpectations: false, ); - _logStatus('==== End navigate through offline DevTools tabs ===='); + _logStatus('End navigate through offline DevTools tabs'); + } + + Future _handleOfflinePerformanceScreen() async { + _logStatus('Loading offline performance data and interacting'); + await loadSampleData(controller, performanceLargeFileName); + + // Select a handful of frames. + final frames = find.byType(FlutterFramesChartItem); + for (var i = 0; i < 5; i++) { + await controller.tap(frames.at(i)); + await controller.pump(shortPumpDuration); + } + + // Open the Timeline Events tab. + await controller.tap(find.widgetWithText(InkWell, 'Timeline Events')); + await controller.pump(longPumpDuration); + + // Select more frames. + for (var i = 5; i < 10; i++) { + await controller.tap(frames.at(i)); + await controller.pump(shortPumpDuration); + } + + // Scroll through the frames chart. + final scrollbarFinder = find.descendant( + // ignore: invalid_use_of_visible_for_testing_member, valid use for benchmark tests. + of: find.byType(FramesChart), + matching: find.byType(Scrollbar), + ); + final scrollbar = controller.firstWidget(scrollbarFinder); + await scrollbar.controller!.animateTo( + scrollbar.controller!.position.maxScrollExtent, + duration: defaultDuration, + curve: defaultCurve, + ); + await controller.pump(); + + _logStatus('End loading offline performance data and interacting'); + } } void _logStatus(String log) { // ignore: avoid_print, intentional test logging. print('==== $log ===='); +} const Duration _animationCheckingInterval = Duration(milliseconds: 50); diff --git a/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart b/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart index f4ecc3a637f..2bb76db652c 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart @@ -2,7 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'package:devtools_app/devtools_app.dart'; import 'package:devtools_app/initialization.dart'; +import 'package:devtools_app_shared/utils.dart'; import 'package:flutter/material.dart'; import 'package:web_benchmarks/client.dart'; @@ -38,6 +40,12 @@ class DevToolsRecorder extends WidgetRecorder { @override Future run() async { + // Set the environment parameters global. + setGlobal( + DevToolsEnvironmentParameters, + ExternalDevToolsEnvironmentParameters(), + ); + // ignore: invalid_use_of_visible_for_testing_member, valid use for benchmark tests. await initializeDevTools(); return super.run(); From d1eadc87321922064510186f913093b924ae3c24 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Wed, 6 Dec 2023 09:44:21 -0800 Subject: [PATCH 12/19] Revert "bm chkpt" This reverts commit 37121b74994d34b7608b4175e3354eb79e35831e. --- packages/devtools_app/pubspec.yaml | 2 - .../test_benchmarks/test_infra/client.dart | 3 -- .../test_benchmarks/test_infra/common.dart | 3 +- .../test_infra/devtools_automator.dart | 49 +------------------ .../test_infra/devtools_recorder.dart | 8 --- 5 files changed, 3 insertions(+), 62 deletions(-) diff --git a/packages/devtools_app/pubspec.yaml b/packages/devtools_app/pubspec.yaml index 1fd5ace83e4..4d8762c98b2 100644 --- a/packages/devtools_app/pubspec.yaml +++ b/packages/devtools_app/pubspec.yaml @@ -173,5 +173,3 @@ dependency_overrides: path: ../devtools_test devtools_extensions: path: ../devtools_extensions - web_benchmarks: - path: ../../../packages/packages/web_benchmarks diff --git a/packages/devtools_app/test_benchmarks/test_infra/client.dart b/packages/devtools_app/test_benchmarks/test_infra/client.dart index 655aaf153d5..8d6b7b71e36 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/client.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/client.dart @@ -13,9 +13,6 @@ final Map benchmarks = { DevToolsBenchmark.navigateThroughOfflineScreens.id: () => DevToolsRecorder( benchmark: DevToolsBenchmark.navigateThroughOfflineScreens, ), - DevToolsBenchmark.offlinePerformanceScreen.id: () => DevToolsRecorder( - benchmark: DevToolsBenchmark.offlinePerformanceScreen, - ), }; /// Runs the client of the DevTools web benchmarks. diff --git a/packages/devtools_app/test_benchmarks/test_infra/common.dart b/packages/devtools_app/test_benchmarks/test_infra/common.dart index 985cbc603cc..5c9078b3652 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/common.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/common.dart @@ -13,8 +13,7 @@ const String benchmarkInitialPage = ''; const String devtoolsBenchmarkPrefix = 'devtools'; enum DevToolsBenchmark { - navigateThroughOfflineScreens, - offlinePerformanceScreen; + navigateThroughOfflineScreens; String get id => '${devtoolsBenchmarkPrefix}_$name'; } diff --git a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart index 446cabbe9c4..c229a80d717 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart @@ -5,10 +5,7 @@ import 'dart:async'; import 'package:devtools_app/devtools_app.dart'; -import 'package:devtools_app/src/screens/performance/panes/flutter_frames/flutter_frames_chart.dart'; -import 'package:devtools_app_shared/ui.dart'; import 'package:devtools_test/helpers.dart'; -import 'package:devtools_test/test_data.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -42,7 +39,7 @@ class DevToolsAutomater { // the zone set up in `lib/web_benchmarks.dart` in `flutter/flutter`. Future.delayed(safePumpDuration, automateDevToolsGestures); return DevToolsApp( - defaultScreens(sampleData: sampleData), + defaultScreens(), AnalyticsController(enabled: false, firstRun: false), ); } @@ -53,8 +50,6 @@ class DevToolsAutomater { switch (benchmark) { case DevToolsBenchmark.navigateThroughOfflineScreens: await _handleNavigateThroughOfflineScreens(); - case DevToolsBenchmark.offlinePerformanceScreen: - await _handleOfflinePerformanceScreen(); } // At the end of the test, mark as finished. @@ -88,52 +83,12 @@ class DevToolsAutomater { controller, runWithExpectations: false, ); - _logStatus('End navigate through offline DevTools tabs'); - } - - Future _handleOfflinePerformanceScreen() async { - _logStatus('Loading offline performance data and interacting'); - await loadSampleData(controller, performanceLargeFileName); - - // Select a handful of frames. - final frames = find.byType(FlutterFramesChartItem); - for (var i = 0; i < 5; i++) { - await controller.tap(frames.at(i)); - await controller.pump(shortPumpDuration); - } - - // Open the Timeline Events tab. - await controller.tap(find.widgetWithText(InkWell, 'Timeline Events')); - await controller.pump(longPumpDuration); - - // Select more frames. - for (var i = 5; i < 10; i++) { - await controller.tap(frames.at(i)); - await controller.pump(shortPumpDuration); - } - - // Scroll through the frames chart. - final scrollbarFinder = find.descendant( - // ignore: invalid_use_of_visible_for_testing_member, valid use for benchmark tests. - of: find.byType(FramesChart), - matching: find.byType(Scrollbar), - ); - final scrollbar = controller.firstWidget(scrollbarFinder); - await scrollbar.controller!.animateTo( - scrollbar.controller!.position.maxScrollExtent, - duration: defaultDuration, - curve: defaultCurve, - ); - await controller.pump(); - - _logStatus('End loading offline performance data and interacting'); - } + _logStatus('==== End navigate through offline DevTools tabs ===='); } void _logStatus(String log) { // ignore: avoid_print, intentional test logging. print('==== $log ===='); -} const Duration _animationCheckingInterval = Duration(milliseconds: 50); diff --git a/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart b/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart index 2bb76db652c..f4ecc3a637f 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart @@ -2,9 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'package:devtools_app/devtools_app.dart'; import 'package:devtools_app/initialization.dart'; -import 'package:devtools_app_shared/utils.dart'; import 'package:flutter/material.dart'; import 'package:web_benchmarks/client.dart'; @@ -40,12 +38,6 @@ class DevToolsRecorder extends WidgetRecorder { @override Future run() async { - // Set the environment parameters global. - setGlobal( - DevToolsEnvironmentParameters, - ExternalDevToolsEnvironmentParameters(), - ); - // ignore: invalid_use_of_visible_for_testing_member, valid use for benchmark tests. await initializeDevTools(); return super.run(); From 1249f955d23d2021780a4ef54d9f68240da9f1ff Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Wed, 6 Dec 2023 11:57:57 -0800 Subject: [PATCH 13/19] fix bracket --- .../test_benchmarks/test_infra/devtools_automator.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart index c229a80d717..56348a4b94b 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart @@ -84,6 +84,7 @@ class DevToolsAutomater { runWithExpectations: false, ); _logStatus('==== End navigate through offline DevTools tabs ===='); + } } void _logStatus(String log) { From 9996041376a644cc8d0dfafd5f2293b42c362e67 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Wed, 6 Dec 2023 12:04:02 -0800 Subject: [PATCH 14/19] fix bracket --- .../test_benchmarks/test_infra/devtools_automator.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart index 56348a4b94b..08f98bbba3c 100644 --- a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart +++ b/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart @@ -90,6 +90,7 @@ class DevToolsAutomater { void _logStatus(String log) { // ignore: avoid_print, intentional test logging. print('==== $log ===='); +} const Duration _animationCheckingInterval = Duration(milliseconds: 50); From 5920068372bd49464df484495bf67b4cf7ec8219 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Wed, 6 Dec 2023 14:39:35 -0800 Subject: [PATCH 15/19] move under benchmark --- packages/devtools_app/{test_benchmarks => benchmark}/README.md | 0 .../devtools_benchmarks_test.dart | 2 +- .../{test_benchmarks => benchmark}/run_benchmarks.dart | 2 +- .../{test_benchmarks => benchmark}/test_infra/client.dart | 0 .../{test_benchmarks => benchmark}/test_infra/common.dart | 0 .../test_infra/devtools_automator.dart | 0 .../test_infra/devtools_recorder.dart | 0 .../test_infra/project_root_directory.dart | 0 .../{test_benchmarks => benchmark}/web_bundle_size_test.dart | 0 9 files changed, 2 insertions(+), 2 deletions(-) rename packages/devtools_app/{test_benchmarks => benchmark}/README.md (100%) rename packages/devtools_app/{test_benchmarks => benchmark}/devtools_benchmarks_test.dart (97%) rename packages/devtools_app/{test_benchmarks => benchmark}/run_benchmarks.dart (94%) rename packages/devtools_app/{test_benchmarks => benchmark}/test_infra/client.dart (100%) rename packages/devtools_app/{test_benchmarks => benchmark}/test_infra/common.dart (100%) rename packages/devtools_app/{test_benchmarks => benchmark}/test_infra/devtools_automator.dart (100%) rename packages/devtools_app/{test_benchmarks => benchmark}/test_infra/devtools_recorder.dart (100%) rename packages/devtools_app/{test_benchmarks => benchmark}/test_infra/project_root_directory.dart (100%) rename packages/devtools_app/{test_benchmarks => benchmark}/web_bundle_size_test.dart (100%) diff --git a/packages/devtools_app/test_benchmarks/README.md b/packages/devtools_app/benchmark/README.md similarity index 100% rename from packages/devtools_app/test_benchmarks/README.md rename to packages/devtools_app/benchmark/README.md diff --git a/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart similarity index 97% rename from packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart rename to packages/devtools_app/benchmark/devtools_benchmarks_test.dart index 767be97dbd1..a64ebd3c7c5 100644 --- a/packages/devtools_app/test_benchmarks/devtools_benchmarks_test.dart +++ b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart @@ -36,7 +36,7 @@ void main() { final taskResult = await serveWebBenchmark( benchmarkAppDirectory: projectRootDirectory(), - entryPoint: 'test_benchmarks/test_infra/client.dart', + entryPoint: 'benchmark/test_infra/client.dart', useCanvasKit: true, treeShakeIcons: false, initialPage: benchmarkInitialPage, diff --git a/packages/devtools_app/test_benchmarks/run_benchmarks.dart b/packages/devtools_app/benchmark/run_benchmarks.dart similarity index 94% rename from packages/devtools_app/test_benchmarks/run_benchmarks.dart rename to packages/devtools_app/benchmark/run_benchmarks.dart index 091ac493bf6..1da34fde834 100644 --- a/packages/devtools_app/test_benchmarks/run_benchmarks.dart +++ b/packages/devtools_app/benchmark/run_benchmarks.dart @@ -16,7 +16,7 @@ Future main() async { final taskResult = await serveWebBenchmark( benchmarkAppDirectory: projectRootDirectory(), - entryPoint: 'test_benchmarks/test_infra/client.dart', + entryPoint: 'benchmark/test_infra/client.dart', useCanvasKit: true, treeShakeIcons: false, initialPage: benchmarkInitialPage, diff --git a/packages/devtools_app/test_benchmarks/test_infra/client.dart b/packages/devtools_app/benchmark/test_infra/client.dart similarity index 100% rename from packages/devtools_app/test_benchmarks/test_infra/client.dart rename to packages/devtools_app/benchmark/test_infra/client.dart diff --git a/packages/devtools_app/test_benchmarks/test_infra/common.dart b/packages/devtools_app/benchmark/test_infra/common.dart similarity index 100% rename from packages/devtools_app/test_benchmarks/test_infra/common.dart rename to packages/devtools_app/benchmark/test_infra/common.dart diff --git a/packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart b/packages/devtools_app/benchmark/test_infra/devtools_automator.dart similarity index 100% rename from packages/devtools_app/test_benchmarks/test_infra/devtools_automator.dart rename to packages/devtools_app/benchmark/test_infra/devtools_automator.dart diff --git a/packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart b/packages/devtools_app/benchmark/test_infra/devtools_recorder.dart similarity index 100% rename from packages/devtools_app/test_benchmarks/test_infra/devtools_recorder.dart rename to packages/devtools_app/benchmark/test_infra/devtools_recorder.dart diff --git a/packages/devtools_app/test_benchmarks/test_infra/project_root_directory.dart b/packages/devtools_app/benchmark/test_infra/project_root_directory.dart similarity index 100% rename from packages/devtools_app/test_benchmarks/test_infra/project_root_directory.dart rename to packages/devtools_app/benchmark/test_infra/project_root_directory.dart diff --git a/packages/devtools_app/test_benchmarks/web_bundle_size_test.dart b/packages/devtools_app/benchmark/web_bundle_size_test.dart similarity index 100% rename from packages/devtools_app/test_benchmarks/web_bundle_size_test.dart rename to packages/devtools_app/benchmark/web_bundle_size_test.dart From 24b0d3d146d41f707151015e92bc93fbfbfe9523 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Wed, 6 Dec 2023 14:50:59 -0800 Subject: [PATCH 16/19] fix ci script --- tool/ci/benchmark_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/ci/benchmark_tests.sh b/tool/ci/benchmark_tests.sh index e41e278cb6f..770b7a2471b 100755 --- a/tool/ci/benchmark_tests.sh +++ b/tool/ci/benchmark_tests.sh @@ -10,5 +10,5 @@ set -ex source ./tool/ci/setup.sh pushd $DEVTOOLS_DIR/packages/devtools_app -flutter test test_benchmarks/ +flutter test benchmark/ popd From ebd556f3ba429ec72a434fbb4afe8607067d391e Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Wed, 6 Dec 2023 15:20:23 -0800 Subject: [PATCH 17/19] split size and performance --- .github/workflows/build.yaml | 24 ++++++++++++++++--- tool/ci/benchmark_performance.sh | 14 +++++++++++ .../{benchmark_tests.sh => benchmark_size.sh} | 2 +- 3 files changed, 36 insertions(+), 4 deletions(-) create mode 100755 tool/ci/benchmark_performance.sh rename tool/ci/{benchmark_tests.sh => benchmark_size.sh} (85%) mode change 100755 => 100644 diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0ef356fb8cb..70e661e86e1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -256,7 +256,7 @@ jobs: DEVTOOLS_PACKAGE: devtools_extensions run: ./tool/ci/bots.sh - benchmarks: + benchmark-performance: name: benchmarks needs: flutter-prep runs-on: ubuntu-latest @@ -271,8 +271,26 @@ jobs: path: | ./tool/flutter-sdk key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} - - name: tool/ci/benchmark_tests.sh - run: ./tool/ci/benchmark_tests.sh + - name: tool/ci/benchmark_performance.sh + run: ./tool/ci/benchmark_performance.sh + + benchmark-size: + name: benchmarks + needs: flutter-prep + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - name: git clone + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac + - name: Load Cached Flutter SDK + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 + with: + path: | + ./tool/flutter-sdk + key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} + - name: tool/ci/benchmark_size.sh + run: ./tool/ci/benchmark_size.sh # TODO(https://github.com/flutter/devtools/issues/1715): add a windows compatible version of tool/ci/bots.sh diff --git a/tool/ci/benchmark_performance.sh b/tool/ci/benchmark_performance.sh new file mode 100755 index 00000000000..0eb541ff5d0 --- /dev/null +++ b/tool/ci/benchmark_performance.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Fast fail the script on failures. +set -ex + +source ./tool/ci/setup.sh + +pushd $DEVTOOLS_DIR/packages/devtools_app +flutter test benchmark/devtools_benchmarks_test.dart +popd diff --git a/tool/ci/benchmark_tests.sh b/tool/ci/benchmark_size.sh old mode 100755 new mode 100644 similarity index 85% rename from tool/ci/benchmark_tests.sh rename to tool/ci/benchmark_size.sh index 770b7a2471b..c3ca5b4adeb --- a/tool/ci/benchmark_tests.sh +++ b/tool/ci/benchmark_size.sh @@ -10,5 +10,5 @@ set -ex source ./tool/ci/setup.sh pushd $DEVTOOLS_DIR/packages/devtools_app -flutter test benchmark/ +flutter test benchmark/web_bundle_size_test.dart popd From 08765700ba01ce2e0a8ceaa535b925087ec2a75c Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Wed, 6 Dec 2023 15:33:25 -0800 Subject: [PATCH 18/19] names and readme --- .github/workflows/build.yaml | 4 +-- packages/devtools_app/benchmark/README.md | 40 ++++++++++++++++++++--- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 70e661e86e1..a0df3a40f31 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -257,7 +257,7 @@ jobs: run: ./tool/ci/bots.sh benchmark-performance: - name: benchmarks + name: benchmark-performance needs: flutter-prep runs-on: ubuntu-latest strategy: @@ -275,7 +275,7 @@ jobs: run: ./tool/ci/benchmark_performance.sh benchmark-size: - name: benchmarks + name: benchmark-size needs: flutter-prep runs-on: ubuntu-latest strategy: diff --git a/packages/devtools_app/benchmark/README.md b/packages/devtools_app/benchmark/README.md index 9a411407aef..6e4ca0995bc 100644 --- a/packages/devtools_app/benchmark/README.md +++ b/packages/devtools_app/benchmark/README.md @@ -1,12 +1,42 @@ # DevTools benchmark tests -## Running a benchmark test locally +There are two types of benchmarks that we currently support: size and performance. +- devtools_benchmarks_test.dart (measures DevTools frame times) +- web_bundle_size_test.dart (measures DevTools release build size) -To run a benchmark test locally, run: +The benchmark tests are run automatically on the CI. +See the "benchmark-performance" and "benchmark-size" jobs. + +## Running benchmark tests locally + +> [!NOTE] +> The performance and size benchmarks cannot be run concurrently +> (e.g. by running `flutter test benchmark/`). See the [#caveats](#caveats) +> section below. + +### Performance benchmarks + +To run the performance benchmark tests locally, run: +```sh +dart run run_benchmarks.dart +``` + +To run the test that verifies we can run benchmark tests, run: +```sh +flutter test benchmark/devtools_benchmarks_test.dart +``` + +### Size benchmarks + +To run the size benchmark test locally, run: ```sh -flutter test test_benchmarks/ +flutter test benchmark/web_bundle_size_test.dart ``` -## Running a benchmark test on the CI +### Caveats -The benchmark tests are run automatically on the CI. See the "benchmarks" job. +The size benchmark must be ran by itself because it actually modifies the +`devtools_app/build` folder to create and measure the release build web bundle size. +If this test is ran while other tests are running, it can affect the measurements +that the size benchmark test takes, and it can affect the DevTools build that +the other running tests are using with. From 883d9c5f14fc50e5fdc1968e36b738108c9c7448 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Wed, 6 Dec 2023 16:08:47 -0800 Subject: [PATCH 19/19] permissions --- tool/ci/benchmark_size.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tool/ci/benchmark_size.sh diff --git a/tool/ci/benchmark_size.sh b/tool/ci/benchmark_size.sh old mode 100644 new mode 100755