From cc2f844c93d5f5218568e7368ba45dc11bd1d610 Mon Sep 17 00:00:00 2001 From: John McCutchan Date: Thu, 14 Mar 2024 07:46:35 -0700 Subject: [PATCH] Add `et query tests` and `et test` commands - `et query tests` enumerates all test binaries encoded in BUILD.gn files. - `et test` (builds, then) runs a set of tests in parallel - Tests --- tools/engine_tool/lib/src/build_utils.dart | 12 +- .../lib/src/commands/build_command.dart | 18 +-- .../engine_tool/lib/src/commands/command.dart | 24 ++- .../lib/src/commands/command_runner.dart | 2 + .../lib/src/commands/lint_command.dart | 115 +-------------- .../lib/src/commands/query_command.dart | 56 ++++++- .../lib/src/commands/run_command.dart | 22 +-- .../lib/src/commands/test_command.dart | 81 ++++++++++ tools/engine_tool/lib/src/gn_utils.dart | 137 +++++++++++++++++ tools/engine_tool/lib/src/proc_utils.dart | 139 +++++++++++++++++- tools/engine_tool/test/fixtures.dart | 12 ++ tools/engine_tool/test/gn_utils_test.dart | 75 ++++++++++ .../engine_tool/test/query_command_test.dart | 76 ++++++---- tools/engine_tool/test/test_command_test.dart | 86 +++++++++++ tools/engine_tool/test/utils.dart | 114 ++++++++++++++ 15 files changed, 783 insertions(+), 186 deletions(-) create mode 100644 tools/engine_tool/lib/src/commands/test_command.dart create mode 100644 tools/engine_tool/lib/src/gn_utils.dart create mode 100644 tools/engine_tool/test/gn_utils_test.dart create mode 100644 tools/engine_tool/test/test_command_test.dart create mode 100644 tools/engine_tool/test/utils.dart diff --git a/tools/engine_tool/lib/src/build_utils.dart b/tools/engine_tool/lib/src/build_utils.dart index 0be0e587258d4..433f57ce9ab6c 100644 --- a/tools/engine_tool/lib/src/build_utils.dart +++ b/tools/engine_tool/lib/src/build_utils.dart @@ -66,11 +66,9 @@ void debugCheckBuilds(List builds) { } /// Build the build target in the environment. -Future runBuild( - Environment environment, - Build build, { - List extraGnArgs = const [], -}) async { +Future runBuild(Environment environment, Build build, + {List extraGnArgs = const [], + List targets = const []}) async { // If RBE config files aren't in the tree, then disable RBE. final String rbeConfigPath = p.join( environment.engine.srcDir.path, @@ -79,10 +77,11 @@ Future runBuild( 'rbe', ); final List gnArgs = [ - ...extraGnArgs, if (!io.Directory(rbeConfigPath).existsSync()) '--no-rbe', + ...extraGnArgs, ]; + // TODO(loic-sharma): Fetch dependencies if needed. final BuildRunner buildRunner = BuildRunner( platform: environment.platform, processRunner: environment.processRunner, @@ -91,6 +90,7 @@ Future runBuild( build: build, extraGnArgs: gnArgs, runTests: false, + extraNinjaArgs: targets, ); Spinner? spinner; diff --git a/tools/engine_tool/lib/src/commands/build_command.dart b/tools/engine_tool/lib/src/commands/build_command.dart index 366a831be2069..fc6a81dad54f8 100644 --- a/tools/engine_tool/lib/src/commands/build_command.dart +++ b/tools/engine_tool/lib/src/commands/build_command.dart @@ -17,25 +17,12 @@ final class BuildCommand extends CommandBase { }) { builds = runnableBuilds(environment, configs); debugCheckBuilds(builds); - argParser.addOption( - configFlag, - abbr: 'c', - defaultsTo: 'host_debug', - help: 'Specify the build config to use', - allowed: [ - for (final Build config in runnableBuilds(environment, configs)) - config.name, - ], - allowedHelp: { - for (final Build config in runnableBuilds(environment, configs)) - config.name: config.gn.join(' '), - }, - ); + addConfigOption(argParser, runnableBuilds(environment, configs)); argParser.addFlag( rbeFlag, defaultsTo: true, help: 'RBE is enabled by default when available. Use --no-rbe to ' - 'disable it.', + 'disable it.', ); } @@ -63,7 +50,6 @@ final class BuildCommand extends CommandBase { if (!useRbe) '--no-rbe', ]; - // TODO(loic-sharma): Fetch dependencies if needed. return runBuild(environment, build, extraGnArgs: extraGnArgs); } } diff --git a/tools/engine_tool/lib/src/commands/command.dart b/tools/engine_tool/lib/src/commands/command.dart index 3e61ff388e4a1..3224f7654cc56 100644 --- a/tools/engine_tool/lib/src/commands/command.dart +++ b/tools/engine_tool/lib/src/commands/command.dart @@ -2,17 +2,35 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'package:args/args.dart'; import 'package:args/command_runner.dart'; +import 'package:engine_build_configs/engine_build_configs.dart'; import '../environment.dart'; +import 'flags.dart'; /// The base class that all commands and subcommands should inherit from. abstract base class CommandBase extends Command { /// Constructs the base command. - CommandBase({ - required this.environment - }); + CommandBase({required this.environment}); /// The host system environment. final Environment environment; } + +/// Adds the -c (--config) option to the parser. +void addConfigOption(ArgParser parser, List builds, + {String defaultsTo = 'host_debug'}) { + parser.addOption( + configFlag, + abbr: 'c', + defaultsTo: defaultsTo, + help: 'Specify the build config to use', + allowed: [ + for (final Build config in builds) config.name, + ], + allowedHelp: { + for (final Build config in builds) config.name: config.gn.join(' '), + }, + ); +} diff --git a/tools/engine_tool/lib/src/commands/command_runner.dart b/tools/engine_tool/lib/src/commands/command_runner.dart index cf4bc92f83d54..12c1d09edb832 100644 --- a/tools/engine_tool/lib/src/commands/command_runner.dart +++ b/tools/engine_tool/lib/src/commands/command_runner.dart @@ -13,6 +13,7 @@ import 'format_command.dart'; import 'lint_command.dart'; import 'query_command.dart'; import 'run_command.dart'; +import 'test_command.dart'; const int _usageLineLength = 80; @@ -31,6 +32,7 @@ final class ToolCommandRunner extends CommandRunner { BuildCommand(environment: environment, configs: configs), RunCommand(environment: environment, configs: configs), LintCommand(environment: environment), + TestCommand(environment: environment, configs: configs), ]; commands.forEach(addCommand); diff --git a/tools/engine_tool/lib/src/commands/lint_command.dart b/tools/engine_tool/lib/src/commands/lint_command.dart index 1fa9907f0fc34..75989a4ff1c55 100644 --- a/tools/engine_tool/lib/src/commands/lint_command.dart +++ b/tools/engine_tool/lib/src/commands/lint_command.dart @@ -3,13 +3,11 @@ // found in the LICENSE file. import 'dart:io' show Directory; -import 'dart:math'; import 'package:path/path.dart' as p; import '../dart_utils.dart'; -import '../environment.dart'; -import '../logger.dart'; + import '../proc_utils.dart'; import '../worker_pool.dart'; import 'command.dart'; @@ -113,114 +111,3 @@ final class LintCommand extends CommandBase { return r ? 0 : 1; } } - -/// A WorkerPoolProgressReporter designed to work with ProcessTasks. -class ProcessTaskProgressReporter implements WorkerPoolProgressReporter { - /// Construct a new reporter. - ProcessTaskProgressReporter(this._environment); - - final Environment _environment; - Spinner? _spinner; - bool _finished = false; - int _longestName = 0; - - @override - void onRun(Set tasks) { - for (final WorkerTask task in tasks) { - _longestName = max(_longestName, task.name.length); - } - } - - @override - void onFinish() { - _finished = true; - _updateSpinner([]); - } - - List _makeProcessTaskList(WorkerPool pool) { - final List runningTasks = []; - for (final WorkerTask task in pool.running) { - if (task is! ProcessTask) { - continue; - } - runningTasks.add(task); - } - return runningTasks; - } - - @override - void onTaskStart(WorkerPool pool, WorkerTask task) { - final List running = _makeProcessTaskList(pool); - _updateSpinner(running); - } - - @override - void onTaskDone(WorkerPool pool, WorkerTask task, [Object? err]) { - final List running = _makeProcessTaskList(pool); - task as ProcessTask; - final ProcessArtifacts pa = task.processArtifacts; - final String dt = _formatDurationShort(task.runTime); - if (pa.exitCode != 0) { - final String paPath = task.processArtifactsPath; - _environment.logger.clearLine(); - _environment.logger.status( - 'FAIL: ${task.name.padLeft(_longestName)} after $dt [details in $paPath]'); - } else { - _environment.logger.clearLine(); - _environment.logger - .status('OKAY: ${task.name.padLeft(_longestName)} after $dt'); - } - _updateSpinner(running); - } - - void _updateSpinner(List tasks) { - if (_spinner != null) { - _spinner!.finish(); - _spinner = null; - } - if (_finished) { - return; - } - _environment.logger.clearLine(); - String runStatus = '['; - for (final ProcessTask pt in tasks) { - if (runStatus != '[') { - runStatus += ' '; - } - runStatus += pt.name; - } - if (tasks.isNotEmpty) { - runStatus += '...'; - } - runStatus += '] '; - _environment.logger.status('Linting $runStatus', newline: false); - _spinner = _environment.logger.startSpinner(); - } -} - -String _formatDurationShort(Duration dur) { - int micros = dur.inMicroseconds; - String r = ''; - if (micros >= Duration.microsecondsPerMinute) { - final int minutes = micros ~/ Duration.microsecondsPerMinute; - micros -= minutes * Duration.microsecondsPerMinute; - r += '${minutes}m'; - } - if (micros >= Duration.microsecondsPerSecond) { - final int seconds = micros ~/ Duration.microsecondsPerSecond; - micros -= seconds * Duration.microsecondsPerSecond; - if (r.isNotEmpty) { - r += '.'; - } - r += '${seconds}s'; - } - if (micros >= Duration.microsecondsPerMillisecond) { - final int millis = micros ~/ Duration.microsecondsPerMillisecond; - micros -= millis * Duration.microsecondsPerMillisecond; - if (r.isNotEmpty) { - r += '.'; - } - r += '${millis}ms'; - } - return r; -} diff --git a/tools/engine_tool/lib/src/commands/query_command.dart b/tools/engine_tool/lib/src/commands/query_command.dart index b70ee43ae8989..6f4c6ee367e37 100644 --- a/tools/engine_tool/lib/src/commands/query_command.dart +++ b/tools/engine_tool/lib/src/commands/query_command.dart @@ -2,8 +2,14 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:io'; + import 'package:engine_build_configs/engine_build_configs.dart'; +import 'package:path/path.dart' as p; + +import '../build_utils.dart'; +import '../gn_utils.dart'; import 'command.dart'; import 'flags.dart'; @@ -42,6 +48,10 @@ final class QueryCommand extends CommandBase { environment: environment, configs: configs, )); + addSubcommand(QueryTestsCommand( + environment: environment, + configs: configs, + )); } /// Build configurations loaded from the engine from under ci/builders. @@ -55,9 +65,9 @@ final class QueryCommand extends CommandBase { 'and tests.'; } -/// The 'query builds' command. +/// The 'query builders' command. final class QueryBuildersCommand extends CommandBase { - /// Constructs the 'query build' command. + /// Constructs the 'query builders' command. QueryBuildersCommand({ required super.environment, required this.configs, @@ -120,3 +130,45 @@ final class QueryBuildersCommand extends CommandBase { return 0; } } + +/// The query tests command. +final class QueryTestsCommand extends CommandBase { + /// Constructs the 'query tests' command. + QueryTestsCommand({ + required super.environment, + required this.configs, + }) { + builds = runnableBuilds(environment, configs); + debugCheckBuilds(builds); + addConfigOption(argParser, runnableBuilds(environment, configs)); + } + + /// Build configurations loaded from the engine from under ci/builders. + final Map configs; + + /// List of compatible builds. + late final List builds; + + @override + String get name => 'tests'; + + @override + String get description => 'Provides information about test targets'; + + @override + Future run() async { + final String configName = argResults![configFlag] as String; + final Build? build = + builds.where((Build build) => build.name == configName).firstOrNull; + if (build == null) { + environment.logger.error('Could not find config $configName'); + return 1; + } + final Map targets = await findTestTargets(environment, + Directory(p.join(environment.engine.outDir.path, build.ninja.config))); + for (final TestTarget target in targets.values) { + environment.logger.status(target.label); + } + return 0; + } +} diff --git a/tools/engine_tool/lib/src/commands/run_command.dart b/tools/engine_tool/lib/src/commands/run_command.dart index 3976aef6f769f..35e5657c58941 100644 --- a/tools/engine_tool/lib/src/commands/run_command.dart +++ b/tools/engine_tool/lib/src/commands/run_command.dart @@ -21,27 +21,15 @@ final class RunCommand extends CommandBase { }) { builds = runnableBuilds(environment, configs); debugCheckBuilds(builds); - - argParser.addOption( - configFlag, - abbr: 'c', - defaultsTo: '', - help: - 'Specify the build config to use for the target build (usually auto detected)', - allowed: [ - for (final Build build in runnableBuilds(environment, configs)) - build.name, - ], - allowedHelp: { - for (final Build build in runnableBuilds(environment, configs)) - build.name: build.gn.join(' '), - }, - ); + // We default to nothing in order to automatically detect attached devices + // and select an appropriate target from them. + addConfigOption(argParser, runnableBuilds(environment, configs), + defaultsTo: ''); argParser.addFlag( rbeFlag, defaultsTo: true, help: 'RBE is enabled by default when available. Use --no-rbe to ' - 'disable it.', + 'disable it.', ); } diff --git a/tools/engine_tool/lib/src/commands/test_command.dart b/tools/engine_tool/lib/src/commands/test_command.dart new file mode 100644 index 0000000000000..7e7d82faae0ac --- /dev/null +++ b/tools/engine_tool/lib/src/commands/test_command.dart @@ -0,0 +1,81 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:engine_build_configs/engine_build_configs.dart'; +import 'package:path/path.dart' as p; + +import '../build_utils.dart'; +import '../gn_utils.dart'; +import '../proc_utils.dart'; +import '../worker_pool.dart'; +import 'command.dart'; +import 'flags.dart'; + +/// The root 'test' command. +final class TestCommand extends CommandBase { + /// Constructs the 'test' command. + TestCommand({ + required super.environment, + required Map configs, + }) { + builds = runnableBuilds(environment, configs); + debugCheckBuilds(builds); + addConfigOption(argParser, runnableBuilds(environment, configs)); + } + + /// List of compatible builds. + late final List builds; + + @override + String get name => 'test'; + + @override + String get description => 'Runs a test target' + 'et test //flutter/fml/... # Run all test targets in `//flutter/fml/`' + 'et test //flutter/fml:fml_benchmarks # Run a single test target in `//flutter/fml/`'; + + @override + Future run() async { + final String configName = argResults![configFlag] as String; + final Build? build = + builds.where((Build build) => build.name == configName).firstOrNull; + if (build == null) { + environment.logger.error('Could not find config $configName'); + return 1; + } + + final Map allTargets = await findTestTargets( + environment, + Directory(p.join(environment.engine.outDir.path, build.ninja.config))); + final Set selectedTargets = + selectTargets(argResults!.rest, allTargets); + if (selectedTargets.isEmpty) { + environment.logger.error( + 'No build targets matched ${argResults!.rest}\nRun `et query tests` to see list of targets.'); + return 1; + } + // Chop off the '//' prefix. + final List buildTargets = selectedTargets + .map((TestTarget target) => target.label.substring('//'.length)) + .toList(); + // TODO(johnmccutchan): runBuild manipulates buildTargets and adds some + // targets listed in Build. Fix this. + final int buildExitCode = + await runBuild(environment, build, targets: buildTargets); + if (buildExitCode != 0) { + return buildExitCode; + } + final WorkerPool workerPool = + WorkerPool(environment, ProcessTaskProgressReporter(environment)); + final Set tasks = {}; + for (final TestTarget target in selectedTargets) { + final List commandLine = [target.executable.path]; + tasks.add(ProcessTask( + target.label, environment, environment.engine.srcDir, commandLine)); + } + return await workerPool.run(tasks) ? 0 : 1; + } +} diff --git a/tools/engine_tool/lib/src/gn_utils.dart b/tools/engine_tool/lib/src/gn_utils.dart new file mode 100644 index 0000000000000..ace20254ebca7 --- /dev/null +++ b/tools/engine_tool/lib/src/gn_utils.dart @@ -0,0 +1,137 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:process_runner/process_runner.dart'; + +import 'environment.dart'; +import 'proc_utils.dart'; + +/// Canonicalized build targets start with this prefix. +const String buildTargetPrefix = '//'; + +/// A suffix to build targets that recursively selects all child build targets. +const String _buildTargetGlobSuffix = '/...'; + +/// Information about a test build target. +final class TestTarget { + /// Construct a test target. + TestTarget(this.label, this.executable); + + /// The build target label. `//flutter/fml:fml_unittests` + final String label; + + /// The executable file produced after the build target is built. + final File executable; + + @override + String toString() { + return 'target=$label executable=${executable.path}'; + } +} + +/// Returns test targets for a given build directory. +Future> findTestTargets( + Environment environment, Directory buildDir) async { + final Map r = {}; + final List getLabelsCommandLine = [ + gnBinPath(environment), + 'ls', + buildDir.path, + '--type=executable', + '--testonly=true', + '--as=label', + ]; + final List getOutputsCommandLine = [ + gnBinPath(environment), + 'ls', + buildDir.path, + '--type=executable', + '--testonly=true', + '--as=output' + ]; + + // Spawn the two processes concurrently. + final Future futureLabelsResult = + environment.processRunner.runProcess(getLabelsCommandLine, + workingDirectory: environment.engine.srcDir, failOk: true); + final Future futureOutputsResult = + environment.processRunner.runProcess(getOutputsCommandLine, + workingDirectory: environment.engine.srcDir, failOk: true); + + // Await the futures, we need both to complete so the order doesn't matter. + final ProcessRunnerResult labelsResult = await futureLabelsResult; + final ProcessRunnerResult outputsResult = await futureOutputsResult; + + // Handle any process failures. + fatalIfFailed(environment, getLabelsCommandLine, labelsResult); + fatalIfFailed(environment, getOutputsCommandLine, outputsResult); + + // Extract the labels + final String rawLabels = labelsResult.stdout; + final String rawOutputs = outputsResult.stdout; + final List labels = rawLabels.split('\n'); + final List outputs = rawOutputs.split('\n'); + if (labels.length != outputs.length) { + environment.logger.fatal( + 'gn ls output is inconsistent A and B should be the same length:\nA=$labels\nB=$outputs'); + } + // Drop the empty line at the end of the output. + if (labels.isNotEmpty) { + if (labels.last.isNotEmpty || outputs.last.isNotEmpty) { + throw AssertionError('expected last line of output to be blank.'); + } + labels.removeLast(); + outputs.removeLast(); + } + for (int i = 0; i < labels.length; i++) { + final String label = labels[i]; + final String output = outputs[i]; + if (label.isEmpty) { + throw AssertionError('expected line to not be empty.'); + } + if (output.isEmpty) { + throw AssertionError('expected line to not be empty.'); + } + r[label] = TestTarget(label, File(p.join(buildDir.path, output))); + } + return r; +} + +/// Process selectors and filter allTargets for matches. +/// +/// We support: +/// 1) Exact label matches (the '//' prefix will be stripped off). +/// 2) '/...' suffix which selects all targets that match the prefix. +Set selectTargets( + List selectors, Map allTargets) { + final Set selected = {}; + for (String selector in selectors) { + if (!selector.startsWith(buildTargetPrefix)) { + // Insert the prefix when necessary. + selector = '$buildTargetPrefix$selector'; + } + final bool recursiveMatch = selector.endsWith(_buildTargetGlobSuffix); + if (recursiveMatch) { + // Remove the /... suffix. + selector = selector.substring( + 0, selector.length - _buildTargetGlobSuffix.length); + // TODO(johnmccutchan): Accelerate this by using a trie. + for (final TestTarget target in allTargets.values) { + if (target.label.startsWith(selector)) { + selected.add(target); + } + } + } else { + for (final TestTarget target in allTargets.values) { + if (target.label == selector) { + selected.add(target); + } + } + } + } + return selected; +} diff --git a/tools/engine_tool/lib/src/proc_utils.dart b/tools/engine_tool/lib/src/proc_utils.dart index cbcc226bde6be..fa43c80c4c30d 100644 --- a/tools/engine_tool/lib/src/proc_utils.dart +++ b/tools/engine_tool/lib/src/proc_utils.dart @@ -5,12 +5,14 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'dart:math'; import 'package:path/path.dart' as p; import 'package:process_runner/process_runner.dart'; import 'environment.dart'; import 'json_utils.dart'; +import 'logger.dart'; import 'worker_pool.dart'; /// Artifacts from an exited sub-process. @@ -20,6 +22,15 @@ final class ProcessArtifacts { this.cwd, this.commandLine, this.exitCode, this.stdout, this.stderr, {this.pid}); + /// Constructs an instance of ProcessArtifacts from a ProcessRunnerResult + /// and the spawning context. + factory ProcessArtifacts.fromResult( + Directory cwd, List commandLine, ProcessRunnerResult result) { + return ProcessArtifacts( + cwd, commandLine, result.exitCode, result.stdout, result.stderr, + pid: result.pid); + } + /// Constructs an instance of ProcessArtifacts from serialized JSON text. factory ProcessArtifacts.fromJson(String serialized) { final Map artifact = @@ -31,7 +42,9 @@ final class ProcessArtifacts { final int exitCode = intOfJson(artifact, 'exitCode', errors)!; final String stdout = stringOfJson(artifact, 'stdout', errors)!; final String stderr = stringOfJson(artifact, 'stderr', errors)!; - return ProcessArtifacts(cwd, commandLine, exitCode, stdout, stderr); + final int? pid = intOfJson(artifact, 'pid', errors); + return ProcessArtifacts(cwd, commandLine, exitCode, stdout, stderr, + pid: pid); } /// Constructs an instance of ProcessArtifacts from a file containing JSON. @@ -116,3 +129,127 @@ class ProcessTask extends WorkerTask { return _processArtifactsPath!; } } + +/// A WorkerPoolProgressReporter designed to work with ProcessTasks. +class ProcessTaskProgressReporter implements WorkerPoolProgressReporter { + /// Construct a new reporter. + ProcessTaskProgressReporter(this._environment); + + final Environment _environment; + Spinner? _spinner; + bool _finished = false; + int _longestName = 0; + int _doneCount = 0; + int _totalCount = 0; + + @override + void onRun(Set tasks) { + _totalCount = tasks.length; + for (final WorkerTask task in tasks) { + assert(task is ProcessTask); + _longestName = max(_longestName, task.name.length); + } + } + + @override + void onFinish() { + _finished = true; + _updateSpinner({}); + } + + @override + void onTaskStart(WorkerPool pool, WorkerTask task) { + _updateSpinner(pool.running); + } + + @override + void onTaskDone(WorkerPool pool, WorkerTask task, [Object? err]) { + _doneCount++; + task as ProcessTask; + final ProcessArtifacts pa = task.processArtifacts; + final String dt = _formatDurationShort(task.runTime); + if (pa.exitCode != 0) { + final String paPath = task.processArtifactsPath; + _environment.logger.clearLine(); + _environment.logger.status('FAIL: $dt ${task.name} [details in $paPath]'); + } else { + _environment.logger.clearLine(); + _environment.logger.status('OKAY: $dt ${task.name}'); + } + _updateSpinner(pool.running); + } + + void _updateSpinner(Set tasks) { + if (_spinner != null) { + _spinner!.finish(); + _spinner = null; + } + if (_finished) { + return; + } + _environment.logger.clearLine(); + final String taskName = tasks.isEmpty ? '' : tasks.first.name; + final String etc = tasks.length > 1 ? '... [${tasks.length}]' : ''; + _environment.logger.status( + 'Running $_doneCount/$_totalCount $taskName$etc ', + newline: false); + _spinner = _environment.logger.startSpinner(); + } + + String _formatDurationShort(Duration dur) { + int micros = dur.inMicroseconds; + String r = ''; + if (micros >= Duration.microsecondsPerMinute) { + final int minutes = micros ~/ Duration.microsecondsPerMinute; + micros -= minutes * Duration.microsecondsPerMinute; + r += '${minutes}m'; + } + if (micros >= Duration.microsecondsPerSecond) { + final int seconds = micros ~/ Duration.microsecondsPerSecond; + micros -= seconds * Duration.microsecondsPerSecond; + if (r.isNotEmpty) { + r += '.'; + } + r += '${seconds}s'; + } + if (micros >= Duration.microsecondsPerMillisecond) { + final int millis = micros ~/ Duration.microsecondsPerMillisecond; + micros -= millis * Duration.microsecondsPerMillisecond; + if (r.isNotEmpty) { + r += '.'; + } + r += '${millis}ms'; + } + return r.padLeft(15); + } +} + +/// If result.exitCode != 0, will call logger.fatal with relevant information +/// and terminate the program. +void fatalIfFailed(Environment environment, List commandLine, + ProcessRunnerResult result) { + if (result.exitCode == 0) { + return; + } + environment.logger.fatal( + 'Process "${commandLine.join(' ')}" failed exitCode=${result.exitCode}\n' + 'STDOUT:\n${result.stdout}' + 'STDERR:\n${result.stderr}'); +} + +/// Ensures that pathToBinary includes a '.exe' suffix on relevant platforms. +String exePath(Environment environment, String pathToBinary) { + String suffix = ''; + if (environment.platform.isWindows) { + suffix = '.exe'; + } + return '$pathToBinary$suffix'; +} + +/// Returns the path to the gn binary. +String gnBinPath(Environment environment) { + return exePath( + environment, + p.join(environment.engine.srcDir.path, 'flutter', 'third_party', 'gn', + 'gn')); +} diff --git a/tools/engine_tool/test/fixtures.dart b/tools/engine_tool/test/fixtures.dart index b450136d7ca37..16096fe0ad618 100644 --- a/tools/engine_tool/test/fixtures.dart +++ b/tools/engine_tool/test/fixtures.dart @@ -194,3 +194,15 @@ String attachedDevices() => ''' } ] '''; + +// NOTE: The final empty line is intentional. +String gnLsTestOutputs() => ''' +display_list_unittests +flow_unittests +fml_arc_unittests'''; + +// NOTE: The empty blank line is intentional. +String gnLsTestLabels() => ''' +//flutter/display_list:display_list_unittests +//flutter/flow:flow_unittests +//flutter/fml:fml_arc_unittests'''; diff --git a/tools/engine_tool/test/gn_utils_test.dart b/tools/engine_tool/test/gn_utils_test.dart new file mode 100644 index 0000000000000..e2a9846d73042 --- /dev/null +++ b/tools/engine_tool/test/gn_utils_test.dart @@ -0,0 +1,75 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io' as io; + +import 'package:engine_repo_tools/engine_repo_tools.dart'; +import 'package:engine_tool/src/environment.dart'; +import 'package:engine_tool/src/gn_utils.dart'; +import 'package:litetest/litetest.dart'; + +import 'utils.dart'; + +void main() { + final Engine engine; + try { + engine = Engine.findWithin(); + } catch (e) { + io.stderr.writeln(e); + io.exitCode = 1; + return; + } + + final List cannedProcesses = [ + CannedProcess((List command) => command.contains('--as=label'), + stdout: ''' +//flutter/display_list:display_list_unittests +//flutter/flow:flow_unittests +//flutter/fml:fml_arc_unittests +'''), + CannedProcess((List command) => command.contains('--as=output'), + stdout: ''' +display_list_unittests +flow_unittests +fml_arc_unittests +''') + ]; + + test('find test targets', () async { + final TestEnvironment testEnvironment = + TestEnvironment(engine, cannedProcesses: cannedProcesses); + final Environment env = testEnvironment.environment; + final Map testTargets = + await findTestTargets(env, engine.outDir); + expect(testTargets.length, equals(3)); + expect(testTargets['//flutter/display_list:display_list_unittests'], + notEquals(null)); + expect( + testTargets['//flutter/display_list:display_list_unittests']! + .executable + .path, + endsWith('display_list_unittests')); + }); + + test('process queue failure', () async { + final TestEnvironment testEnvironment = + TestEnvironment(engine, cannedProcesses: cannedProcesses); + final Environment env = testEnvironment.environment; + final Map testTargets = + await findTestTargets(env, engine.outDir); + expect(selectTargets(['//...'], testTargets).length, equals(3)); + expect( + selectTargets(['//flutter/display_list'], testTargets).length, + equals(0)); + expect( + selectTargets(['//flutter/display_list/...'], testTargets) + .length, + equals(1)); + expect( + selectTargets(['//flutter/display_list:display_list_unittests'], + testTargets) + .length, + equals(1)); + }); +} diff --git a/tools/engine_tool/test/query_command_test.dart b/tools/engine_tool/test/query_command_test.dart index 4abed0318507c..b570ad2e28dac 100644 --- a/tools/engine_tool/test/query_command_test.dart +++ b/tools/engine_tool/test/query_command_test.dart @@ -10,14 +10,11 @@ import 'package:engine_build_configs/engine_build_configs.dart'; import 'package:engine_repo_tools/engine_repo_tools.dart'; import 'package:engine_tool/src/commands/command_runner.dart'; import 'package:engine_tool/src/environment.dart'; -import 'package:engine_tool/src/logger.dart'; import 'package:litetest/litetest.dart'; import 'package:logging/logging.dart' as log; -import 'package:platform/platform.dart'; -import 'package:process_fakes/process_fakes.dart'; -import 'package:process_runner/process_runner.dart'; import 'fixtures.dart' as fixtures; +import 'utils.dart'; void main() { final Engine engine; @@ -54,27 +51,29 @@ void main() { 'win_test_config': winTestConfig, }; - Environment linuxEnv(Logger logger) { - return Environment( - abi: ffi.Abi.linuxX64, - engine: engine, - platform: FakePlatform( - operatingSystem: Platform.linux, - resolvedExecutable: io.Platform.resolvedExecutable), - processRunner: ProcessRunner( - processManager: FakeProcessManager(), - ), - logger: logger, - ); - } - List stringsFromLogs(List logs) { return logs.map((log.LogRecord r) => r.message).toList(); } + final List cannedProcesses = [ + CannedProcess((List command) => command.contains('--as=label'), + stdout: ''' +//flutter/display_list:display_list_unittests +//flutter/flow:flow_unittests +//flutter/fml:fml_arc_unittests +'''), + CannedProcess((List command) => command.contains('--as=output'), + stdout: ''' +display_list_unittests +flow_unittests +fml_arc_unittests +''') + ]; + test('query command returns builds for the host platform.', () async { - final Logger logger = Logger.test(); - final Environment env = linuxEnv(logger); + final TestEnvironment testEnvironment = TestEnvironment(engine, + abi: ffi.Abi.linuxX64, cannedProcesses: cannedProcesses); + final Environment env = testEnvironment.environment; final ToolCommandRunner runner = ToolCommandRunner( environment: env, configs: configs, @@ -85,7 +84,7 @@ void main() { ]); expect(result, equals(0)); expect( - stringsFromLogs(logger.testLogs), + stringsFromLogs(env.logger.testLogs), equals([ 'Add --verbose to see detailed information about each builder\n', '\n', @@ -105,8 +104,9 @@ void main() { test('query command with --builder returns only from the named builder.', () async { - final Logger logger = Logger.test(); - final Environment env = linuxEnv(logger); + final TestEnvironment testEnvironment = TestEnvironment(engine, + abi: ffi.Abi.linuxX64, cannedProcesses: cannedProcesses); + final Environment env = testEnvironment.environment; final ToolCommandRunner runner = ToolCommandRunner( environment: env, configs: configs, @@ -119,7 +119,7 @@ void main() { ]); expect(result, equals(0)); expect( - stringsFromLogs(logger.testLogs), + stringsFromLogs(env.logger.testLogs), equals([ 'Add --verbose to see detailed information about each builder\n', '\n', @@ -132,8 +132,9 @@ void main() { }); test('query command with --all returns all builds.', () async { - final Logger logger = Logger.test(); - final Environment env = linuxEnv(logger); + final TestEnvironment testEnvironment = TestEnvironment(engine, + abi: ffi.Abi.linuxX64, cannedProcesses: cannedProcesses); + final Environment env = testEnvironment.environment; final ToolCommandRunner runner = ToolCommandRunner( environment: env, configs: configs, @@ -145,8 +146,29 @@ void main() { ]); expect(result, equals(0)); expect( - logger.testLogs.length, + env.logger.testLogs.length, equals(30), ); }); + + test('query tests', () async { + final TestEnvironment testEnvironment = TestEnvironment(engine, + abi: ffi.Abi.linuxX64, cannedProcesses: cannedProcesses); + final Environment env = testEnvironment.environment; + final ToolCommandRunner runner = ToolCommandRunner( + environment: env, + configs: configs, + ); + final int result = await runner.run([ + 'query', + 'tests', + ]); + expect(result, equals(0)); + expect( + env.logger.testLogs.length, + equals(3), + ); + expect(env.logger.testLogs[0].message, + startsWith('//flutter/display_list:display_list_unittests')); + }); } diff --git a/tools/engine_tool/test/test_command_test.dart b/tools/engine_tool/test/test_command_test.dart new file mode 100644 index 0000000000000..d7629cedbeb79 --- /dev/null +++ b/tools/engine_tool/test/test_command_test.dart @@ -0,0 +1,86 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert' as convert; +import 'dart:ffi' as ffi show Abi; +import 'dart:io' as io; + +import 'package:engine_build_configs/engine_build_configs.dart'; +import 'package:engine_repo_tools/engine_repo_tools.dart'; +import 'package:engine_tool/src/commands/command_runner.dart'; +import 'package:engine_tool/src/environment.dart'; +import 'package:litetest/litetest.dart'; + +import 'fixtures.dart' as fixtures; +import 'utils.dart'; + +void main() { + final Engine engine; + try { + engine = Engine.findWithin(); + } catch (e) { + io.stderr.writeln(e); + io.exitCode = 1; + return; + } + + final BuilderConfig linuxTestConfig = BuilderConfig.fromJson( + path: 'ci/builders/linux_test_config.json', + map: convert.jsonDecode(fixtures.testConfig('Linux')) + as Map, + ); + + final BuilderConfig macTestConfig = BuilderConfig.fromJson( + path: 'ci/builders/mac_test_config.json', + map: convert.jsonDecode(fixtures.testConfig('Mac-12')) + as Map, + ); + + final BuilderConfig winTestConfig = BuilderConfig.fromJson( + path: 'ci/builders/win_test_config.json', + map: convert.jsonDecode(fixtures.testConfig('Windows-11')) + as Map, + ); + + final Map configs = { + 'linux_test_config': linuxTestConfig, + 'linux_test_config2': linuxTestConfig, + 'mac_test_config': macTestConfig, + 'win_test_config': winTestConfig, + }; + + final List cannedProcesses = [ + CannedProcess((List command) => command.contains('--as=label'), + stdout: ''' +//flutter/display_list:display_list_unittests +//flutter/flow:flow_unittests +//flutter/fml:fml_arc_unittests +'''), + CannedProcess((List command) => command.contains('--as=output'), + stdout: ''' +display_list_unittests +flow_unittests +fml_arc_unittests +''') + ]; + + test('test command executes test', () async { + final TestEnvironment testEnvironment = TestEnvironment(engine, + abi: ffi.Abi.linuxX64, cannedProcesses: cannedProcesses); + final Environment env = testEnvironment.environment; + final ToolCommandRunner runner = ToolCommandRunner( + environment: env, + configs: configs, + ); + final int result = await runner.run([ + 'test', + '//flutter/display_list:display_list_unittests', + ]); + expect(result, equals(0)); + expect(testEnvironment.processHistory.length, greaterThan(3)); + final int offset = testEnvironment.processHistory.length - 1; + expect(testEnvironment.processHistory[offset].command[0], + endsWith('display_list_unittests')); + }); +} diff --git a/tools/engine_tool/test/utils.dart b/tools/engine_tool/test/utils.dart new file mode 100644 index 0000000000000..02abc466e10f2 --- /dev/null +++ b/tools/engine_tool/test/utils.dart @@ -0,0 +1,114 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:ffi' as ffi show Abi; +import 'dart:io' as io; + +import 'package:engine_repo_tools/engine_repo_tools.dart'; +import 'package:engine_tool/src/environment.dart'; +import 'package:engine_tool/src/logger.dart'; +import 'package:platform/platform.dart'; +import 'package:process_fakes/process_fakes.dart'; +import 'package:process_runner/process_runner.dart'; + +/// Each CannedProcess has a command matcher and the result of an executed +/// process. The matcher is used to determine when to use a registered +/// CannedProcess. +class CannedProcess { + CannedProcess( + this.commandMatcher, { + int exitCode = 0, + String stdout = '', + String stderr = '', + }) : _exitCode = exitCode, + _stdout = stdout, + _stderr = stderr; + + final bool Function(List command) commandMatcher; + final int _exitCode; + final String _stdout; + final String _stderr; + + FakeProcess get fakeProcess { + return FakeProcess(exitCode: _exitCode, stdout: _stdout, stderr: _stderr); + } +} + +/// ExecutedProcess includes the command and the result. +class ExecutedProcess { + ExecutedProcess(this.command, this.result); + final List command; + final FakeProcess result; + + @override + String toString() { + return command.join(' '); + } +} + +/// TestEnvironment includes an Environment with some test-specific extras. +class TestEnvironment { + TestEnvironment( + Engine engine, { + Logger? logger, + ffi.Abi abi = ffi.Abi.macosArm64, + this.cannedProcesses = const [], + }) { + logger ??= Logger.test(); + environment = Environment( + abi: abi, + engine: engine, + platform: FakePlatform( + operatingSystem: _operatingSystemForAbi(abi), + resolvedExecutable: io.Platform.resolvedExecutable), + processRunner: ProcessRunner( + processManager: FakeProcessManager(onStart: (List command) { + final FakeProcess processResult = + _getCannedResult(command, cannedProcesses); + processHistory.add(ExecutedProcess(command, processResult)); + return processResult; + }, onRun: (List command) { + throw UnimplementedError('onRun'); + })), + logger: logger, + ); + } + + /// Environment. + late final Environment environment; + + /// List of CannedProcesses that are registered in this environment. + final List cannedProcesses; + + /// A history of all executed processes. + final List processHistory = []; +} + +String _operatingSystemForAbi(ffi.Abi abi) { + switch (abi) { + case ffi.Abi.linuxArm: + case ffi.Abi.linuxArm64: + case ffi.Abi.linuxIA32: + case ffi.Abi.linuxX64: + case ffi.Abi.linuxRiscv32: + case ffi.Abi.linuxRiscv64: + return Platform.linux; + case ffi.Abi.macosArm64: + case ffi.Abi.macosX64: + return Platform.macOS; + default: + throw UnimplementedError('Unhandled abi=$abi'); + } +} + +FakeProcess _getCannedResult( + List command, List cannedProcesses) { + for (final CannedProcess cp in cannedProcesses) { + final bool matched = cp.commandMatcher(command); + if (matched) { + return cp.fakeProcess; + } + } + return FakeProcess(); +}