diff --git a/.cirrus.yml b/.cirrus.yml index 14537b2c7b1d2..2203c8b064cb1 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -36,9 +36,12 @@ task: cd $ENGINE_PATH/src ./flutter/testing/run_tests.sh host_debug_unopt test_web_engine_script: | + google-chrome --version + cd $ENGINE_PATH/src/flutter/web_sdk/web_engine_tester + $ENGINE_PATH/src/out/host_debug_unopt/dart-sdk/bin/pub get cd $ENGINE_PATH/src/flutter/lib/web_ui $ENGINE_PATH/src/out/host_debug_unopt/dart-sdk/bin/pub get - $ENGINE_PATH/src/out/host_debug_unopt/dart-sdk/bin/dart dev/test.dart + CHROME_NO_SANDBOX=true $ENGINE_PATH/src/out/host_debug_unopt/dart-sdk/bin/dart dev/test.dart fetch_framework_script: | mkdir -p $FRAMEWORK_PATH cd $FRAMEWORK_PATH diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 8e21402b2bb5b..80398d737f6bb 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -346,7 +346,6 @@ FILE: ../../../flutter/lib/ui/window/viewport_metrics.cc FILE: ../../../flutter/lib/ui/window/viewport_metrics.h FILE: ../../../flutter/lib/ui/window/window.cc FILE: ../../../flutter/lib/ui/window/window.h -FILE: ../../../flutter/lib/web_ui/dev/test.dart FILE: ../../../flutter/lib/web_ui/lib/assets/houdini_painter.js FILE: ../../../flutter/lib/web_ui/lib/src/engine.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/alarm_clock.dart diff --git a/ci/licenses_golden/tool_signature b/ci/licenses_golden/tool_signature index ec6b163fb143d..6b401d8521d80 100644 --- a/ci/licenses_golden/tool_signature +++ b/ci/licenses_golden/tool_signature @@ -1,2 +1,2 @@ -Signature: 24e0fa6ad08ae80380158c2b4ba44c65 +Signature: d510e80277a674c258a08394950ac485 diff --git a/lib/web_ui/.gitignore b/lib/web_ui/.gitignore new file mode 100644 index 0000000000000..567609b1234a9 --- /dev/null +++ b/lib/web_ui/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/lib/web_ui/build.yaml b/lib/web_ui/build.yaml new file mode 100644 index 0000000000000..86391d013bb95 --- /dev/null +++ b/lib/web_ui/build.yaml @@ -0,0 +1,11 @@ +targets: + $default: + builders: + build_web_compilers|entrypoint: + options: + compiler: dart2js + dart2js_args: + - --no-minify + - --enable-asserts + generate_for: + - test/**.dart diff --git a/lib/web_ui/dev/environment.dart b/lib/web_ui/dev/environment.dart new file mode 100644 index 0000000000000..46a2542138062 --- /dev/null +++ b/lib/web_ui/dev/environment.dart @@ -0,0 +1,167 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io' as io; +import 'package:args/args.dart' as args; +import 'package:path/path.dart' as pathlib; + +/// Contains various environment variables, such as common file paths and command-line options. +Environment get environment { + _environment ??= Environment(); + return _environment; +} +Environment _environment; + +args.ArgParser get _argParser { + return args.ArgParser() + ..addMultiOption( + 'target', + abbr: 't', + help: 'The path to the target to run. When omitted, runs all targets.', + ) + ..addMultiOption( + 'shard', + abbr: 's', + help: 'The category of tasks to run.', + ) + ..addFlag( + 'debug', + help: 'Pauses the browser before running a test, giving you an ' + 'opportunity to add breakpoints or inspect loaded code before ' + 'running the code.', + ); +} + +/// Contains various environment variables, such as common file paths and command-line options. +class Environment { + /// Command-line arguments. + static List commandLineArguments; + + factory Environment() { + if (commandLineArguments == null) { + io.stderr.writeln('Command-line arguments not set.'); + io.exit(1); + } + + final args.ArgResults options = _argParser.parse(commandLineArguments); + final List shards = options['shard']; + final bool isDebug = options['debug']; + final List targets = options['target']; + + final io.File self = io.File.fromUri(io.Platform.script); + final io.Directory engineSrcDir = self.parent.parent.parent.parent.parent; + final io.Directory outDir = io.Directory(pathlib.join(engineSrcDir.path, 'out')); + final io.Directory hostDebugUnoptDir = io.Directory(pathlib.join(outDir.path, 'host_debug_unopt')); + final io.Directory dartSdkDir = io.Directory(pathlib.join(hostDebugUnoptDir.path, 'dart-sdk')); + final io.Directory webUiRootDir = io.Directory(pathlib.join(engineSrcDir.path, 'flutter', 'lib', 'web_ui')); + + for (io.Directory expectedDirectory in [engineSrcDir, outDir, hostDebugUnoptDir, dartSdkDir, webUiRootDir]) { + if (!expectedDirectory.existsSync()) { + io.stderr.writeln('$expectedDirectory does not exist.'); + io.exit(1); + } + } + + return Environment._( + self: self, + webUiRootDir: webUiRootDir, + engineSrcDir: engineSrcDir, + outDir: outDir, + hostDebugUnoptDir: hostDebugUnoptDir, + dartSdkDir: dartSdkDir, + requestedShards: shards, + isDebug: isDebug, + targets: targets, + ); + } + + Environment._({ + this.self, + this.webUiRootDir, + this.engineSrcDir, + this.outDir, + this.hostDebugUnoptDir, + this.dartSdkDir, + this.requestedShards, + this.isDebug, + this.targets, + }); + + /// The Dart script that's currently running. + final io.File self; + + /// Path to the "web_ui" package sources. + final io.Directory webUiRootDir; + + /// Path to the engine's "src" directory. + final io.Directory engineSrcDir; + + /// Path to the engine's "out" directory. + /// + /// This is where you'll find the ninja output, such as the Dart SDK. + final io.Directory outDir; + + /// The "host_debug_unopt" build of the Dart SDK. + final io.Directory hostDebugUnoptDir; + + /// The root of the Dart SDK. + final io.Directory dartSdkDir; + + /// Shards specified on the command-line. + final List requestedShards; + + /// Whether to start the browser in debug mode. + /// + /// In this mode the browser pauses before running the test to allow + /// you set breakpoints or inspect the code. + final bool isDebug; + + /// Paths to targets to run, e.g. a single test. + final List targets; + + /// The "dart" executable file. + String get dartExecutable => pathlib.join(dartSdkDir.path, 'bin', 'dart'); + + /// The "pub" executable file. + String get pubExecutable => pathlib.join(dartSdkDir.path, 'bin', 'pub'); + + /// The "dart2js" executable file. + String get dart2jsExecutable => pathlib.join(dartSdkDir.path, 'bin', 'dart2js'); + + /// Path to where github.com/flutter/engine is checked out inside the engine workspace. + io.Directory get flutterDirectory => io.Directory(pathlib.join(engineSrcDir.path, 'flutter')); + io.Directory get webSdkRootDir => io.Directory(pathlib.join( + flutterDirectory.path, + 'web_sdk', + )); + + /// Path to the "web_engine_tester" package. + io.Directory get goldenTesterRootDir => io.Directory(pathlib.join( + webSdkRootDir.path, + 'web_engine_tester', + )); + + /// Path to the "build" directory, generated by "package:build_runner". + /// + /// This is where compiled output goes. + io.Directory get webUiBuildDir => io.Directory(pathlib.join( + webUiRootDir.path, + 'build', + )); + + /// Path to the ".dart_tool" directory, generated by various Dart tools. + io.Directory get webUiDartToolDir => io.Directory(pathlib.join( + webUiRootDir.path, + '.dart_tool', + )); +} + +String _which(String executable) { + final io.ProcessResult result = io.Process.runSync('which', [executable]); + if (result.exitCode != 0) { + io.stderr.writeln(result.stderr); + io.exit(result.exitCode); + } + return result.stdout; +} diff --git a/lib/web_ui/dev/test.dart b/lib/web_ui/dev/test.dart index 15ce692c85ce9..5654e28a7d922 100644 --- a/lib/web_ui/dev/test.dart +++ b/lib/web_ui/dev/test.dart @@ -2,14 +2,22 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:async'; import 'dart:io' as io; import 'package:path/path.dart' as pathlib; -final Environment environment = Environment(); +import 'environment.dart'; +import 'test_runner.dart'; -void main() async { +// A "shard" is a named subset of tasks this script runs. If not specified, +// it runs all shards. That's what we do on CI. +const Map _kShardNameToCode = { + 'licenses': _checkLicenseHeaders, + 'tests': runTests, +}; + +void main(List args) async { + Environment.commandLineArguments = args; if (io.Directory.current.absolute.path != environment.webUiRootDir.absolute.path) { io.stderr.writeln('Current directory is not the root of the web_ui package directory.'); io.stderr.writeln('web_ui directory is: ${environment.webUiRootDir.absolute.path}'); @@ -17,8 +25,27 @@ void main() async { io.exit(1); } - await _checkLicenseHeaders(); - await _runTests(); + _copyAhemFontIntoWebUi(); + + final List shardsToRun = environment.requestedShards.isNotEmpty + ? environment.requestedShards + : _kShardNameToCode.keys.toList(); + + for (String shard in shardsToRun) { + print('Running shard $shard'); + if (!_kShardNameToCode.containsKey(shard)) { + io.stderr.writeln(''' +ERROR: + Unsupported test shard: $shard. + Supported test shards: ${_kShardNameToCode.keys.join(', ')} +TESTS FAILED +'''.trim()); + io.exit(1); + } + await _kShardNameToCode[shard](); + } + // Sometimes the Dart VM refuses to quit. + io.exit(io.exitCode); } void _checkLicenseHeaders() { @@ -26,7 +53,6 @@ void _checkLicenseHeaders() { _expect(allSourceFiles.isNotEmpty, 'Dart source listing of ${environment.webUiRootDir.path} must not be empty.'); final List allDartPaths = allSourceFiles.map((f) => f.path).toList(); - print(allDartPaths.join('\n')); for (String expectedDirectory in const ['lib', 'test', 'dev', 'tool']) { final String expectedAbsoluteDirectory = pathlib.join(environment.webUiRootDir.path, expectedDirectory); @@ -37,6 +63,7 @@ void _checkLicenseHeaders() { } allSourceFiles.forEach(_expectLicenseHeader); + print('License headers OK!'); } final _copyRegex = RegExp(r'// Copyright 2013 The Flutter Authors\. All rights reserved\.'); @@ -69,123 +96,27 @@ List _flatListSourceFiles(io.Directory directory) { return directory .listSync(recursive: true) .whereType() - .where((f) => f.path.endsWith('.dart') || f.path.endsWith('.js')) + .where((f) { + if (!f.path.endsWith('.dart') && !f.path.endsWith('.js')) { + // Not a source file we're checking. + return false; + } + if (pathlib.isWithin(environment.webUiBuildDir.path, f.path) || + pathlib.isWithin(environment.webUiDartToolDir.path, f.path)) { + // Generated files. + return false; + } + return true; + }) .toList(); } -Future _runTests() async { - _copyAhemFontIntoWebUi(); - - final List testFiles = io.Directory('test') - .listSync(recursive: true) - .whereType() - .map((io.File file) => file.path) - .where((String path) => path.endsWith('_test.dart')) - .toList(); - - final io.Process pubRunTest = await io.Process.start( - environment.pubExecutable, - [ - 'run', - 'test', - '--preset=cirrus', - '--platform=chrome', - ...testFiles, - ], - ); - - final StreamSubscription stdoutSub = pubRunTest.stdout.listen(io.stdout.add); - final StreamSubscription stderrSub = pubRunTest.stderr.listen(io.stderr.add); - final int exitCode = await pubRunTest.exitCode; - stdoutSub.cancel(); - stderrSub.cancel(); - - if (exitCode != 0) { - io.stderr.writeln('Test process exited with exit code $exitCode'); - io.exit(1); - } -} - void _copyAhemFontIntoWebUi() { - final io.File sourceAhemTtf = io.File(pathlib.join(environment.flutterDirectory.path, 'third_party', 'txt', 'third_party', 'fonts', 'ahem.ttf')); - final String destinationAhemTtfPath = pathlib.join(environment.webUiRootDir.path, 'lib', 'assets', 'ahem.ttf'); + final io.File sourceAhemTtf = io.File(pathlib.join( + environment.flutterDirectory.path, 'third_party', 'txt', 'third_party', 'fonts', 'ahem.ttf' + )); + final String destinationAhemTtfPath = pathlib.join( + environment.webUiRootDir.path, 'lib', 'assets', 'ahem.ttf' + ); sourceAhemTtf.copySync(destinationAhemTtfPath); } - -class Environment { - factory Environment() { - final io.File self = io.File.fromUri(io.Platform.script); - final io.Directory webUiRootDir = self.parent.parent; - final io.Directory engineSrcDir = webUiRootDir.parent.parent.parent; - final io.Directory outDir = io.Directory(pathlib.join(engineSrcDir.path, 'out')); - final io.Directory hostDebugUnoptDir = io.Directory(pathlib.join(outDir.path, 'host_debug_unopt')); - final String dartExecutable = pathlib.canonicalize(io.File(_which(io.Platform.executable)).absolute.path); - final io.Directory dartSdkDir = io.File(dartExecutable).parent.parent; - - // Googlers frequently have their Dart SDK misconfigured for open-source projects. Let's help them out. - if (dartExecutable.startsWith('/usr/lib/google-dartlang')) { - io.stderr.writeln('ERROR: Using unsupported version of the Dart SDK: $dartExecutable'); - io.exit(1); - } - - return Environment._( - self: self, - webUiRootDir: webUiRootDir, - engineSrcDir: engineSrcDir, - outDir: outDir, - hostDebugUnoptDir: hostDebugUnoptDir, - dartExecutable: dartExecutable, - dartSdkDir: dartSdkDir, - ); - } - - Environment._({ - this.self, - this.webUiRootDir, - this.engineSrcDir, - this.outDir, - this.hostDebugUnoptDir, - this.dartSdkDir, - this.dartExecutable, - }); - - final io.File self; - final io.Directory webUiRootDir; - final io.Directory engineSrcDir; - final io.Directory outDir; - final io.Directory hostDebugUnoptDir; - final io.Directory dartSdkDir; - final String dartExecutable; - - String get pubExecutable => pathlib.join(dartSdkDir.path, 'bin', 'pub'); - io.Directory get flutterDirectory => io.Directory(pathlib.join(engineSrcDir.path, 'flutter')); - - @override - String toString() { - return ''' -runTest.dart script: - ${self.path} -web_ui directory: - ${webUiRootDir.path} -engine/src directory: - ${engineSrcDir.path} -out directory: - ${outDir.path} -out/host_debug_unopt directory: - ${hostDebugUnoptDir.path} -Dart SDK directory: - ${dartSdkDir.path} -dart executable: - ${dartExecutable} -'''; - } -} - -String _which(String executable) { - final io.ProcessResult result = io.Process.runSync('which', [executable]); - if (result.exitCode != 0) { - io.stderr.writeln(result.stderr); - io.exit(result.exitCode); - } - return result.stdout; -} diff --git a/lib/web_ui/dev/test_platform.dart b/lib/web_ui/dev/test_platform.dart new file mode 100644 index 0000000000000..f327f28be2a1c --- /dev/null +++ b/lib/web_ui/dev/test_platform.dart @@ -0,0 +1,888 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; +import 'dart:typed_data'; + +import 'package:async/async.dart'; +import 'package:http_multi_server/http_multi_server.dart'; +import 'package:package_resolver/package_resolver.dart'; +import 'package:path/path.dart' as p; +import 'package:pedantic/pedantic.dart'; +import 'package:pool/pool.dart'; +import 'package:shelf/shelf.dart' as shelf; +import 'package:shelf/shelf_io.dart' as shelf_io; +import 'package:shelf_static/shelf_static.dart'; +import 'package:shelf_web_socket/shelf_web_socket.dart'; +import 'package:shelf_packages_handler/shelf_packages_handler.dart'; +import 'package:stack_trace/stack_trace.dart'; +import 'package:stream_channel/stream_channel.dart'; +import 'package:typed_data/typed_buffers.dart'; +import 'package:web_socket_channel/web_socket_channel.dart'; + +import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports +import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports +import 'package:test_core/src/runner/runner_suite.dart'; // ignore: implementation_imports +import 'package:test_core/src/runner/platform.dart'; // ignore: implementation_imports +import 'package:test_core/src/util/stack_trace_mapper.dart'; // ignore: implementation_imports +import 'package:test_api/src/utils.dart'; // ignore: implementation_imports +import 'package:test_core/src/runner/suite.dart'; // ignore: implementation_imports +import 'package:test_core/src/runner/plugin/platform_helpers.dart'; // ignore: implementation_imports +import 'package:test_core/src/runner/environment.dart'; // ignore: implementation_imports + +import 'package:test_core/src/util/io.dart'; // ignore: implementation_imports +import 'package:test_core/src/runner/configuration.dart'; // ignore: implementation_imports +import 'package:test_core/src/runner/load_exception.dart'; // ignore: implementation_imports +import 'package:webkit_inspection_protocol/webkit_inspection_protocol.dart' + as wip; + +import 'environment.dart' as env; + +/// The port number Chrome exposes for debugging. +const int _kChromeDevtoolsPort = 12345; + +class BrowserPlatform extends PlatformPlugin { + /// Starts the server. + /// + /// [root] is the root directory that the server should serve. It defaults to + /// the working directory. + static Future start({String root}) async { + var server = shelf_io.IOServer(await HttpMultiServer.loopback(0)); + return BrowserPlatform._( + server, + Configuration.current, + p.fromUri(await Isolate.resolvePackageUri( + Uri.parse('package:test/src/runner/browser/static/favicon.ico'))), + root: root); + } + + /// The test runner configuration. + final Configuration _config; + + /// The underlying server. + final shelf.Server _server; + + /// A randomly-generated secret. + /// + /// This is used to ensure that other users on the same system can't snoop + /// on data being served through this server. + final _secret = Uri.encodeComponent(randomBase64(24)); + + /// The URL for this server. + Uri get url => _server.url.resolve(_secret + '/'); + + /// A [OneOffHandler] for servicing WebSocket connections for + /// [BrowserManager]s. + /// + /// This is one-off because each [BrowserManager] can only connect to a single + /// WebSocket, + final OneOffHandler _webSocketHandler = OneOffHandler(); + + /// A [PathHandler] used to serve compiled JS. + final PathHandler _jsHandler = PathHandler(); + + /// The root directory served statically by this server. + final String _root; + + /// The HTTP client to use when caching JS files in `pub serve`. + final HttpClient _http; + + /// Whether [close] has been called. + bool get _closed => _closeMemo.hasRun; + + BrowserPlatform._(this._server, Configuration config, String faviconPath, + {String root}) + : _config = config, + _root = root == null ? p.current : root, + _http = config.pubServeUrl == null ? null : HttpClient() { + var cascade = shelf.Cascade().add(_webSocketHandler.handler); + + if (_config.pubServeUrl == null) { + // We server static files from here (JS, HTML, etc) + final String staticFilePath = + config.suiteDefaults.precompiledPath ?? _root; + cascade = cascade + .add(_screeshotHandler) + .add(packagesDirHandler()) + .add(_jsHandler.handler) + .add(createStaticHandler(staticFilePath, + // Precompiled directories often contain symlinks + serveFilesOutsidePath: + config.suiteDefaults.precompiledPath != null)) + .add(_wrapperHandler); + } + + var pipeline = shelf.Pipeline() + .addMiddleware(PathHandler.nestedIn(_secret)) + .addHandler(cascade.handler); + + _server.mount(shelf.Cascade() + .add(createFileHandler(faviconPath)) + .add(pipeline) + .handler); + } + + Future _screeshotHandler(shelf.Request request) async { + if (!request.requestedUri.path.endsWith('/screenshot')) { + return shelf.Response.notFound( + 'This request is not handled by the screenshot handler'); + } + + final String payload = await request.readAsString(); + final Map requestData = json.decode(payload); + final String filename = requestData['filename']; + final bool write = requestData['write']; + final String result = await _diffScreenshot(filename, write); + return shelf.Response.ok(json.encode(result)); + } + + Future _diffScreenshot(String filename, bool write) async { + const String _kGoldensDirectory = 'test/golden_files'; + final wip.ChromeConnection chromeConnection = + wip.ChromeConnection('localhost', _kChromeDevtoolsPort); + final wip.ChromeTab chromeTab = await chromeConnection.getTab( + (wip.ChromeTab chromeTab) => chromeTab.url.contains('localhost')); + final wip.WipConnection wipConnection = await chromeTab.connect(); + final wip.WipResponse response = + await wipConnection.sendCommand('Page.captureScreenshot'); + final Uint8List bytes = base64.decode(response.result['data']); + final File file = File(p.join(_kGoldensDirectory, filename)); + if (write) { + file.writeAsBytesSync(bytes, flush: true); + return 'Golden file $filename was updated. You can remove "write: true" in the call to matchGoldenFile.'; + } + if (!file.existsSync()) { + return ''' + Golden file $filename does not exist. + + To automatically create this file call matchGoldenFile('$filename', write: true). + '''; + } + final List goldenBytes = file.readAsBytesSync(); + final int lengths = bytes.length; + for (int i = 0; i < lengths; i++) { + if (goldenBytes[i] != bytes[i]) { + if (write) { + file.writeAsBytesSync(bytes, flush: true); + return 'Golden file $filename was updated. You can remove "write: true" in the call to matchGoldenFile.'; + } else { + final File failedFile = File(p.join(file.parent.path, '.${p.basename(file.path)}')); + failedFile.writeAsBytesSync(bytes, flush: true); + + // TODO(yjbanov): do not fail Cirrus builds. They currently fail because Chrome produces + // different pictures. We need to pin Chrome versions and use a fuzzy image + // comparator. + if (Platform.environment['CIRRUS_CI'] == 'true') { + return 'OK'; + } + + return ''' + Golden file ${file.path} did not match the image generated by the test. + + The generated image was written to ${failedFile.path}. + + To update the golden file call matchGoldenFile('$filename', write: true). + '''; + } + } + } + return 'OK'; + } + + /// A handler that serves wrapper files used to bootstrap tests. + shelf.Response _wrapperHandler(shelf.Request request) { + var path = p.fromUri(request.url); + + if (path.endsWith('.html')) { + var test = p.withoutExtension(path) + '.dart'; + + // Link to the Dart wrapper. + var scriptBase = htmlEscape.convert(p.basename(test)); + var link = ''; + + return shelf.Response.ok(''' + + + + ${htmlEscape.convert(test)} Test + $link + + + + ''', headers: {'Content-Type': 'text/html'}); + } + + return shelf.Response.notFound('Not found.'); + } + + /// Loads the test suite at [path] on the platform [platform]. + /// + /// This will start a browser to load the suite if one isn't already running. + /// Throws an [ArgumentError] if `platform.platform` isn't a browser. + Future load(String path, SuitePlatform platform, + SuiteConfiguration suiteConfig, Object message) async { + if (suiteConfig.precompiledPath == null) { + throw Exception('This test platform only supports precompiled JS.'); + } + var browser = platform.runtime; + assert(suiteConfig.runtimes.contains(browser.identifier)); + + if (!browser.isBrowser) { + throw ArgumentError('$browser is not a browser.'); + } + + var htmlPath = p.withoutExtension(path) + '.html'; + if (File(htmlPath).existsSync() && + !File(htmlPath).readAsStringSync().contains('packages/test/dart.js')) { + throw LoadException( + path, + '"${htmlPath}" must contain .'); + } + + if (_closed) return null; + Uri suiteUrl = url.resolveUri( + p.toUri(p.withoutExtension(p.relative(path, from: _root)) + '.html')); + + if (_closed) return null; + + var browserManager = await _browserManagerFor(browser); + if (_closed || browserManager == null) return null; + + var suite = await browserManager.load(path, suiteUrl, suiteConfig, message); + if (_closed) return null; + return suite; + } + + StreamChannel loadChannel(String path, SuitePlatform platform) => + throw UnimplementedError(); + + Future _browserManager; + + /// Returns the [BrowserManager] for [runtime], which should be a browser. + /// + /// If no browser manager is running yet, starts one. + Future _browserManagerFor(Runtime browser) { + if (_browserManager != null) return _browserManager; + + var completer = Completer.sync(); + var path = _webSocketHandler.create(webSocketHandler(completer.complete)); + var webSocketUrl = url.replace(scheme: 'ws').resolve(path); + var hostUrl = (_config.pubServeUrl == null ? url : _config.pubServeUrl) + .resolve('packages/web_engine_tester/static/index.html') + .replace(queryParameters: { + 'managerUrl': webSocketUrl.toString(), + 'debug': _config.pauseAfterLoad.toString() + }); + + var future = BrowserManager.start(browser, hostUrl, completer.future, + debug: _config.pauseAfterLoad); + + // Store null values for browsers that error out so we know not to load them + // again. + _browserManager = future.catchError((_) => null); + + return future; + } + + /// Close all the browsers that the server currently has open. + /// + /// Note that this doesn't close the server itself. Browser tests can still be + /// loaded, they'll just spawn new browsers. + Future closeEphemeral() async { + final BrowserManager result = await _browserManager; + if (result != null) { + await result.close(); + } + } + + /// Closes the server and releases all its resources. + /// + /// Returns a [Future] that completes once the server is closed and its + /// resources have been fully released. + Future close() { + return _closeMemo.runOnce(() async { + final List> futures = >[]; + futures.add(Future.microtask(() async { + final BrowserManager result = await _browserManager; + if (result != null) { + await result.close(); + } + })); + futures.add(_server.close()); + + await Future.wait(futures); + + if (_config.pubServeUrl != null) { + _http.close(); + } + }); + } + + final _closeMemo = AsyncMemoizer(); +} + +/// A Shelf handler that provides support for one-time handlers. +/// +/// This is useful for handlers that only expect to be hit once before becoming +/// invalid and don't need to have a persistent URL. +class OneOffHandler { + /// A map from URL paths to handlers. + final _handlers = Map(); + + /// The counter of handlers that have been activated. + var _counter = 0; + + /// The actual [shelf.Handler] that dispatches requests. + shelf.Handler get handler => _onRequest; + + /// Creates a new one-off handler that forwards to [handler]. + /// + /// Returns a string that's the URL path for hitting this handler, relative to + /// the URL for the one-off handler itself. + /// + /// [handler] will be unmounted as soon as it receives a request. + String create(shelf.Handler handler) { + var path = _counter.toString(); + _handlers[path] = handler; + _counter++; + return path; + } + + /// Dispatches [request] to the appropriate handler. + FutureOr _onRequest(shelf.Request request) { + var components = p.url.split(request.url.path); + if (components.isEmpty) return shelf.Response.notFound(null); + + var path = components.removeAt(0); + var handler = _handlers.remove(path); + if (handler == null) return shelf.Response.notFound(null); + return handler(request.change(path: path)); + } +} + +/// A handler that routes to sub-handlers based on exact path prefixes. +class PathHandler { + /// A trie of path components to handlers. + final _paths = _Node(); + + /// The shelf handler. + shelf.Handler get handler => _onRequest; + + /// Returns middleware that nests all requests beneath the URL prefix + /// [beneath]. + static shelf.Middleware nestedIn(String beneath) { + return (handler) { + var pathHandler = PathHandler()..add(beneath, handler); + return pathHandler.handler; + }; + } + + /// Routes requests at or under [path] to [handler]. + /// + /// If [path] is a parent or child directory of another path in this handler, + /// the longest matching prefix wins. + void add(String path, shelf.Handler handler) { + var node = _paths; + for (var component in p.url.split(path)) { + node = node.children.putIfAbsent(component, () => _Node()); + } + node.handler = handler; + } + + FutureOr _onRequest(shelf.Request request) { + shelf.Handler handler; + int handlerIndex; + var node = _paths; + var components = p.url.split(request.url.path); + for (var i = 0; i < components.length; i++) { + node = node.children[components[i]]; + if (node == null) break; + if (node.handler == null) continue; + handler = node.handler; + handlerIndex = i; + } + + if (handler == null) return shelf.Response.notFound('Not found.'); + + return handler( + request.change(path: p.url.joinAll(components.take(handlerIndex + 1)))); + } +} + +/// A trie node. +class _Node { + shelf.Handler handler; + final children = Map(); +} + +/// A class that manages the connection to a single running browser. +/// +/// This is in charge of telling the browser which test suites to load and +/// converting its responses into [Suite] objects. +class BrowserManager { + /// The browser instance that this is connected to via [_channel]. + final Browser _browser; + + /// The [Runtime] for [_browser]. + final Runtime _runtime; + + /// The channel used to communicate with the browser. + /// + /// This is connected to a page running `static/host.dart`. + MultiChannel _channel; + + /// A pool that ensures that limits the number of initial connections the + /// manager will wait for at once. + /// + /// This isn't the *total* number of connections; any number of iframes may be + /// loaded in the same browser. However, the browser can only load so many at + /// once, and we want a timeout in case they fail so we only wait for so many + /// at once. + final _pool = Pool(8); + + /// The ID of the next suite to be loaded. + /// + /// This is used to ensure that the suites can be referred to consistently + /// across the client and server. + int _suiteID = 0; + + /// Whether the channel to the browser has closed. + bool _closed = false; + + /// The completer for [_BrowserEnvironment.displayPause]. + /// + /// This will be `null` as long as the browser isn't displaying a pause + /// screen. + CancelableCompleter _pauseCompleter; + + /// The controller for [_BrowserEnvironment.onRestart]. + final _onRestartController = StreamController.broadcast(); + + /// The environment to attach to each suite. + Future<_BrowserEnvironment> _environment; + + /// Controllers for every suite in this browser. + /// + /// These are used to mark suites as debugging or not based on the browser's + /// pings. + final _controllers = Set(); + + // A timer that's reset whenever we receive a message from the browser. + // + // Because the browser stops running code when the user is actively debugging, + // this lets us detect whether they're debugging reasonably accurately. + RestartableTimer _timer; + + /// Starts the browser identified by [runtime] and has it connect to [url]. + /// + /// [url] should serve a page that establishes a WebSocket connection with + /// this process. That connection, once established, should be emitted via + /// [future]. If [debug] is true, starts the browser in debug mode, with its + /// debugger interfaces on and detected. + /// + /// The [settings] indicate how to invoke this browser's executable. + /// + /// Returns the browser manager, or throws an [Exception] if a + /// connection fails to be established. + static Future start( + Runtime runtime, Uri url, Future future, + {bool debug = false}) { + var browser = _newBrowser(url, runtime, debug: debug); + + var completer = Completer(); + + browser.onExit.then((_) { + throw Exception('${runtime.name} exited before connecting.'); + }).catchError((error, StackTrace stackTrace) { + if (completer.isCompleted) return; + completer.completeError(error, stackTrace); + }); + + future.then((webSocket) { + if (completer.isCompleted) return; + completer.complete(BrowserManager._(browser, runtime, webSocket)); + }).catchError((error, StackTrace stackTrace) { + browser.close(); + if (completer.isCompleted) return; + completer.completeError(error, stackTrace); + }); + + return completer.future.timeout(Duration(seconds: 30), onTimeout: () { + browser.close(); + throw Exception('Timed out waiting for ${runtime.name} to connect.'); + }); + } + + /// Starts the browser identified by [browser] using [settings] and has it load [url]. + /// + /// If [debug] is true, starts the browser in debug mode. + static Browser _newBrowser(Uri url, Runtime browser, {bool debug = false}) { + return Chrome(url, debug: debug); + } + + /// Creates a new BrowserManager that communicates with [browser] over + /// [webSocket]. + BrowserManager._(this._browser, this._runtime, WebSocketChannel webSocket) { + // The duration should be short enough that the debugging console is open as + // soon as the user is done setting breakpoints, but long enough that a test + // doing a lot of synchronous work doesn't trigger a false positive. + // + // Start this canceled because we don't want it to start ticking until we + // get some response from the iframe. + _timer = RestartableTimer(Duration(seconds: 3), () { + for (var controller in _controllers) { + controller.setDebugging(true); + } + }) + ..cancel(); + + // Whenever we get a message, no matter which child channel it's for, we the + // know browser is still running code which means the user isn't debugging. + _channel = MultiChannel( + webSocket.cast().transform(jsonDocument).changeStream((stream) { + return stream.map((message) { + if (!_closed) _timer.reset(); + for (var controller in _controllers) { + controller.setDebugging(false); + } + + return message; + }); + })); + + _environment = _loadBrowserEnvironment(); + _channel.stream + .listen((message) => _onMessage(message as Map), onDone: close); + } + + /// Loads [_BrowserEnvironment]. + Future<_BrowserEnvironment> _loadBrowserEnvironment() async { + return _BrowserEnvironment(this, await _browser.observatoryUrl, + await _browser.remoteDebuggerUrl, _onRestartController.stream); + } + + /// Tells the browser the load a test suite from the URL [url]. + /// + /// [url] should be an HTML page with a reference to the JS-compiled test + /// suite. [path] is the path of the original test suite file, which is used + /// for reporting. [suiteConfig] is the configuration for the test suite. + Future load(String path, Uri url, SuiteConfiguration suiteConfig, + Object message) async { + url = url.replace( + fragment: Uri.encodeFull(jsonEncode({ + 'metadata': suiteConfig.metadata.serialize(), + 'browser': _runtime.identifier + }))); + + var suiteID = _suiteID++; + RunnerSuiteController controller; + closeIframe() { + if (_closed) return; + _controllers.remove(controller); + _channel.sink.add({'command': 'closeSuite', 'id': suiteID}); + } + + // The virtual channel will be closed when the suite is closed, in which + // case we should unload the iframe. + var virtualChannel = _channel.virtualChannel(); + var suiteChannelID = virtualChannel.id; + var suiteChannel = virtualChannel + .transformStream(StreamTransformer.fromHandlers(handleDone: (sink) { + closeIframe(); + sink.close(); + })); + + return await _pool.withResource(() async { + _channel.sink.add({ + 'command': 'loadSuite', + 'url': url.toString(), + 'id': suiteID, + 'channel': suiteChannelID + }); + + try { + controller = deserializeSuite(path, currentPlatform(_runtime), + suiteConfig, await _environment, suiteChannel, message); + + final String mapPath = p.join( + env.environment.webUiRootDir.path, + 'build', + '$path.js.map', + ); + final JSStackTraceMapper mapper = JSStackTraceMapper( + await File(mapPath).readAsString(), + mapUrl: p.toUri(mapPath), + packageResolver: await PackageResolver.current.asSync, + sdkRoot: p.toUri(sdkDir), + ); + + controller.channel('test.browser.mapper').sink.add(mapper.serialize()); + + _controllers.add(controller); + return await controller.suite; + } catch (_) { + closeIframe(); + rethrow; + } + }); + } + + /// An implementation of [Environment.displayPause]. + CancelableOperation _displayPause() { + if (_pauseCompleter != null) return _pauseCompleter.operation; + + _pauseCompleter = CancelableCompleter(onCancel: () { + _channel.sink.add({'command': 'resume'}); + _pauseCompleter = null; + }); + + _pauseCompleter.operation.value.whenComplete(() { + _pauseCompleter = null; + }); + + _channel.sink.add({'command': 'displayPause'}); + + return _pauseCompleter.operation; + } + + /// The callback for handling messages received from the host page. + void _onMessage(Map message) { + switch (message['command'] as String) { + case 'ping': + break; + + case 'restart': + _onRestartController.add(null); + break; + + case 'resume': + if (_pauseCompleter != null) _pauseCompleter.complete(); + break; + + default: + // Unreachable. + assert(false); + break; + } + } + + /// Closes the manager and releases any resources it owns, including closing + /// the browser. + Future close() => _closeMemoizer.runOnce(() { + _closed = true; + _timer.cancel(); + if (_pauseCompleter != null) _pauseCompleter.complete(); + _pauseCompleter = null; + _controllers.clear(); + return _browser.close(); + }); + final _closeMemoizer = AsyncMemoizer(); +} + +/// An implementation of [Environment] for the browser. +/// +/// All methods forward directly to [BrowserManager]. +class _BrowserEnvironment implements Environment { + final BrowserManager _manager; + + final supportsDebugging = true; + + final Uri observatoryUrl; + + final Uri remoteDebuggerUrl; + + final Stream onRestart; + + _BrowserEnvironment(this._manager, this.observatoryUrl, + this.remoteDebuggerUrl, this.onRestart); + + CancelableOperation displayPause() => _manager._displayPause(); +} + +/// An interface for running browser instances. +/// +/// This is intentionally coarse-grained: browsers are controlled primary from +/// inside a single tab. Thus this interface only provides support for closing +/// the browser and seeing if it closes itself. +/// +/// Any errors starting or running the browser process are reported through +/// [onExit]. +abstract class Browser { + String get name; + + /// The Observatory URL for this browser. + /// + /// This will return `null` for browsers that aren't running the Dart VM, or + /// if the Observatory URL can't be found. + Future get observatoryUrl => null; + + /// The remote debugger URL for this browser. + /// + /// This will return `null` for browsers that don't support remote debugging, + /// or if the remote debugging URL can't be found. + Future get remoteDebuggerUrl => null; + + /// The underlying process. + /// + /// This will fire once the process has started successfully. + Future get _process => _processCompleter.future; + final _processCompleter = Completer(); + + /// Whether [close] has been called. + var _closed = false; + + /// A future that completes when the browser exits. + /// + /// If there's a problem starting or running the browser, this will complete + /// with an error. + Future get onExit => _onExitCompleter.future; + final _onExitCompleter = Completer(); + + /// Standard IO streams for the underlying browser process. + final _ioSubscriptions = []; + + /// Creates a new browser. + /// + /// This is intended to be called by subclasses. They pass in [startBrowser], + /// which asynchronously returns the browser process. Any errors in + /// [startBrowser] (even those raised asynchronously after it returns) are + /// piped to [onExit] and will cause the browser to be killed. + Browser(Future startBrowser()) { + // Don't return a Future here because there's no need for the caller to wait + // for the process to actually start. They should just wait for the HTTP + // request instead. + runZoned(() async { + var process = await startBrowser(); + _processCompleter.complete(process); + + var output = Uint8Buffer(); + drainOutput(Stream> stream) { + try { + _ioSubscriptions + .add(stream.listen(output.addAll, cancelOnError: true)); + } on StateError catch (_) {} + } + + // If we don't drain the stdout and stderr the process can hang. + drainOutput(process.stdout); + drainOutput(process.stderr); + + var exitCode = await process.exitCode; + + // This hack dodges an otherwise intractable race condition. When the user + // presses Control-C, the signal is sent to the browser and the test + // runner at the same time. It's possible for the browser to exit before + // the [Browser.close] is called, which would trigger the error below. + // + // A negative exit code signals that the process exited due to a signal. + // However, it's possible that this signal didn't come from the user's + // Control-C, in which case we do want to throw the error. The only way to + // resolve the ambiguity is to wait a brief amount of time and see if this + // browser is actually closed. + if (!_closed && exitCode < 0) { + await Future.delayed(Duration(milliseconds: 200)); + } + + if (!_closed && exitCode != 0) { + var outputString = utf8.decode(output); + var message = '$name failed with exit code $exitCode.'; + if (outputString.isNotEmpty) { + message += '\nStandard output:\n$outputString'; + } + + throw Exception(message); + } + + _onExitCompleter.complete(); + }, onError: (error, StackTrace stackTrace) { + // Ignore any errors after the browser has been closed. + if (_closed) return; + + // Make sure the process dies even if the error wasn't fatal. + _process.then((process) => process.kill()); + + if (stackTrace == null) stackTrace = Trace.current(); + if (_onExitCompleter.isCompleted) return; + _onExitCompleter.completeError( + Exception('Failed to run $name: ${getErrorMessage(error)}.'), + stackTrace); + }); + } + + /// Kills the browser process. + /// + /// Returns the same [Future] as [onExit], except that it won't emit + /// exceptions. + Future close() async { + _closed = true; + + // If we don't manually close the stream the test runner can hang. + // For example this happens with Chrome Headless. + // See SDK issue: https://github.com/dart-lang/sdk/issues/31264 + for (var stream in _ioSubscriptions) { + unawaited(stream.cancel()); + } + + (await _process).kill(); + + // Swallow exceptions. The user should explicitly use [onExit] for these. + return onExit.catchError((_) {}); + } +} + +/// A class for running an instance of Chrome. +/// +/// Most of the communication with the browser is expected to happen via HTTP, +/// so this exposes a bare-bones API. The browser starts as soon as the class is +/// constructed, and is killed when [close] is called. +/// +/// Any errors starting or running the process are reported through [onExit]. +class Chrome extends Browser { + @override + final name = 'Chrome'; + + @override + final Future remoteDebuggerUrl; + + /// Starts a new instance of Chrome open to the given [url], which may be a + /// [Uri] or a [String]. + factory Chrome(Uri url, {bool debug = false}) { + var remoteDebuggerCompleter = Completer.sync(); + return Chrome._(() async { + final bool isChromeNoSandbox = Platform.environment['CHROME_NO_SANDBOX'] == 'true'; + var dir = createTempDir(); + var args = [ + '--user-data-dir=$dir', + url.toString(), + if (!debug) '--headless', + if (isChromeNoSandbox) '--no-sandbox', + '--disable-extensions', + '--disable-popup-blocking', + '--bwsi', + '--no-first-run', + '--no-default-browser-check', + '--disable-default-apps', + '--disable-translate', + '--remote-debugging-port=$_kChromeDevtoolsPort', + ]; + + var process = await Process.start('google-chrome', args); + + remoteDebuggerCompleter.complete(getRemoteDebuggerUrl( + Uri.parse('http://localhost:$_kChromeDevtoolsPort'))); + + unawaited(process.exitCode + .then((_) => Directory(dir).deleteSync(recursive: true))); + + return process; + }, remoteDebuggerCompleter.future); + } + + Chrome._(Future startBrowser(), this.remoteDebuggerUrl) + : super(startBrowser); +} diff --git a/lib/web_ui/dev/test_runner.dart b/lib/web_ui/dev/test_runner.dart new file mode 100644 index 0000000000000..8e8657bfcc56a --- /dev/null +++ b/lib/web_ui/dev/test_runner.dart @@ -0,0 +1,179 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:io' as io; + +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as path; +import 'package:test_core/src/runner/hack_register_platform.dart' as hack; // ignore: implementation_imports +import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports +import 'package:test_core/src/executable.dart' as test; // ignore: implementation_imports + +import 'test_platform.dart'; +import 'environment.dart'; + +Future runTests() async { + await _buildHostPage(); + await _buildTests(); + + if (environment.targets.isEmpty) { + await _runAllTests(); + } else { + await _runSingleTest(environment.targets); + } +} + +Future _runSingleTest(List targets) async { + await _runTestBatch(targets, concurrency: 1, expectFailure: false); + _checkExitCode(); +} + +Future _runAllTests() async { + final io.Directory testDir = io.Directory(path.join( + environment.webUiRootDir.path, + 'test', + )); + + // Separate screenshot tests from unit-tests. Screenshot tests must run + // one at a time. Otherwise, they will end up screenshotting each other. + // This is not an issue for unit-tests. + const String failureSmokeTestPath = 'test/golden_tests/golden_failure_smoke_test.dart'; + final List screenshotTestFiles = []; + final List unitTestFiles = []; + + for (io.File testFile in testDir.listSync(recursive: true).whereType()) { + final String testFilePath = path.relative(testFile.path, from: environment.webUiRootDir.path); + if (!testFilePath.endsWith('_test.dart')) { + // Not a test file at all. Skip. + continue; + } + if (testFilePath.endsWith(failureSmokeTestPath)) { + // A smoke test that fails on purpose. Skip. + continue; + } + if (path.split(testFilePath).contains('golden_tests')) { + screenshotTestFiles.add(testFilePath); + } else { + unitTestFiles.add(testFilePath); + } + } + + // This test returns a non-zero exit code on purpose. Run it separately. + if (io.Platform.environment['CIRRUS_CI'] != 'true') { + await _runTestBatch( + [failureSmokeTestPath], + concurrency: 1, + expectFailure: true, + ); + _checkExitCode(); + } + + // Run all unit-tests as a single batch. + await _runTestBatch(unitTestFiles, concurrency: 10, expectFailure: false); + _checkExitCode(); + + // Run screenshot tests one at a time. + for (String testFilePath in screenshotTestFiles) { + await _runTestBatch([testFilePath], concurrency: 1, expectFailure: false); + _checkExitCode(); + } +} + +void _checkExitCode() { + if (io.exitCode != 0) { + io.stderr.writeln('Process exited with exit code ${io.exitCode}.'); + io.exit(1); + } +} + +// TODO(yjbanov): skip rebuild if host.dart hasn't changed. +Future _buildHostPage() async { + final io.Process pubRunTest = await io.Process.start( + environment.dart2jsExecutable, + [ + 'lib/static/host.dart', + '-o', + 'lib/static/host.dart.js', + ], + workingDirectory: environment.goldenTesterRootDir.path, + ); + + final StreamSubscription stdoutSub = pubRunTest.stdout.listen(io.stdout.add); + final StreamSubscription stderrSub = pubRunTest.stderr.listen(io.stderr.add); + final int exitCode = await pubRunTest.exitCode; + stdoutSub.cancel(); + stderrSub.cancel(); + + if (exitCode != 0) { + io.stderr.writeln('Failed to compile tests. Compiler exited with exit code $exitCode'); + io.exit(1); + } +} + +Future _buildTests() async { + // TODO(yjbanov): learn to build only requested tests: https://github.com/flutter/flutter/issues/37810 + final io.Process pubRunTest = await io.Process.start( + environment.pubExecutable, + [ + 'run', + 'build_runner', + 'build', + 'test', + '-o', + 'build', + ], + ); + + final StreamSubscription stdoutSub = pubRunTest.stdout.listen(io.stdout.add); + final StreamSubscription stderrSub = pubRunTest.stderr.listen(io.stderr.add); + final int exitCode = await pubRunTest.exitCode; + stdoutSub.cancel(); + stderrSub.cancel(); + + if (exitCode != 0) { + io.stderr.writeln('Failed to compile tests. Compiler exited with exit code $exitCode'); + io.exit(1); + } +} + +Future _runTestBatch( + List testFiles, { + @required int concurrency, + @required bool expectFailure, + } +) async { + final List testArgs = [ + '--no-color', + ...['-r', 'compact'], + '--concurrency=$concurrency', + if (environment.isDebug) + '--pause-after-load', + '--platform=chrome', + '--precompiled=${environment.webUiRootDir.path}/build', + '--', + ...testFiles, + ]; + hack.registerPlatformPlugin( + [Runtime.chrome], + () { + return BrowserPlatform.start(root: io.Directory.current.path); + } + ); + await test.main(testArgs); + + if (expectFailure) { + if (io.exitCode != 0) { + // It failed, as expected. + io.exitCode = 0; + } else { + io.stderr.writeln( + 'Tests ${testFiles.join(', ')} did not fail. Expected failure.', + ); + io.exitCode = 1; + } + } + + return io.exitCode; +} diff --git a/lib/web_ui/pubspec.yaml b/lib/web_ui/pubspec.yaml index d1fc33a328ce7..f28c5031011c2 100644 --- a/lib/web_ui/pubspec.yaml +++ b/lib/web_ui/pubspec.yaml @@ -14,3 +14,5 @@ dev_dependencies: build_runner: 1.6.5 build_test: 0.10.8 build_web_compilers: 2.1.5 + web_engine_tester: + path: ../../web_sdk/web_engine_tester diff --git a/lib/web_ui/test/golden_files/.gitignore b/lib/web_ui/test/golden_files/.gitignore new file mode 100644 index 0000000000000..4b2350ab78f90 --- /dev/null +++ b/lib/web_ui/test/golden_files/.gitignore @@ -0,0 +1,2 @@ +# Failed screenshot files +.*.png diff --git a/lib/web_ui/test/golden_files/smoke_test.png b/lib/web_ui/test/golden_files/smoke_test.png new file mode 100644 index 0000000000000..c17b3ba1798f0 Binary files /dev/null and b/lib/web_ui/test/golden_files/smoke_test.png differ diff --git a/lib/web_ui/test/golden_tests/golden_failure_smoke_test.dart b/lib/web_ui/test/golden_tests/golden_failure_smoke_test.dart new file mode 100644 index 0000000000000..b001b4ad1f2cd --- /dev/null +++ b/lib/web_ui/test/golden_tests/golden_failure_smoke_test.dart @@ -0,0 +1,15 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:html' as html; + +import 'package:test/test.dart'; +import 'package:web_engine_tester/golden_tester.dart'; + +void main() { + test('screenshot test reports failure', () async { + html.document.body.innerHtml = 'Text that does not appear on the screenshot!'; + await matchGoldenFile('smoke_test.png'); + }); +} diff --git a/lib/web_ui/test/golden_tests/golden_success_smoke_test.dart b/lib/web_ui/test/golden_tests/golden_success_smoke_test.dart new file mode 100644 index 0000000000000..657f271eea67d --- /dev/null +++ b/lib/web_ui/test/golden_tests/golden_success_smoke_test.dart @@ -0,0 +1,15 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:html' as html; + +import 'package:test/test.dart'; +import 'package:web_engine_tester/golden_tester.dart'; + +void main() { + test('screenshot test reports success', () async { + html.document.body.innerHtml = 'Hello world!'; + await matchGoldenFile('smoke_test.png'); + }); +} diff --git a/tools/licenses/lib/main.dart b/tools/licenses/lib/main.dart index b82367bee20ba..a1521cb9dbe81 100644 --- a/tools/licenses/lib/main.dart +++ b/tools/licenses/lib/main.dart @@ -2044,27 +2044,71 @@ class _RepositoryFlutterDirectory extends _RepositoryDirectory { if (entry.name == 'third_party') return _RepositoryFlutterThirdPartyDirectory(this, entry); if (entry.name == 'lib') - return _RepositoryLibDirectory(entry, this, entry); + return _createLibDirectoryRoot(entry, this); + if (entry.name == 'web_sdk') + return _createWebSdkDirectoryRoot(entry, this); return super.createSubdirectory(entry); } } -// The "lib/" directory containing the source code for "dart:ui" (both native and Web) and -// all its sub-directories. -class _RepositoryLibDirectory extends _RepositoryDirectory { - _RepositoryLibDirectory(this.libRoot, _RepositoryDirectory parent, fs.Directory io) : super(parent, io); +/// A specialized crawler for "github.com/flutter/engine/lib" directory. +/// +/// It includes everything except build tools, test build artifacts, and test code. +_RelativePathBlacklistRepositoryDirectory _createLibDirectoryRoot(fs.Directory entry, _RepositoryDirectory parent) { + return _RelativePathBlacklistRepositoryDirectory( + rootDir: entry, + blacklist: [ + 'web_ui/lib/assets/ahem.ttf', // this gitignored file exists only for testing purposes + RegExp(r'web_ui/build/.*'), // this is compiler-generated output + RegExp(r'web_ui/dev/.*'), // these are build tools; they do not end up in Engine artifacts + RegExp(r'web_ui/test/.*'), // tests do not end up in Engine artifacts + ], + parent: parent, + io: entry, + ); +} + +/// A specialized crawler for "github.com/flutter/engine/web_sdk" directory. +/// +/// It includes everything except the "web_engine_tester" package, which is only +/// used to test the engine itself and is not shipped as part of the Flutter SDK. +_RelativePathBlacklistRepositoryDirectory _createWebSdkDirectoryRoot(fs.Directory entry, _RepositoryDirectory parent) { + return _RelativePathBlacklistRepositoryDirectory( + rootDir: entry, + blacklist: [ + RegExp(r'web_engine_tester/.*'), // contains test code for the engine itself + ], + parent: parent, + io: entry, + ); +} + +/// Walks a [rootDir] recursively, omitting paths that match a [blacklist]. +/// +/// The path patterns in the [blacklist] are specified relative to the [rootDir]. +class _RelativePathBlacklistRepositoryDirectory extends _RepositoryDirectory { + _RelativePathBlacklistRepositoryDirectory({ + @required this.rootDir, + @required this.blacklist, + @required _RepositoryDirectory parent, + @required fs.Directory io, + }) : super(parent, io); - // List of files inside the lib directory that we're not scanning. - static const List _kBlacklist = [ - 'web_ui/lib/assets/ahem.ttf', // this gitignored file exists only for testing purposes - ]; + /// The directory, relative to which the paths are [blacklist]ed. + final fs.Directory rootDir; - final fs.Directory libRoot; + /// Blacklisted path patterns. + /// + /// Paths are assumed relative to [rootDir]. + final List blacklist; @override bool shouldRecurse(fs.IoNode entry) { - final String relativePath = path.relative(entry.fullName, from: libRoot.fullName); - if (_kBlacklist.contains(relativePath)) { + final String relativePath = path.relative(entry.fullName, from: rootDir.fullName); + final bool isBlacklisted = blacklist.any( + (Pattern pattern) => pattern.matchAsPrefix(relativePath) != null, + ); + if (isBlacklisted) { return false; } return super.shouldRecurse(entry); @@ -2072,7 +2116,12 @@ class _RepositoryLibDirectory extends _RepositoryDirectory { @override _RepositoryDirectory createSubdirectory(fs.Directory entry) { - return _RepositoryLibDirectory(libRoot, this, entry); + return _RelativePathBlacklistRepositoryDirectory( + rootDir: rootDir, + blacklist: blacklist, + parent: this, + io: entry, + ); } } diff --git a/web_sdk/web_engine_tester/lib/golden_tester.dart b/web_sdk/web_engine_tester/lib/golden_tester.dart new file mode 100644 index 0000000000000..e4910ec2b3323 --- /dev/null +++ b/web_sdk/web_engine_tester/lib/golden_tester.dart @@ -0,0 +1,32 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:html' as html; + +import 'package:test/test.dart'; + +Future _callScreenshotServer(dynamic requestData) async { + final html.HttpRequest request = await html.HttpRequest.request( + 'screenshot', + method: 'POST', + sendData: json.encode(requestData), + ); + + return json.decode(request.responseText); +} + +/// Attempts to match the current browser state with the screenshot [filename]. +Future matchGoldenFile(String filename, { bool write = false }) async { + final String response = await _callScreenshotServer({ + 'filename': filename, + 'write': write, + }); + if (response == 'OK') { + // Pass + return; + } + fail(response); +} diff --git a/web_sdk/web_engine_tester/lib/static/.gitignore b/web_sdk/web_engine_tester/lib/static/.gitignore new file mode 100644 index 0000000000000..43f163cc11467 --- /dev/null +++ b/web_sdk/web_engine_tester/lib/static/.gitignore @@ -0,0 +1 @@ +host.dart.js* diff --git a/web_sdk/web_engine_tester/lib/static/README.md b/web_sdk/web_engine_tester/lib/static/README.md new file mode 100644 index 0000000000000..259e863cce36e --- /dev/null +++ b/web_sdk/web_engine_tester/lib/static/README.md @@ -0,0 +1 @@ +This directory contains code for the web page that hosts Web Engine tests. diff --git a/web_sdk/web_engine_tester/lib/static/favicon.ico b/web_sdk/web_engine_tester/lib/static/favicon.ico new file mode 100644 index 0000000000000..7ba349b3e628d Binary files /dev/null and b/web_sdk/web_engine_tester/lib/static/favicon.ico differ diff --git a/web_sdk/web_engine_tester/lib/static/host.css b/web_sdk/web_engine_tester/lib/static/host.css new file mode 100644 index 0000000000000..ea1e941c368e2 --- /dev/null +++ b/web_sdk/web_engine_tester/lib/static/host.css @@ -0,0 +1,4 @@ +body { + margin: 0; + padding: 0; +} diff --git a/web_sdk/web_engine_tester/lib/static/host.dart b/web_sdk/web_engine_tester/lib/static/host.dart new file mode 100644 index 0000000000000..5751141851683 --- /dev/null +++ b/web_sdk/web_engine_tester/lib/static/host.dart @@ -0,0 +1,239 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +@JS() +library test.host; + +import 'dart:async'; +import 'dart:convert'; +import 'dart:html'; + +import 'package:stack_trace/stack_trace.dart'; +import 'package:stream_channel/stream_channel.dart'; +import 'package:js/js.dart'; + +/// A class defined in content shell, used to control its behavior. +@JS() +class _TestRunner { + external void waitUntilDone(); +} + +/// Returns the current content shell runner, or `null` if none exists. +@JS() +external _TestRunner get testRunner; + +/// A class that exposes the test API to JS. +/// +/// These are exposed so that tools like IDEs can interact with them via remote +/// debugging. +@JS() +@anonymous +class _JSApi { + /// Causes the test runner to resume running, as though the user had clicked + /// the "play" button. + external Function get resume; + + /// Causes the test runner to restart the current test once it finishes + /// running. + external Function get restartCurrent; + + external factory _JSApi({void resume(), void restartCurrent()}); +} + +/// Sets the top-level `dartTest` object so that it's visible to JS. +@JS('dartTest') +external set _jsApi(_JSApi api); + +/// The iframes created for each loaded test suite, indexed by the suite id. +final Map _iframes = {}; + +/// Subscriptions created for each loaded test suite, indexed by the suite id. +final Map>> _subscriptions = >>{}; + +/// The URL for the current page. +final Uri _currentUrl = Uri.parse(window.location.href); + +/// Code that runs in the browser and loads test suites at the server's behest. +/// +/// One instance of this runs for each browser. When the server tells it to load +/// a test, it starts an iframe pointing at that test's code; from then on, it +/// just relays messages between the two. +/// +/// The browser uses two layers of [MultiChannel]s when communicating with the +/// server: +/// +/// server +/// │ +/// (WebSocket) +/// │ +/// ┏━ host.html ━━━━━━━━┿━━━━━━━━━━━━━━━━━┓ +/// ┃ │ ┃ +/// ┃ ┌──────┬───MultiChannel─────┐ ┃ +/// ┃ │ │ │ │ │ ┃ +/// ┃ host suite suite suite suite ┃ +/// ┃ │ │ │ │ ┃ +/// ┗━━━━━━━━━━━┿━━━━━━┿━━━━━━┿━━━━━━┿━━━━━┛ +/// │ │ │ │ +/// │ ... ... ... +/// │ +/// (MessageChannel) +/// │ +/// ┏━ suite.html (in iframe) ┿━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +/// ┃ │ ┃ +/// ┃ ┌──────────MultiChannel┬─────────┐ ┃ +/// ┃ │ │ │ │ │ ┃ +/// ┃ IframeListener test test test running test ┃ +/// ┃ ┃ +/// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ +/// +/// The host (this code) has a [MultiChannel] that splits the WebSocket +/// connection with the server. One connection is used for the host itself to +/// receive messages like "load a suite at this URL", and the rest are +/// connected to each test suite's iframe via a [MessageChannel]. +/// +/// Each iframe then has its own [MultiChannel] which takes its +/// [MessageChannel] connection and splits it again. One connection is used for +/// the [IframeListener], which sends messages like "here are all the tests in +/// this suite". The rest are used for each test, receiving messages like +/// "start running". A new connection is also created whenever a test begins +/// running to send status messages about its progress. +/// +/// It's of particular note that the suite's [MultiChannel] connection uses the +/// host's purely as a transport layer; neither is aware that the other is also +/// using [MultiChannel]. This is necessary, since the host doesn't share memory +/// with the suites and thus can't share its [MultiChannel] with them, but it +/// does mean that the server needs to be sure to nest its [MultiChannel]s at +/// the same place the client does. +void main() { + // This tells content_shell not to close immediately after the page has + // rendered. + testRunner?.waitUntilDone(); + + if (_currentUrl.queryParameters['debug'] == 'true') { + document.body.classes.add('debug'); + } + + runZoned(() { + final MultiChannel serverChannel = _connectToServer(); + serverChannel.stream.listen((dynamic message) { + if (message['command'] == 'loadSuite') { + final int channelId = message['channel']; + final String url = message['url']; + final int messageId = message['id']; + final VirtualChannel suiteChannel = serverChannel.virtualChannel(channelId); + final StreamChannel iframeChannel = _connectToIframe(url, messageId); + suiteChannel.pipe(iframeChannel); + } else if (message['command'] == 'displayPause') { + document.body.classes.add('paused'); + } else if (message['command'] == 'resume') { + document.body.classes.remove('paused'); + } else { + assert(message['command'] == 'closeSuite'); + _iframes.remove(message['id']).remove(); + + for (StreamSubscription subscription in _subscriptions.remove(message['id'])) { + subscription.cancel(); + } + } + }); + + // Send periodic pings to the test runner so it can know when the browser is + // paused for debugging. + Timer.periodic(Duration(seconds: 1), + (_) => serverChannel.sink.add({'command': 'ping'})); + + _jsApi = _JSApi(resume: allowInterop(() { + if (!document.body.classes.remove('paused')) { + return; + } + serverChannel.sink.add({'command': 'resume'}); + }), restartCurrent: allowInterop(() { + serverChannel.sink.add({'command': 'restart'}); + })); + }, onError: (dynamic error, StackTrace stackTrace) { + print('$error\n${Trace.from(stackTrace).terse}'); + }); +} + +/// Creates a [MultiChannel] connection to the server, using a [WebSocket] as +/// the underlying protocol. +MultiChannel _connectToServer() { + // The `managerUrl` query parameter contains the WebSocket URL of the remote + // [BrowserManager] with which this communicates. + final WebSocket webSocket = WebSocket(_currentUrl.queryParameters['managerUrl']); + + final StreamChannelController controller = StreamChannelController(sync: true); + webSocket.onMessage.listen((MessageEvent message) { + final String data = message.data; + controller.local.sink.add(jsonDecode(data)); + }); + + controller.local.stream + .listen((dynamic message) => webSocket.send(jsonEncode(message))); + + return MultiChannel(controller.foreign); +} + +/// Creates an iframe with `src` [url] and establishes a connection to it using +/// a [MessageChannel]. +/// +/// [id] identifies the suite loaded in this iframe. +StreamChannel _connectToIframe(String url, int id) { + final IFrameElement iframe = IFrameElement(); + _iframes[id] = iframe; + iframe + ..src = url + ..width = '500' + ..height = '500'; + document.body.children.add(iframe); + + // Use this to communicate securely with the iframe. + final MessageChannel channel = MessageChannel(); + final StreamChannelController controller = StreamChannelController(sync: true); + + // Use this to avoid sending a message to the iframe before it's sent a + // message to us. This ensures that no messages get dropped on the floor. + final Completer readyCompleter = Completer(); + + final List> subscriptions = >[]; + _subscriptions[id] = subscriptions; + + subscriptions.add(window.onMessage.listen((dynamic message) { + // A message on the Window can theoretically come from any website. It's + // very unlikely that a malicious site would care about hacking someone's + // unit tests, let alone be able to find the test server while it's + // running, but it's good practice to check the origin anyway. + if (message.origin != window.location.origin) { + return; + } + + if (message.data['href'] != iframe.src) { + return; + } + + message.stopPropagation(); + + if (message.data['ready'] == true) { + // This message indicates that the iframe is actively listening for + // events, so the message channel's second port can now be transferred. + iframe.contentWindow.postMessage('port', window.location.origin, [channel.port2]); + readyCompleter.complete(); + } else if (message.data['exception'] == true) { + // This message from `dart.js` indicates that an exception occurred + // loading the test. + controller.local.sink.add(message.data['data']); + } + })); + + subscriptions.add(channel.port1.onMessage.listen((dynamic message) { + controller.local.sink.add(message.data['data']); + })); + + subscriptions.add(controller.local.stream.listen((dynamic message) async { + await readyCompleter.future; + channel.port1.postMessage(message); + })); + + return controller.foreign; +} diff --git a/web_sdk/web_engine_tester/lib/static/index.html b/web_sdk/web_engine_tester/lib/static/index.html new file mode 100644 index 0000000000000..68a04fd28c273 --- /dev/null +++ b/web_sdk/web_engine_tester/lib/static/index.html @@ -0,0 +1,10 @@ + + + + Web Engine Test Runner + + + + + + diff --git a/web_sdk/web_engine_tester/pubspec.yaml b/web_sdk/web_engine_tester/pubspec.yaml new file mode 100644 index 0000000000000..278f280158d26 --- /dev/null +++ b/web_sdk/web_engine_tester/pubspec.yaml @@ -0,0 +1,10 @@ +name: web_engine_tester + +environment: + sdk: ">=2.2.0 <3.0.0" + +dependencies: + js: 0.6.1+1 + stream_channel: 2.0.0 + test: 1.6.5 + webkit_inspection_protocol: 0.5.0