diff --git a/ci/builders/linux_fuchsia.json b/ci/builders/linux_fuchsia.json index 50a58b1436412..ca1c999c09d56 100644 --- a/ci/builders/linux_fuchsia.json +++ b/ci/builders/linux_fuchsia.json @@ -192,6 +192,10 @@ "--upload" ] }, + { + "name": "run_tests test", + "script": "flutter/testing/fuchsia/run_tests_test.py" + }, { "name": "x64 emulator based debug tests", "language": "python3", diff --git a/testing/fuchsia/run_tests.py b/testing/fuchsia/run_tests.py index e99c722f68262..e833329925f1a 100755 --- a/testing/fuchsia/run_tests.py +++ b/testing/fuchsia/run_tests.py @@ -18,7 +18,7 @@ import sys from subprocess import CompletedProcess -from typing import List, Set +from typing import Any, Iterable, List, Mapping, NamedTuple, Set # The import is coming from vpython wheel and pylint cannot find it. import yaml # pylint: disable=import-error @@ -29,10 +29,7 @@ # is guaranteed. sys.path.insert( - 0, - os.path.join( - os.path.dirname(__file__), '../../tools/fuchsia/test_scripts/test/' - ) + 0, os.path.join(os.path.dirname(__file__), '../../tools/fuchsia/test_scripts/test/') ) # pylint: disable=import-error, wrong-import-position @@ -51,13 +48,16 @@ OUT_DIR = os.path.join(DIR_SRC_ROOT, 'out', VARIANT) -class BundledTestRunner(TestRunner): +# Visible for testing +class TestCase(NamedTuple): + package: str + args: str = '' + + +class _BundledTestRunner(TestRunner): # private, use bundled_test_runner_of function instead. - def __init__( - self, target_id: str, package_deps: Set[str], tests: List[str], - logs_dir: str - ): + def __init__(self, target_id: str, package_deps: Set[str], tests: List[TestCase], logs_dir: str): super().__init__(OUT_DIR, [], None, target_id, list(package_deps)) self.tests = tests self.logs_dir = logs_dir @@ -65,10 +65,11 @@ def __init__( def run_test(self) -> CompletedProcess: returncode = 0 for test in self.tests: - # pylint: disable=protected-access + assert test.package.endswith('.cm') test_runner = ExecutableTestRunner( - OUT_DIR, [], test, self._target_id, None, self.logs_dir, [], None + OUT_DIR, test.args.split(), test.package, self._target_id, None, self.logs_dir, [], None ) + # pylint: disable=protected-access test_runner._package_deps = self._package_deps result = test_runner.run_test().returncode logging.info('Result of test %s is %s', test, result) @@ -77,33 +78,8 @@ def run_test(self) -> CompletedProcess: return CompletedProcess(args='', returncode=returncode) -def bundled_test_runner_of(target_id: str) -> BundledTestRunner: - log_dir = os.environ.get('FLUTTER_LOGS_DIR', '/tmp/log') - with open(os.path.join(os.path.dirname(__file__), 'test_suites.yaml'), - 'r') as file: - tests = yaml.safe_load(file) - # TODO(zijiehe-google-com): Run tests with extra test arguments, - # https://github.com/flutter/flutter/issues/140179. - tests = list( - filter( - lambda test: test['test_command'].startswith('test run ') and test[ - 'test_command'].endswith('.cm'), tests - ) - ) - # TODO(zijiehe-google-com): Run tests with dart aot, - # https://github.com/flutter/flutter/issues/140179. - tests = list( - filter( - lambda test: 'run_with_dart_aot' not in test or test[ - 'run_with_dart_aot'] != 'true', tests - ) - ) - tests = list( - filter( - lambda test: 'variant' not in test or VARIANT == test['variant'], - tests - ) - ) +# Visible for testing +def resolve_packages(tests: Iterable[Mapping[str, Any]]) -> Set[str]: packages = set() for test in tests: if 'package' in test: @@ -128,14 +104,43 @@ def bundled_test_runner_of(target_id: str) -> BundledTestRunner: resolved_packages.add(new_package) else: resolved_packages.add(os.path.join(OUT_DIR, package)) - return BundledTestRunner( - target_id, resolved_packages, - [test['test_command'][len('test run '):] for test in tests], log_dir - ) + return resolved_packages + + +# Visible for testing +def build_test_cases(tests: Iterable[Mapping[str, Any]]) -> List[TestCase]: + test_cases = [] + for test in [t['test_command'] for t in tests]: + assert test.startswith('test run ') + test = test[len('test run '):] + if ' -- ' in test: + package, args = test.split(' -- ', 1) + test_cases.append(TestCase(package=package, args=args)) + else: + test_cases.append(TestCase(package=test)) + return test_cases + + +def _bundled_test_runner_of(target_id: str) -> _BundledTestRunner: + log_dir = os.environ.get('FLUTTER_LOGS_DIR', '/tmp/log') + with open(os.path.join(os.path.dirname(__file__), 'test_suites.yaml'), 'r') as file: + tests = yaml.safe_load(file) + # TODO(zijiehe-google-com): Run tests with dart aot, + # https://github.com/flutter/flutter/issues/140179. + def dart_jit(test) -> bool: + return 'run_with_dart_aot' not in test or test['run_with_dart_aot'] != 'true' + + # TODO(zijiehe-google-com): Run all tests in release build, + # https://github.com/flutter/flutter/issues/140179. + def variant(test) -> bool: + return 'variant' not in test or test['variant'] == VARIANT + + tests = [t for t in tests if dart_jit(t) and variant(t)] + return _BundledTestRunner(target_id, resolve_packages(tests), build_test_cases(tests), log_dir) def _get_test_runner(runner_args: argparse.Namespace, *_) -> TestRunner: - return bundled_test_runner_of(runner_args.target_id) + return _bundled_test_runner_of(runner_args.target_id) if __name__ == '__main__': diff --git a/testing/fuchsia/run_tests_test.py b/testing/fuchsia/run_tests_test.py new file mode 100755 index 0000000000000..82b5d5aae433e --- /dev/null +++ b/testing/fuchsia/run_tests_test.py @@ -0,0 +1,63 @@ +#!/usr/bin/env vpython3 + +# [VPYTHON:BEGIN] +# python_version: "3.8" +# wheel < +# name: "infra/python/wheels/pyyaml/${platform}_${py_python}_${py_abi}" +# version: "version:5.4.1.chromium.1" +# > +# [VPYTHON:END] + +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import shutil +import unittest + +from pathlib import Path + +# pylint is looking for a wrong //testing/run_tests.py. +# pylint: disable=no-member +import run_tests + +run_tests.OUT_DIR = '/tmp/out/fuchsia_debug_x64' +os.makedirs(run_tests.OUT_DIR, exist_ok=True) + + +class RunTestsTest(unittest.TestCase): + + def test_resolve_both_package_and_packages(self): + packages = run_tests.resolve_packages([{'package': 'abc'}, {'packages': ['abc', 'def']}]) + self.assertEqual( + packages, {os.path.join(run_tests.OUT_DIR, 'abc'), + os.path.join(run_tests.OUT_DIR, 'def')} + ) + + def test_resolve_package_make_symbolic_link(self): + Path(os.path.join(run_tests.OUT_DIR, 'abc-0.far')).touch() + packages = run_tests.resolve_packages([ + {'package': 'abc-0.far'}, + ]) + self.assertEqual(packages, {os.path.join(run_tests.OUT_DIR, 'abc.far')}) + self.assertTrue(os.path.islink(os.path.join(run_tests.OUT_DIR, 'abc.far'))) + + def test_build_test_cases_with_arguments(self): + test_cases = run_tests.build_test_cases([ + {'test_command': 'test run abc'}, + {'test_command': 'test run def -- --args'}, + ]) + self.assertEqual( + test_cases, + [run_tests.TestCase(package='abc'), + run_tests.TestCase(package='def', args='--args')] + ) + + +if __name__ == '__main__': + try: + unittest.main() + finally: + # Clean up the temporary files. Since it's in /tmp/, ignore the errors. + shutil.rmtree(run_tests.OUT_DIR, ignore_errors=True) diff --git a/testing/fuchsia/test_suites.yaml b/testing/fuchsia/test_suites.yaml index e1ed249563b6d..c315a8aadf8cd 100644 --- a/testing/fuchsia/test_suites.yaml +++ b/testing/fuchsia/test_suites.yaml @@ -27,6 +27,7 @@ package: testing_tests-0.far - test_command: test run fuchsia-pkg://fuchsia.com/txt_tests#meta/txt_tests.cm -- --gtest_filter=-ParagraphTest.* package: txt_tests-0.far + variant: fuchsia_debug_x64 - test_command: test run fuchsia-pkg://fuchsia.com/ui_tests#meta/ui_tests.cm package: ui_tests-0.far variant: fuchsia_debug_x64