From c735b218a9393eb1a472fd7edbc3ad3793784e5a Mon Sep 17 00:00:00 2001 From: Hakkyu Kim Date: Wed, 14 Jul 2021 15:49:34 +0900 Subject: [PATCH 1/4] Drive examples script --- tools/drive_examples.py | 206 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100755 tools/drive_examples.py diff --git a/tools/drive_examples.py b/tools/drive_examples.py new file mode 100755 index 000000000..9630b7576 --- /dev/null +++ b/tools/drive_examples.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +import argparse +import subprocess +import sys +import os + +_INDENTATION = ' ' +_TERM_RED = '\033[1;31m' +_TERM_GREEN = '\033[1;32m' +_TERM_YELLOW = '\033[1;33m' +_TERM_EMPTY = '\033[0m' + + +class PluginResult: + def __init__(self, run_state, details=[]): + self.run_state = run_state + self.details = details + + @classmethod + def success(cls): + return cls('succeeded') + + # Skip may be used for testing on unsupported device profiles. + @classmethod + def skip(cls, reason): + return cls('skipped', details=[reason]) + + @classmethod + def fail(cls, errors=[]): + return cls('failed', details=errors) + + +def parse_args(args): + parser = argparse.ArgumentParser( + description='A script to run multiple tizen plugin driver tests.') + + parser.add_argument('--plugins', type=str, nargs='*', default=[], + help='Specifies which plugins to test. If it is not specified and --run-on-changed-packages is also not specified, then it will include every plugin under packages.') + parser.add_argument('--exclude', type=str, nargs='*', default=[], + help='Exclude plugins from test. Excluding plugins also works with --run-on-changed-packages flag.') + parser.add_argument('--run-on-changed-plugins', + default=False, action='store_true', help='Run the test on changed plugins. If --plugins is specified, this flag is ignored.') + parser.add_argument('--base-sha', type=str, default='', + help='The base sha used to determine git diff. This is useful when --run-on-changed-packages is specified. If not specified, merge-base is used as base sha.') + + return parser.parse_args(args) + + +def drive_example(plugin_dir): + example_dir = os.path.join(plugin_dir, 'example') + if not os.path.isdir(example_dir): + return PluginResult.fail('Missing example directory (use --exclude if this is intentional).') + + pubspec_path = os.path.join(example_dir, 'pubspec') + if not os.path.isfile(f'{pubspec_path}.yaml') and not os.path.isfile(f'{pubspec_path}.yml'): + return PluginResult.fail('Missing pubspec file in example directory') + + driver_paths = [] + driver_dir = os.path.join(example_dir, 'test_driver') + if os.path.isdir(driver_dir): + for driver_name in os.listdir(driver_dir): + driver_path = os.path.join(driver_dir, driver_name) + if os.path.isfile(driver_path) and driver_name.endswith('_test.dart'): + driver_paths.append(driver_path) + + test_target_paths = [] + integration_test_dir = os.path.join(example_dir, 'integration_test') + if os.path.isdir(integration_test_dir): + for test_target_name in os.listdir(integration_test_dir): + test_target_path = os.path.join( + integration_test_dir, test_target_name) + if os.path.isfile(test_target_path) and test_target_name.endswith('_test.dart'): + test_target_paths.append(test_target_path) + + errors = [] + tests_ran = False + if not driver_paths: + print(f'{_INDENTATION}No driver tests found for {os.path.basename(plugin_dir)}') + errors.append(f'No driver files for {os.path.basename(plugin_dir)}') + elif not test_target_paths: + driver_relpaths = [os.path.relpath(driver_path, os.path.dirname( + plugin_dir)) for driver_path in driver_paths] + print( + f'Found {driver_relpaths}, but no integration_test/*_test.dart files.') + errors.append(f'No test files for {driver_relpaths}') + + for driver_path in driver_paths: + for test_target_path in test_target_paths: + tests_ran = True + print(f'{_INDENTATION}Running --driver {os.path.basename(driver_path)} --target {os.path.basename(test_target_path)}') + completed_process = subprocess.run('flutter-tizen pub get', + shell=True, cwd=example_dir) + if completed_process.returncode != 0: + if not completed_process.stderr: + errors.append( + 'pub get failed, see the output above for details.') + else: + errors.append(completed_process.stderr) + continue + try: + completed_process = subprocess.run( + f'flutter-tizen drive --driver={driver_path} --target={test_target_path}', shell=True, cwd=example_dir, timeout=300) + if completed_process.returncode != 0: + if not completed_process.stderr: + errors.append( + 'flutter-tizen drive failed, see the output above for details.') + else: + errors.append(completed_process.stderr) + continue + except subprocess.TimeoutExpired: + errors.append('Timeout expired') + + if not tests_ran: + print( + f'{_INDENTATION}No driver tests were run for {os.path.basename(plugin_dir)}.') + errors.append('No tests ran (use --exclude if this is intentional).') + + return PluginResult.success() if len(errors) == 0 else PluginResult.fail(errors) + + +def main(argv): + args = parse_args(argv[1:]) + + packages_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), '../packages')) + + plugin_names = [] + if len(args.plugins) == 0 and args.run_on_changed_plugins: + base_sha = args.base_sha + if base_sha == '': + base_sha = subprocess.run('git merge-base --fork-point FETCH_HEAD HEAD', shell=True, + cwd=packages_dir, encoding='utf-8', stdout=subprocess.PIPE).stdout.strip() + if base_sha == '': + base_sha = subprocess.run( + 'git merge-base FETCH_HEAD HEAD', shell=True, cwd=packages_dir, encoding='utf-8', stdout=subprocess.PIPE).stdout.strip() + changed_files = subprocess.run( + f'git diff --name-only {base_sha} HEAD', shell=True, cwd=packages_dir, encoding='utf-8', stdout=subprocess.PIPE).stdout.strip().splitlines() + + changed_plugins = [] + for changed_file in changed_files: + relpath = os.path.relpath( + changed_file, start=os.path.dirname(packages_dir)) + path_segments = relpath.split('/') + if 'packages' not in path_segments: + continue + index = path_segments.index('packages') + if index < len(path_segments): + changed_plugins.append(path_segments[index + 1]) + plugin_names = list(set(changed_plugins)) + else: + for plugin_name in os.listdir(packages_dir): + plugin_dir = os.path.join(packages_dir, plugin_name) + if not os.path.isdir(plugin_dir): + continue + if len(args.plugins) > 0 and plugin_name not in args.plugins: + continue + plugin_names.append(plugin_name) + + excluded_plugins = [ + plugin_name for plugin_name in plugin_names if plugin_name in args.exclude] + plugin_names = [ + plugin_name for plugin_name in plugin_names if plugin_name not in args.exclude] + + test_num = 0 + total_plugin_num = len(plugin_names) + results = {} + for plugin_name in plugin_names: + plugin_dir = os.path.join(packages_dir, plugin_name) + test_num += 1 + print( + f'{_INDENTATION}Testing for {plugin_name} ({test_num}/{total_plugin_num})') + results[plugin_name] = drive_example(plugin_dir) + + print(f'============= TEST RESULT =============') + failed_plugins = [] + for plugin_name, result in results.items(): + color = _TERM_GREEN + if result.run_state == 'failed': + color = _TERM_RED + + print( + f'{color}{result.run_state.upper()}: {plugin_name}{_TERM_EMPTY}') + if result.run_state != 'succeeded': + print(f'{_INDENTATION}DETAILS: {result.details}') + if result.run_state == 'failed': + failed_plugins.append(plugin_name) + + exit_code = 0 + if len(failed_plugins) > 0: + print(f'{_TERM_RED}============= TEST FAILED ============={_TERM_EMPTY}') + for failed_plugin in failed_plugins: + print( + f'FAILED: {failed_plugin} DETAILS: {results[failed_plugin].details}') + exit_code = 1 + elif total_plugin_num == 0: + print('No tests ran.') + else: + print(f'{_TERM_GREEN}All tests passed.{_TERM_EMPTY}') + + for excluded_plugin in excluded_plugins: + print(f'{_TERM_YELLOW}EXCLUDED: {excluded_plugin}{_TERM_EMPTY}') + exit(exit_code) + + +if __name__ == "__main__": + main(sys.argv) From 19ae034322466e701b28596b0aa3150dd45eb527 Mon Sep 17 00:00:00 2001 From: Hakkyu Kim Date: Thu, 29 Jul 2021 20:57:38 +0900 Subject: [PATCH 2/4] Update script to work with flutter-tizen test --- tools/drive_examples.py | 206 ------------------------ tools/run_integration_test.py | 295 ++++++++++++++++++++++++++++++++++ 2 files changed, 295 insertions(+), 206 deletions(-) delete mode 100755 tools/drive_examples.py create mode 100755 tools/run_integration_test.py diff --git a/tools/drive_examples.py b/tools/drive_examples.py deleted file mode 100755 index 9630b7576..000000000 --- a/tools/drive_examples.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import subprocess -import sys -import os - -_INDENTATION = ' ' -_TERM_RED = '\033[1;31m' -_TERM_GREEN = '\033[1;32m' -_TERM_YELLOW = '\033[1;33m' -_TERM_EMPTY = '\033[0m' - - -class PluginResult: - def __init__(self, run_state, details=[]): - self.run_state = run_state - self.details = details - - @classmethod - def success(cls): - return cls('succeeded') - - # Skip may be used for testing on unsupported device profiles. - @classmethod - def skip(cls, reason): - return cls('skipped', details=[reason]) - - @classmethod - def fail(cls, errors=[]): - return cls('failed', details=errors) - - -def parse_args(args): - parser = argparse.ArgumentParser( - description='A script to run multiple tizen plugin driver tests.') - - parser.add_argument('--plugins', type=str, nargs='*', default=[], - help='Specifies which plugins to test. If it is not specified and --run-on-changed-packages is also not specified, then it will include every plugin under packages.') - parser.add_argument('--exclude', type=str, nargs='*', default=[], - help='Exclude plugins from test. Excluding plugins also works with --run-on-changed-packages flag.') - parser.add_argument('--run-on-changed-plugins', - default=False, action='store_true', help='Run the test on changed plugins. If --plugins is specified, this flag is ignored.') - parser.add_argument('--base-sha', type=str, default='', - help='The base sha used to determine git diff. This is useful when --run-on-changed-packages is specified. If not specified, merge-base is used as base sha.') - - return parser.parse_args(args) - - -def drive_example(plugin_dir): - example_dir = os.path.join(plugin_dir, 'example') - if not os.path.isdir(example_dir): - return PluginResult.fail('Missing example directory (use --exclude if this is intentional).') - - pubspec_path = os.path.join(example_dir, 'pubspec') - if not os.path.isfile(f'{pubspec_path}.yaml') and not os.path.isfile(f'{pubspec_path}.yml'): - return PluginResult.fail('Missing pubspec file in example directory') - - driver_paths = [] - driver_dir = os.path.join(example_dir, 'test_driver') - if os.path.isdir(driver_dir): - for driver_name in os.listdir(driver_dir): - driver_path = os.path.join(driver_dir, driver_name) - if os.path.isfile(driver_path) and driver_name.endswith('_test.dart'): - driver_paths.append(driver_path) - - test_target_paths = [] - integration_test_dir = os.path.join(example_dir, 'integration_test') - if os.path.isdir(integration_test_dir): - for test_target_name in os.listdir(integration_test_dir): - test_target_path = os.path.join( - integration_test_dir, test_target_name) - if os.path.isfile(test_target_path) and test_target_name.endswith('_test.dart'): - test_target_paths.append(test_target_path) - - errors = [] - tests_ran = False - if not driver_paths: - print(f'{_INDENTATION}No driver tests found for {os.path.basename(plugin_dir)}') - errors.append(f'No driver files for {os.path.basename(plugin_dir)}') - elif not test_target_paths: - driver_relpaths = [os.path.relpath(driver_path, os.path.dirname( - plugin_dir)) for driver_path in driver_paths] - print( - f'Found {driver_relpaths}, but no integration_test/*_test.dart files.') - errors.append(f'No test files for {driver_relpaths}') - - for driver_path in driver_paths: - for test_target_path in test_target_paths: - tests_ran = True - print(f'{_INDENTATION}Running --driver {os.path.basename(driver_path)} --target {os.path.basename(test_target_path)}') - completed_process = subprocess.run('flutter-tizen pub get', - shell=True, cwd=example_dir) - if completed_process.returncode != 0: - if not completed_process.stderr: - errors.append( - 'pub get failed, see the output above for details.') - else: - errors.append(completed_process.stderr) - continue - try: - completed_process = subprocess.run( - f'flutter-tizen drive --driver={driver_path} --target={test_target_path}', shell=True, cwd=example_dir, timeout=300) - if completed_process.returncode != 0: - if not completed_process.stderr: - errors.append( - 'flutter-tizen drive failed, see the output above for details.') - else: - errors.append(completed_process.stderr) - continue - except subprocess.TimeoutExpired: - errors.append('Timeout expired') - - if not tests_ran: - print( - f'{_INDENTATION}No driver tests were run for {os.path.basename(plugin_dir)}.') - errors.append('No tests ran (use --exclude if this is intentional).') - - return PluginResult.success() if len(errors) == 0 else PluginResult.fail(errors) - - -def main(argv): - args = parse_args(argv[1:]) - - packages_dir = os.path.abspath(os.path.join( - os.path.dirname(__file__), '../packages')) - - plugin_names = [] - if len(args.plugins) == 0 and args.run_on_changed_plugins: - base_sha = args.base_sha - if base_sha == '': - base_sha = subprocess.run('git merge-base --fork-point FETCH_HEAD HEAD', shell=True, - cwd=packages_dir, encoding='utf-8', stdout=subprocess.PIPE).stdout.strip() - if base_sha == '': - base_sha = subprocess.run( - 'git merge-base FETCH_HEAD HEAD', shell=True, cwd=packages_dir, encoding='utf-8', stdout=subprocess.PIPE).stdout.strip() - changed_files = subprocess.run( - f'git diff --name-only {base_sha} HEAD', shell=True, cwd=packages_dir, encoding='utf-8', stdout=subprocess.PIPE).stdout.strip().splitlines() - - changed_plugins = [] - for changed_file in changed_files: - relpath = os.path.relpath( - changed_file, start=os.path.dirname(packages_dir)) - path_segments = relpath.split('/') - if 'packages' not in path_segments: - continue - index = path_segments.index('packages') - if index < len(path_segments): - changed_plugins.append(path_segments[index + 1]) - plugin_names = list(set(changed_plugins)) - else: - for plugin_name in os.listdir(packages_dir): - plugin_dir = os.path.join(packages_dir, plugin_name) - if not os.path.isdir(plugin_dir): - continue - if len(args.plugins) > 0 and plugin_name not in args.plugins: - continue - plugin_names.append(plugin_name) - - excluded_plugins = [ - plugin_name for plugin_name in plugin_names if plugin_name in args.exclude] - plugin_names = [ - plugin_name for plugin_name in plugin_names if plugin_name not in args.exclude] - - test_num = 0 - total_plugin_num = len(plugin_names) - results = {} - for plugin_name in plugin_names: - plugin_dir = os.path.join(packages_dir, plugin_name) - test_num += 1 - print( - f'{_INDENTATION}Testing for {plugin_name} ({test_num}/{total_plugin_num})') - results[plugin_name] = drive_example(plugin_dir) - - print(f'============= TEST RESULT =============') - failed_plugins = [] - for plugin_name, result in results.items(): - color = _TERM_GREEN - if result.run_state == 'failed': - color = _TERM_RED - - print( - f'{color}{result.run_state.upper()}: {plugin_name}{_TERM_EMPTY}') - if result.run_state != 'succeeded': - print(f'{_INDENTATION}DETAILS: {result.details}') - if result.run_state == 'failed': - failed_plugins.append(plugin_name) - - exit_code = 0 - if len(failed_plugins) > 0: - print(f'{_TERM_RED}============= TEST FAILED ============={_TERM_EMPTY}') - for failed_plugin in failed_plugins: - print( - f'FAILED: {failed_plugin} DETAILS: {results[failed_plugin].details}') - exit_code = 1 - elif total_plugin_num == 0: - print('No tests ran.') - else: - print(f'{_TERM_GREEN}All tests passed.{_TERM_EMPTY}') - - for excluded_plugin in excluded_plugins: - print(f'{_TERM_YELLOW}EXCLUDED: {excluded_plugin}{_TERM_EMPTY}') - exit(exit_code) - - -if __name__ == "__main__": - main(sys.argv) diff --git a/tools/run_integration_test.py b/tools/run_integration_test.py new file mode 100755 index 000000000..1106ee116 --- /dev/null +++ b/tools/run_integration_test.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +"""A script that helps running integration tests for multiple Tizen plugins. + +To run integrations tests for all plugins under packages/, +run tools/run_integration_test.py in the root of this repository. + +Here are some of the common use cases. Assume packages contain plugins +a, b, c, d, and e. +- running specific packages: +tools/run_integration_test.py --plugins a b c # runs a, b, c +- excluding some packages: +tools/run_integration_test.py --exclude a b c # runs d, e +- exclude has precedence: +tools/run_integration_test.py --plugins a b c --exclude b # runs a c +""" +import argparse +import subprocess +import sys +import os +import time +import re + +_TERM_RED = '\033[1;31m' +_TERM_GREEN = '\033[1;32m' +_TERM_YELLOW = '\033[1;33m' +_TERM_EMPTY = '\033[0m' + +_LOG_PATTERN = r'\d\d:\d\d\s+([(\+\d+\s+)|(~\d+\s+)|(\-\d+\s+)]+):\s+(.*)' + + +class PluginResult: + """A class that specifies the result of a plugin integration test. + + Attributes: + run_state: The result of the test. Can be either succeeded for failed. + details: A list of details about the test result. (e.g. reasons for failure.) + """ + + def __init__(self, run_state, details=[]): + self.run_state = run_state + self.details = details + + @classmethod + def success(cls): + return cls('succeeded') + + @classmethod + def fail(cls, errors=[]): + return cls('failed', details=errors) + + +def parse_args(args): + parser = argparse.ArgumentParser( + description='A script to run multiple tizen plugin driver tests.') + + parser.add_argument( + '--plugins', + type=str, + nargs='*', + default=[], + help='Specifies which plugins to test. If it is not specified and \ + --run-on-changed-packages is also not specified, \ + then it will include every plugin under packages.') + parser.add_argument('--exclude', + type=str, + nargs='*', + default=[], + help='Exclude plugins from test.') + parser.add_argument( + '--run-on-changed-plugins', + default=False, + action='store_true', + help='Run the test on changed plugins. If --plugins is specified, \ + this flag is ignored.') + parser.add_argument( + '--base-sha', + type=str, + default='', + help='The base sha used to determine git diff. This is useful when \ + --run-on-changed-packages is specified. If not specified, \ + merge-base is used as base sha.') + parser.add_argument( + '--timeout', + type=int, + default=120, + help='Timeout limit of each integration test in seconds. \ + Default is 120 seconds.') + + return parser.parse_args(args) + + +def run_integration_test(plugin_dir, timeout): + """Runs integration test in the example package for plugin_dir + + Currently the tools assumes that there's only one example package per plugin. + + Args: + plugin_dir (str): The path to a single plugin directory. + timeout (int): Time limit in seconds before cancelling the test. + + Returns: + PluginResult: The result of the plugin integration test. + """ + example_dir = os.path.join(plugin_dir, 'example') + if not os.path.isdir(example_dir): + return PluginResult.fail([ + 'Missing example directory (use --exclude if this is intentional).' + ]) + + pubspec_path = os.path.join(example_dir, 'pubspec') + if not os.path.isfile(f'{pubspec_path}.yaml') and not os.path.isfile( + f'{pubspec_path}.yml'): + # TODO: Support multiple example packages. + return PluginResult.fail(['Missing pubspec file in example directory']) + + integration_test_dir = os.path.join(example_dir, 'integration_test') + if not os.path.isdir(integration_test_dir) or not os.listdir( + integration_test_dir): + return PluginResult.fail([ + 'Missing integration tests (use --exclude if this is intentional).' + ]) + + errors = [] + completed_process = subprocess.run('flutter-tizen pub get', + shell=True, + cwd=example_dir, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) + if completed_process.returncode != 0: + if not completed_process.stderr: + errors.append('pub get failed. Make sure the pubspec file \ + in your project is valid.') + else: + errors.append(completed_process.stderr) + return PluginResult.fail(errors) + + is_timed_out = False + process = subprocess.Popen('flutter-tizen test integration_test', + shell=True, + cwd=example_dir, + universal_newlines=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) + last_line = '' + start = time.time() + for line in process.stdout: + match = re.search(_LOG_PATTERN, line) + last_match = re.search(_LOG_PATTERN, last_line) + if match and last_match and last_match.group(2) == match.group(2): + sys.stdout.write(f'\r{line.strip()}') + else: + sys.stdout.write(f'\n{line.strip()}') + sys.stdout.flush() + last_line = line + if time.time() - start > timeout: + process.kill() + is_timed_out = True + break + sys.stdout.write('\n') + sys.stdout.flush() + process.wait() + + if is_timed_out: + errors.append("""Timeout expired. The test may need more time to finish. +If you expect the test to finish before timeout, check if the tests +require device screen to be awake or if they require manually +clicking the UI button for permissions.""") + return PluginResult.fail(errors) + if last_line.strip() == 'No tests ran.': + return PluginResult.fail(['No tests ran.']) + + match = re.search(_LOG_PATTERN, last_line.strip()) + if not match: + return PluginResult.fail(['Log message is not formatted correctly.']) + + # In some cases, the command returns 0 for failed cases, so we check again + # with the last log message. + exit_code = process.returncode + if match.group(2) == 'All tests passed!': + exit_code = 0 + elif match.group(2) == 'Some tests failed.': + errors.append( + 'flutter-tizen test integration_test failed, see the output above for details.' + ) + exit_code = 1 + + if exit_code == 0: + return PluginResult.success() + else: + return PluginResult.fail(errors) + + +def main(argv): + args = parse_args(argv[1:]) + + packages_dir = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../packages')) + existing_plugins = os.listdir(packages_dir) + for plugin in args.plugins: + if plugin not in existing_plugins: + print(f'{plugin} package does not exist, ignoring input...') + + plugin_names = [] + if len(args.plugins) == 0 and args.run_on_changed_plugins: + base_sha = args.base_sha + if base_sha == '': + base_sha = subprocess.run( + 'git merge-base --fork-point FETCH_HEAD HEAD', + shell=True, + cwd=packages_dir, + encoding='utf-8', + stdout=subprocess.PIPE).stdout.strip() + if base_sha == '': + base_sha = subprocess.run( + 'git merge-base FETCH_HEAD HEAD', + shell=True, + cwd=packages_dir, + encoding='utf-8', + stdout=subprocess.PIPE).stdout.strip() + + changed_files = subprocess.run( + f'git diff {base_sha} HEAD', + shell=True, + cwd=packages_dir, + encoding='utf-8', + stdout=subprocess.PIPE).stdout.strip().splitlines() + + changed_plugins = [] + for changed_file in changed_files: + relpath = os.path.relpath(changed_file, start=packages_dir) + path_segments = relpath.split('/') + if 'packages' not in path_segments: + continue + index = path_segments.index('packages') + if index < len(path_segments): + changed_plugins.append(path_segments[index + 1]) + plugin_names = list(set(changed_plugins)) + else: + for plugin_name in existing_plugins: + if not os.path.isdir(os.path.join(packages_dir, plugin_name)): + continue + if len(args.plugins) > 0 and plugin_name not in args.plugins: + continue + plugin_names.append(plugin_name) + + excluded_plugins = [] + testing_plugins = [] + for plugin_name in plugin_names: + if plugin_name in args.exclude: + excluded_plugins.append(plugin_name) + else: + testing_plugins.append(plugin_name) + + test_num = 0 + total_plugin_num = len(testing_plugins) + results = {} + for testing_plugin in testing_plugins: + test_num += 1 + print( + f'============= Testing for {testing_plugin} ({test_num}/{total_plugin_num}) =============' + ) + results[testing_plugin] = run_integration_test( + os.path.join(packages_dir, testing_plugin), args.timeout) + + print(f'============= TEST RESULT =============') + failed_plugins = [] + for testing_plugin, result in results.items(): + color = _TERM_GREEN + if result.run_state == 'failed': + color = _TERM_RED + + print( + f'{color}{result.run_state.upper()}: {testing_plugin}{_TERM_EMPTY}') + if result.run_state != 'succeeded': + for detail in result.details: + print(f'{detail}') + if result.run_state == 'failed': + failed_plugins.append(testing_plugin) + + exit_code = 0 + if len(failed_plugins) > 0: + print(f'{_TERM_RED}Some tests failed.{_TERM_EMPTY}') + exit_code = 1 + elif total_plugin_num == 0: + print('No tests ran.') + else: + print(f'{_TERM_GREEN}All tests passed!{_TERM_EMPTY}') + + for excluded_plugin in excluded_plugins: + print(f'{_TERM_YELLOW}EXCLUDED: {excluded_plugin}{_TERM_EMPTY}') + exit(exit_code) + + +if __name__ == "__main__": + main(sys.argv) From 70985046131c94a7afa3fdc010ec0fa25a3689b7 Mon Sep 17 00:00:00 2001 From: Hakkyu Kim Date: Fri, 30 Jul 2021 11:53:13 +0900 Subject: [PATCH 3/4] Handle some edge cases --- tools/run_integration_test.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tools/run_integration_test.py b/tools/run_integration_test.py index 1106ee116..259056188 100755 --- a/tools/run_integration_test.py +++ b/tools/run_integration_test.py @@ -134,6 +134,22 @@ def run_integration_test(plugin_dir, timeout): errors.append(completed_process.stderr) return PluginResult.fail(errors) + # This prevents ci test failures when depending packages accidentally + # publishes breaking changes as minor or patch versions. + # (e.g. package_info_plus 1.0.4 breaks PackageInfo's public constructor.) + completed_process = subprocess.run('flutter-tizen pub downgrade', + shell=True, + cwd=example_dir, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) + if completed_process.returncode != 0: + if not completed_process.stderr: + errors.append('pub downgrade failed. Make sure the pubspec file \ + in your project is valid.') + else: + errors.append(completed_process.stderr) + return PluginResult.fail(errors) + is_timed_out = False process = subprocess.Popen('flutter-tizen test integration_test', shell=True, @@ -168,6 +184,10 @@ def run_integration_test(plugin_dir, timeout): return PluginResult.fail(errors) if last_line.strip() == 'No tests ran.': return PluginResult.fail(['No tests ran.']) + elif last_line.strip().startswith('No devices found'): + return PluginResult.fail([ + 'The runner cannot find any devices to run tests. Check if the hosted test server has connections to Tizen devices.' + ]) match = re.search(_LOG_PATTERN, last_line.strip()) if not match: From b8d946edfd559d742a55d2f5bd14c610dc6ffdd6 Mon Sep 17 00:00:00 2001 From: Hakkyu Kim Date: Wed, 4 Aug 2021 09:51:11 +0900 Subject: [PATCH 4/4] Update based on review --- tools/run_integration_test.py | 46 ++++++++++++----------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/tools/run_integration_test.py b/tools/run_integration_test.py index 259056188..712a3b5e6 100755 --- a/tools/run_integration_test.py +++ b/tools/run_integration_test.py @@ -28,7 +28,7 @@ _LOG_PATTERN = r'\d\d:\d\d\s+([(\+\d+\s+)|(~\d+\s+)|(\-\d+\s+)]+):\s+(.*)' -class PluginResult: +class TestResult: """A class that specifies the result of a plugin integration test. Attributes: @@ -51,7 +51,7 @@ def fail(cls, errors=[]): def parse_args(args): parser = argparse.ArgumentParser( - description='A script to run multiple tizen plugin driver tests.') + description='A script to run multiple Tizen plugin driver tests.') parser.add_argument( '--plugins', @@ -67,7 +67,7 @@ def parse_args(args): default=[], help='Exclude plugins from test.') parser.add_argument( - '--run-on-changed-plugins', + '--run-on-changed-packages', default=False, action='store_true', help='Run the test on changed plugins. If --plugins is specified, \ @@ -99,11 +99,11 @@ def run_integration_test(plugin_dir, timeout): timeout (int): Time limit in seconds before cancelling the test. Returns: - PluginResult: The result of the plugin integration test. + TestResult: The result of the plugin integration test. """ example_dir = os.path.join(plugin_dir, 'example') if not os.path.isdir(example_dir): - return PluginResult.fail([ + return TestResult.fail([ 'Missing example directory (use --exclude if this is intentional).' ]) @@ -111,12 +111,12 @@ def run_integration_test(plugin_dir, timeout): if not os.path.isfile(f'{pubspec_path}.yaml') and not os.path.isfile( f'{pubspec_path}.yml'): # TODO: Support multiple example packages. - return PluginResult.fail(['Missing pubspec file in example directory']) + return TestResult.fail(['Missing pubspec file in example directory']) integration_test_dir = os.path.join(example_dir, 'integration_test') if not os.path.isdir(integration_test_dir) or not os.listdir( integration_test_dir): - return PluginResult.fail([ + return TestResult.fail([ 'Missing integration tests (use --exclude if this is intentional).' ]) @@ -132,23 +132,7 @@ def run_integration_test(plugin_dir, timeout): in your project is valid.') else: errors.append(completed_process.stderr) - return PluginResult.fail(errors) - - # This prevents ci test failures when depending packages accidentally - # publishes breaking changes as minor or patch versions. - # (e.g. package_info_plus 1.0.4 breaks PackageInfo's public constructor.) - completed_process = subprocess.run('flutter-tizen pub downgrade', - shell=True, - cwd=example_dir, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) - if completed_process.returncode != 0: - if not completed_process.stderr: - errors.append('pub downgrade failed. Make sure the pubspec file \ - in your project is valid.') - else: - errors.append(completed_process.stderr) - return PluginResult.fail(errors) + return TestResult.fail(errors) is_timed_out = False process = subprocess.Popen('flutter-tizen test integration_test', @@ -181,17 +165,17 @@ def run_integration_test(plugin_dir, timeout): If you expect the test to finish before timeout, check if the tests require device screen to be awake or if they require manually clicking the UI button for permissions.""") - return PluginResult.fail(errors) + return TestResult.fail(errors) if last_line.strip() == 'No tests ran.': - return PluginResult.fail(['No tests ran.']) + return TestResult.fail(['No tests ran.']) elif last_line.strip().startswith('No devices found'): - return PluginResult.fail([ + return TestResult.fail([ 'The runner cannot find any devices to run tests. Check if the hosted test server has connections to Tizen devices.' ]) match = re.search(_LOG_PATTERN, last_line.strip()) if not match: - return PluginResult.fail(['Log message is not formatted correctly.']) + return TestResult.fail(['Log message is not formatted correctly.']) # In some cases, the command returns 0 for failed cases, so we check again # with the last log message. @@ -205,9 +189,9 @@ def run_integration_test(plugin_dir, timeout): exit_code = 1 if exit_code == 0: - return PluginResult.success() + return TestResult.success() else: - return PluginResult.fail(errors) + return TestResult.fail(errors) def main(argv): @@ -221,7 +205,7 @@ def main(argv): print(f'{plugin} package does not exist, ignoring input...') plugin_names = [] - if len(args.plugins) == 0 and args.run_on_changed_plugins: + if len(args.plugins) == 0 and args.run_on_changed_packages: base_sha = args.base_sha if base_sha == '': base_sha = subprocess.run(