diff --git a/tools/commands/integration_test.py b/tools/commands/integration_test.py index b2104a6ef..e5bd7a518 100755 --- a/tools/commands/integration_test.py +++ b/tools/commands/integration_test.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - """A script that helps running integration tests for multiple Tizen plugins. To run integrations tests for all plugins under packages/, @@ -15,13 +14,14 @@ tools/run_integration_test.py --plugins a b c --exclude b # runs a c """ -import argparse import os import re import subprocess import sys import time +import yaml + import commands.command_utils as command_utils _TERM_RED = '\033[1;31m' @@ -31,6 +31,8 @@ _LOG_PATTERN = r'\d\d:\d\d\s+([(\+\d+\s+)|(~\d+\s+)|(\-\d+\s+)]+):\s+(.*)' +_DEFAULT_TEST_TARGET = 'wearable-5.5' + class TestResult: """A class that specifies the result of a plugin integration test. @@ -40,56 +42,97 @@ class TestResult: details: A list of details about the test result. (e.g. reasons for failure.) """ - def __init__(self, run_state, details=[]): + def __init__(self, plugin_name, run_state, test_target='', details=[]): + self.plugin_name = plugin_name + self.test_target = test_target self.run_state = run_state self.details = details @classmethod - def success(cls): - return cls('succeeded') + def success(cls, plugin_name, test_target): + return cls(plugin_name, 'succeeded', test_target=test_target) @classmethod - def fail(cls, errors=[]): - return cls('failed', details=errors) + def fail(cls, plugin_name, test_target, errors=[]): + return cls(plugin_name, + 'failed', + test_target=test_target, + details=errors) def parse_args(args): - parser = command_utils.get_options_parser( - plugins=True, exclude=True, run_on_changed_packages=True, base_sha=True, timeout=True, command='test') + parser = command_utils.get_options_parser(plugins=True, + exclude=True, + run_on_changed_packages=True, + base_sha=True, + timeout=True, + command='test') + parser.add_argument( + '--recipe', + type=str, + default='', + help= + '''The recipe file path. A recipe refers to a yaml file that defines a list of test targets for plugins. +Passing this file will allow the tool to test with specified targets instead of the default target, wearable-5.5. +( +plugins: + a: [wearable-5.5, tv-6.0] + b: [mobile-6.0] + c: [wearable-4.0] +)''') return parser.parse_args(args) -def _integration_test(plugin_dir, timeout): +def _integration_test(plugin_dir, test_targets, timeout): """Runs integration test in the example package for plugin_dir Currently the tools assumes that there's only one example package per plugin. Args: plugin_dir (str): The path to a single plugin directory. + test_targets (List[str]): A list of testing targets. timeout (int): Time limit in seconds before cancelling the test. Returns: TestResult: The result of the plugin integration test. """ + plugin_name = os.path.basename(plugin_dir) + + if not test_targets: + # (TODO: HakkyuKim) Improve logic for setting default targets. + test_targets.append(_DEFAULT_TEST_TARGET) + example_dir = os.path.join(plugin_dir, 'example') if not os.path.isdir(example_dir): - return TestResult.fail([ - 'Missing example directory (use --exclude if this is intentional).' - ]) + return [ + TestResult.fail( + plugin_name, + errors=[ + 'Missing example directory (use --exclude if this is intentional).' + ]) + ] pubspec_path = os.path.join(example_dir, 'pubspec') if not os.path.isfile(f'{pubspec_path}.yaml') and not os.path.isfile( f'{pubspec_path}.yml'): # TODO: Support multiple example packages. - return TestResult.fail(['Missing pubspec file in example directory']) + return [ + TestResult.fail( + plugin_name, + errors=['Missing pubspec file in example directory.']) + ] integration_test_dir = os.path.join(example_dir, 'integration_test') if not os.path.isdir(integration_test_dir) or not os.listdir( integration_test_dir): - return TestResult.fail([ - 'Missing integration tests (use --exclude if this is intentional).' - ]) + return [ + TestResult.fail( + plugin_name, + errors=[ + 'Missing integration tests (use --exclude if this is intentional).' + ]) + ] errors = [] completed_process = subprocess.run('flutter-tizen pub get', @@ -103,95 +146,191 @@ def _integration_test(plugin_dir, timeout): in your project is valid.') else: errors.append(completed_process.stderr) - return TestResult.fail(errors) - - is_timed_out = False - process = subprocess.Popen('flutter-tizen test integration_test', - shell=True, - cwd=example_dir, - universal_newlines=True, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) - last_line = '' - start = time.time() - for line in process.stdout: - match = re.search(_LOG_PATTERN, line) - last_match = re.search(_LOG_PATTERN, last_line) - if match and last_match and last_match.group(2) == match.group(2): - sys.stdout.write(f'\r{line.strip()}') - else: - sys.stdout.write(f'\n{line.strip()}') + return [TestResult.fail(plugin_name, errors=errors)] + + test_results = [] + target_table = _get_target_table() + + for test_target in test_targets: + if test_target not in target_table: + test_results.append( + TestResult.fail( + plugin_name, test_target, + [f'Test runner cannot find target {test_target}.'])) + continue + + is_timed_out = False + process = subprocess.Popen( + f'flutter-tizen -d {target_table[test_target]} test integration_test', + shell=True, + cwd=example_dir, + universal_newlines=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) + last_line = '' + start = time.time() + for line in process.stdout: + match = re.search(_LOG_PATTERN, line) + last_match = re.search(_LOG_PATTERN, last_line) + if match and last_match and last_match.group(2) == match.group(2): + sys.stdout.write(f'\r{line.strip()}') + else: + sys.stdout.write(f'\n{line.strip()}') + sys.stdout.flush() + last_line = line + if time.time() - start > timeout: + process.kill() + is_timed_out = True + break + sys.stdout.write('\n') sys.stdout.flush() - last_line = line - if time.time() - start > timeout: - process.kill() - is_timed_out = True - break - sys.stdout.write('\n') - sys.stdout.flush() - process.wait() - - if is_timed_out: - errors.append("""Timeout expired. The test may need more time to finish. -If you expect the test to finish before timeout, check if the tests -require device screen to be awake or if they require manually -clicking the UI button for permissions.""") - return TestResult.fail(errors) - if last_line.strip() == 'No tests ran.': - return TestResult.fail(['No tests ran.']) - elif last_line.strip().startswith('No devices found'): - return TestResult.fail([ - 'The runner cannot find any devices to run tests. Check if the hosted test server has connections to Tizen devices.' - ]) - - match = re.search(_LOG_PATTERN, last_line.strip()) - if not match: - return TestResult.fail(['Log message is not formatted correctly.']) - - # In some cases, the command returns 0 for failed cases, so we check again - # with the last log message. - exit_code = process.returncode - if match.group(2) == 'All tests passed!': - exit_code = 0 - elif match.group(2) == 'Some tests failed.': - errors.append( - 'flutter-tizen test integration_test failed, see the output above for details.' - ) - exit_code = 1 + process.wait() + + if is_timed_out: + errors.append( + """Timeout expired. The test may need more time to finish. + If you expect the test to finish before timeout, check if the tests + require device screen to be awake or if they require manually + clicking the UI button for permissions.""") + test_results.append( + TestResult.fail(plugin_name, test_target, errors=errors)) + continue + if last_line.strip() == 'No tests ran.': + test_results.append( + TestResult.fail(plugin_name, test_target, ['No tests ran.'])) + continue + elif last_line.strip().startswith('No devices found'): + test_results.append( + TestResult.fail( + plugin_name, test_target, + ['The runner cannot find any devices to run tests.'])) + continue + + match = re.search(_LOG_PATTERN, last_line.strip()) + if not match: + test_results.append( + TestResult.fail(plugin_name, test_target, + ['Log message is not formatted correctly.'])) + continue + + # In some cases, the command returns 0 for failed cases, so we check again + # with the last log message. + exit_code = process.returncode + if match.group(2) == 'All tests passed!': + exit_code = 0 + elif match.group(2) == 'Some tests failed.': + errors.append( + 'flutter-tizen test integration_test failed, see the output above for details.' + ) + exit_code = 1 + + if exit_code == 0: + test_results.append(TestResult.success(plugin_name, test_target)) + else: + test_results.append( + TestResult.fail(plugin_name, test_target, errors=errors)) + + return test_results - if exit_code == 0: - return TestResult.success() - else: - return TestResult.fail(errors) + +def _get_target_table(): + + def _parse_target_info(capability_info: str): + capability_info.rstrip() + profile_name = '' + platform_version = '' + for line in capability_info.split('\n'): + tokens = line.split(':') + if (tokens[0] == 'profile_name'): + profile_name = tokens[1] + elif (tokens[0] == 'platform_version'): + platform_version = tokens[1] + return profile_name, platform_version + + completed_process = subprocess.run('sdb devices', + shell=True, + cwd='.', + universal_newlines=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) + + table = {} + if completed_process.returncode == 0: + lines = completed_process.stdout.rstrip().split('\n') + for line in lines[1:]: + id = line.split(' ')[0] + completed_process = subprocess.run(f'sdb -s {id} capability', + shell=True, + cwd='.', + universal_newlines=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) + profile, platform_version = _parse_target_info( + completed_process.stdout) + if not profile or not platform_version: + print( + f'''Cannot extract profile or platform_version information from device {id}. +profile: {profile} +platform_version: {platform_version}''') + if f'{profile}-{platform_version}' in table: + print( + f'Multiple targets of {profile}-{platform_version} found. Replacing {table[{profile}-{platform_version}]} to {id}...' + ) + table[f'{profile}-{platform_version}'] = id + return table def run_integration_test(argv): args = parse_args(argv) + + test_targets = {} + if args.recipe: + if not os.path.isfile(args.recipe): + print(f'The recipe file {args.recipe} does not exist.') + exit(1) + with open(args.recipe) as f: + try: + test_targets = yaml.load(f.read(), + Loader=yaml.FullLoader)['plugins'] + except yaml.parser.ParserError: + print( + f'The recipe file {args.recipe} is not a valid yaml file.') + exit(1) + packages_dir = command_utils.get_package_dir() testing_plugins, excluded_plugins = command_utils.get_target_plugins( - packages_dir, plugins=args.plugins, exclude=args.exclude, run_on_changed_packages=args.run_on_changed_packages, + packages_dir, + plugins=args.plugins, + exclude=args.exclude, + run_on_changed_packages=args.run_on_changed_packages, base_sha=args.base_sha) test_num = 0 total_plugin_num = len(testing_plugins) - results = {} + results = [] for testing_plugin in testing_plugins: test_num += 1 print( f'============= Testing for {testing_plugin} ({test_num}/{total_plugin_num}) =============' ) - results[testing_plugin] = _integration_test( - os.path.join(packages_dir, testing_plugin), args.timeout) + test_targets_list = [] + if testing_plugin in test_targets: + test_targets_list = test_targets[testing_plugin] + + results.extend( + _integration_test(os.path.join(packages_dir, testing_plugin), + test_targets_list, args.timeout)) print(f'============= TEST RESULT =============') failed_plugins = [] - for testing_plugin, result in results.items(): + for result in results: color = _TERM_GREEN if result.run_state == 'failed': color = _TERM_RED print( - f'{color}{result.run_state.upper()}: {testing_plugin}{_TERM_EMPTY}') + f'{color}{result.run_state.upper()}: {result.plugin_name} {result.test_target}{_TERM_EMPTY}' + ) if result.run_state != 'succeeded': for detail in result.details: print(f'{detail}') diff --git a/tools/commands/requirements.txt b/tools/commands/requirements.txt new file mode 100644 index 000000000..932bd69ef --- /dev/null +++ b/tools/commands/requirements.txt @@ -0,0 +1 @@ +PyYAML==5.4.1