diff --git a/.github/workflows/gh-actions.yml b/.github/workflows/gh-actions.yml index 2833072a07..b8c20f1145 100644 --- a/.github/workflows/gh-actions.yml +++ b/.github/workflows/gh-actions.yml @@ -70,8 +70,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: $RUNNER --test262 - - run: $RUNNER --test262 --build-debug + - run: $RUNNER --test262=update + - run: $RUNNER --test262=update --build-debug Conformance_Tests_ES2015: runs-on: ubuntu-latest diff --git a/tests/test262-es5.1-excludelist.xml b/tests/test262-es5.1-excludelist.xml new file mode 100644 index 0000000000..09376a5e6c --- /dev/null +++ b/tests/test262-es5.1-excludelist.xml @@ -0,0 +1,151 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/run-tests.py b/tools/run-tests.py index ce96fee4c9..96b354bdc3 100755 --- a/tools/run-tests.py +++ b/tools/run-tests.py @@ -198,8 +198,10 @@ def get_arguments(): help='Run jerry-debugger tests') parser.add_argument('--jerry-tests', action='store_true', help='Run jerry-tests') - parser.add_argument('--test262', action='store_true', - help='Run test262 - ES5.1') + parser.add_argument('--test262', default=False, const='default', + nargs='?', choices=['default', 'all', 'update'], + help='Run test262 - ES5.1. default: all tests except excludelist, ' + + 'all: all tests, update: all tests and update excludelist') parser.add_argument('--test262-es2015', default=False, const='default', nargs='?', choices=['default', 'all', 'update'], help='Run test262 - ES2015. default: all tests except excludelist, ' + @@ -437,7 +439,8 @@ def run_test262_test_suite(options): test_cmd.append('--esnext') test_cmd.append(options.test262_esnext) else: - test_cmd.append('--es51') + test_cmd.append('--es5.1') + test_cmd.append(options.test262) if job.test_args: test_cmd.extend(job.test_args) diff --git a/tools/runners/run-test-suite-test262.py b/tools/runners/run-test-suite-test262.py index 4da81d4a09..1c42081047 100755 --- a/tools/runners/run-test-suite-test262.py +++ b/tools/runners/run-test-suite-test262.py @@ -18,7 +18,6 @@ import argparse import os import re -import shutil import subprocess import sys @@ -40,8 +39,10 @@ def get_arguments(): parser.add_argument('--test-dir', metavar='DIR', required=True, help='Directory contains test262 test suite') group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('--es51', action='store_true', - help='Run test262 ES5.1 version') + group.add_argument('--es5.1', dest='es5_1', default=False, const='default', + nargs='?', choices=['default', 'all', 'update'], + help='Run test262 - ES5.1. default: all tests except excludelist, ' + + 'all: all tests, update: all tests and update excludelist') group.add_argument('--es2015', default=False, const='default', nargs='?', choices=['default', 'all', 'update'], help='Run test262 - ES2015. default: all tests except excludelist, ' + @@ -57,20 +58,18 @@ def get_arguments(): if args.es2015: args.test_dir = os.path.join(args.test_dir, 'es2015') - args.test262_harness_dir = os.path.abspath(os.path.dirname(__file__)) args.test262_git_hash = 'fd44cd73dfbce0b515a2474b7cd505d6176a9eb5' args.excludelist_path = os.path.join('tests', 'test262-es6-excludelist.xml') elif args.esnext: args.test_dir = os.path.join(args.test_dir, 'esnext') - args.test262_harness_dir = os.path.abspath(os.path.dirname(__file__)) args.test262_git_hash = '281eb10b2844929a7c0ac04527f5b42ce56509fd' args.excludelist_path = os.path.join('tests', 'test262-esnext-excludelist.xml') else: - args.test_dir = os.path.join(args.test_dir, 'es51') - args.test262_harness_dir = args.test_dir + args.test_dir = os.path.join(args.test_dir, 'es5.1') args.test262_git_hash = 'es5-tests' + args.excludelist_path = os.path.join('tests', 'test262-es5.1-excludelist.xml') - args.mode = args.es2015 or args.esnext + args.mode = args.es2015 or args.esnext or args.es5_1 return args @@ -88,15 +87,6 @@ def prepare_test262_test_suite(args): return_code = subprocess.call(['git', 'checkout', args.test262_git_hash], cwd=args.test_dir) assert not return_code, 'Cloning test262 repository failed - invalid git revision.' - if args.es51: - path_to_remove = os.path.join(args.test_dir, 'test', 'suite', 'bestPractice') - if os.path.isdir(path_to_remove): - shutil.rmtree(path_to_remove) - - path_to_remove = os.path.join(args.test_dir, 'test', 'suite', 'intl402') - if os.path.isdir(path_to_remove): - shutil.rmtree(path_to_remove) - # Since ES2018 iterator's next method is called once during the prologue of iteration, # rather than during each step. The test is incorrect and stuck in an infinite loop. # https://github.com/tc39/test262/pull/1248 fixed the test and it passes on test262-esnext. @@ -187,16 +177,16 @@ def main(args): if sys.version_info.major >= 3: kwargs['errors'] = 'ignore' - if args.es51: - test262_harness_path = os.path.join(args.test262_harness_dir, 'tools/packaging/test262.py') - else: - test262_harness_path = os.path.join(args.test262_harness_dir, 'test262-harness.py') - + test262_harness_dir = os.path.abspath(os.path.dirname(__file__)) + test262_harness_path = os.path.join(test262_harness_dir, 'test262-harness.py') test262_command = get_platform_cmd_prefix() + \ [test262_harness_path, '--command', command, '--tests', args.test_dir, '--summary'] + if args.es5_1: + test262_command.extend(['--es5.1']) + test262_command.extend(['--unmarked_default', 'non_strict']) if 'excludelist_path' in args and args.mode == 'default': test262_command.extend(['--exclude-list', args.excludelist_path]) diff --git a/tools/runners/test262-harness.py b/tools/runners/test262-harness.py index 68f5bd6a1b..99c1ef1cfd 100755 --- a/tools/runners/test262-harness.py +++ b/tools/runners/test262-harness.py @@ -324,6 +324,46 @@ def parse_test_record(src, name, onerror=print): return test_record +_STARS_ES5_1 = re.compile(r"\s*\n\s*\*\s?") +_AT_ATTRS_ES5_1 = re.compile(r"\s*\n\s*\*\s*@") +_RECORD_PATTERN_ES5_1 = re.compile( # Should match anything + r"^((?:(?:\s*\/\/.*)?\s*\n)*)" # header pattern + + r"(?:\/\*\*?((?:\s|\S)*?)\*\/\s*\n)" # capture comment pattern + + r"?((?:\s|\S)*)$" # any pattern +) + + +def strip_stars_es5_1(text): + return _STARS_ES5_1.sub('\n', text).strip() + + +def parse_test_record_es5_1(src, name, onerror=print): + test_record = {} + match = _RECORD_PATTERN_ES5_1.match(src) + if match is None: + onerror('unrecognized: ' + name) + test_record['header'] = match.group(1).strip() + test_record['test'] = match.group(3) # do not trim + if match.group(2): + prop_texts = _AT_ATTRS_ES5_1.split(match.group(2)) + test_record['commentary'] = strip_stars_es5_1(prop_texts[0]) + del prop_texts[0] + for prop_text in prop_texts: + prop_match = re.match(r"^\w+", prop_text) + if prop_match is None: + onerror('Malformed "@" attribute: ' + name) + prop_name = prop_match.group(0) + prop_val = strip_stars_es5_1(prop_text[len(prop_name):]) + + if prop_name in test_record: + onerror('duplicate: ' + prop_name) + if prop_name == 'negative': + test_record['negative'] = {'type': ''} + else: + test_record[prop_name] = prop_val + return test_record + + ####################################################################### # based on test262.py ####################################################################### @@ -344,6 +384,8 @@ def build_options(): help="The command-line to run") result.add_option("--tests", default=path.abspath('.'), help="Path to the tests") + result.add_option('--es5.1', dest='es5_1', default=False, action='store_true', + help='Run test262 ES5.1 version') result.add_option("--exclude-list", default=None, help="Path to the excludelist.xml file") result.add_option("--cat", default=False, action="store_true", @@ -497,7 +539,10 @@ def __init__(self, suite, name, full_path, strict_mode, command_template, module self.strict_mode = strict_mode with open(self.full_path) as file_desc: self.contents = file_desc.read() - test_record = parse_test_record(self.contents, name) + if self.suite.es5_1: + test_record = parse_test_record_es5_1(self.contents, name) + else: + test_record = parse_test_record(self.contents, name) self.test = test_record["test"] del test_record["test"] del test_record["header"] @@ -565,6 +610,25 @@ def get_additional_includes(self): return '\n'.join([self.suite.get_include(include) for include in self.get_include_list()]) def get_source(self): + if self.suite.es5_1: + return self.get_source_es5_1() + return self.get_source_esnext() + + def get_source_es5_1(self): + source = self.suite.get_include('cth.js') + \ + self.suite.get_include('sta.js') + \ + self.suite.get_include('ed.js') + \ + self.suite.get_include('testBuiltInObject.js') + \ + self.suite.get_include('testIntl.js') + \ + self.test + '\n' + + if self.strict_mode: + source = '"use strict";\nvar strict_mode = true;\n' + source + else: + source = 'var strict_mode = false;\n' + source + return source + + def get_source_esnext(self): if self.is_raw(): return self.test @@ -601,8 +665,7 @@ def get_parameter(match): return re.sub(r"\{\{(\w+)\}\}", get_parameter, template) - @staticmethod - def execute(command): + def execute(self, command): if is_windows(): args = '%s' % command else: @@ -617,7 +680,8 @@ def execute(command): stdout=stdout.file_desc, stderr=stderr.file_desc ) - timer = threading.Timer(TEST262_CASE_TIMEOUT, process.kill) + timeout = None if self.suite.es5_1 else TEST262_CASE_TIMEOUT + timer = threading.Timer(timeout, process.kill) timer.start() code = process.wait() timer.cancel() @@ -641,7 +705,7 @@ def run_test_in(self, tmp): 'path': arg }) - (code, out, err) = TestCase.execute(command) + (code, out, err) = self.execute(command) return TestResult(code, out, err, self) def run(self): @@ -715,9 +779,15 @@ def percent_format(partial, total): class TestSuite(object): + #pylint: disable=too-many-instance-attributes def __init__(self, options): - self.test_root = path.join(options.tests, 'test') - self.lib_root = path.join(options.tests, 'harness') + self.es5_1 = options.es5_1 + if self.es5_1: + self.test_root = path.join(options.tests, 'test', 'suite') + self.lib_root = path.join(options.tests, 'test', 'harness') + else: + self.test_root = path.join(options.tests, 'test') + self.lib_root = path.join(options.tests, 'harness') self.strict_only = options.strict_only self.non_strict_only = options.non_strict_only self.unmarked_default = options.unmarked_default