Skip to content

Using a single test262-harness.py for es5.1, es2015 and esnext. #4389

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
lygstate wants to merge 1 commit into from
Closed

Using a single test262-harness.py for es5.1, es2015 and esnext. #4389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/gh-actions.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: $RUNNER --test262
- run: $RUNNER --test262 --build-debug
- run: $RUNNER --test262=update
- run: $RUNNER --test262=update --build-debug

Conformance_Tests_ES2015:
runs-on: ubuntu-latest
Expand Down
151 changes: 151 additions & 0 deletions tests/test262-es5.1-excludelist.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
<?xml version="1.0" encoding="utf-8" ?>
<excludeList>
<test id="bestPractice/Sbp_A1_T1.js"><reason></reason></test>
<test id="bestPractice/Sbp_A2_T1.js"><reason></reason></test>
<test id="bestPractice/Sbp_A2_T2.js"><reason></reason></test>
<test id="bestPractice/Sbp_A3_T1.js"><reason></reason></test>
<test id="bestPractice/Sbp_A3_T2.js"><reason></reason></test>
<test id="bestPractice/Sbp_A4_T1.js"><reason></reason></test>
<test id="bestPractice/Sbp_A4_T2.js"><reason></reason></test>
<test id="bestPractice/Sbp_A5_T2.js"><reason></reason></test>
<test id="intl402/ch06/6.2/6.2.2_a.js"><reason></reason></test>
<test id="intl402/ch06/6.2/6.2.2_b.js"><reason></reason></test>
<test id="intl402/ch06/6.2/6.2.2_c.js"><reason></reason></test>
<test id="intl402/ch06/6.2/6.2.3.js"><reason></reason></test>
<test id="intl402/ch06/6.2/6.2.4.js"><reason></reason></test>
<test id="intl402/ch06/6.3/6.3.1_a.js"><reason></reason></test>
<test id="intl402/ch06/6.3/6.3.1_b.js"><reason></reason></test>
<test id="intl402/ch06/6.4/6.4_a.js"><reason></reason></test>
<test id="intl402/ch06/6.4/6.4_b.js"><reason></reason></test>
<test id="intl402/ch06/6.4/6.4_c.js"><reason></reason></test>
<test id="intl402/ch08/8.0/8.0.js"><reason></reason></test>
<test id="intl402/ch08/8.0/8.0_L15.js"><reason></reason></test>
<test id="intl402/ch09/9.1/9.1_a.js"><reason></reason></test>
<test id="intl402/ch09/9.1/9.1_b.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.1_1.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.1_2.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.1_3.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.1_4.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.1_8_c_ii.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.1_8_c_vi.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.2.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.3_5.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.5_11_g_ii_2.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.5_6.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.6_2.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.6_4.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.6_4_b.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.6_4_c.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.8_1_c.js"><reason></reason></test>
<test id="intl402/ch09/9.2/9.2.8_4.js"><reason></reason></test>
<test id="intl402/ch10/10.1/10.1.1_1.js"><reason></reason></test>
<test id="intl402/ch10/10.1/10.1.1_10.js"><reason></reason></test>
<test id="intl402/ch10/10.1/10.1.1_11.js"><reason></reason></test>
<test id="intl402/ch10/10.1/10.1.1_13.js"><reason></reason></test>
<test id="intl402/ch10/10.1/10.1.1_19_b.js"><reason></reason></test>
<test id="intl402/ch10/10.1/10.1.1_19_c.js"><reason></reason></test>
<test id="intl402/ch10/10.1/10.1.1_20.js"><reason></reason></test>
<test id="intl402/ch10/10.1/10.1.1_23.js"><reason></reason></test>
<test id="intl402/ch10/10.1/10.1.1_6.js"><reason></reason></test>
<test id="intl402/ch10/10.1/10.1.1_a.js"><reason></reason></test>
<test id="intl402/ch10/10.1/10.1.2.1_4.js"><reason></reason></test>
<test id="intl402/ch10/10.1/10.1.2_a.js"><reason></reason></test>
<test id="intl402/ch10/10.1/10.1.3.js"><reason></reason></test>
<test id="intl402/ch10/10.1/10.1_L15.js"><reason></reason></test>
<test id="intl402/ch10/10.2/10.2.1.js"><reason></reason></test>
<test id="intl402/ch10/10.2/10.2.2_L15.js"><reason></reason></test>
<test id="intl402/ch10/10.2/10.2.2_a.js"><reason></reason></test>
<test id="intl402/ch10/10.2/10.2.2_b.js"><reason></reason></test>
<test id="intl402/ch10/10.2/10.2.3_b.js"><reason></reason></test>
<test id="intl402/ch10/10.3/10.3.1.js"><reason></reason></test>
<test id="intl402/ch10/10.3/10.3.2_1_a_L15.js"><reason></reason></test>
<test id="intl402/ch10/10.3/10.3.2_1_c.js"><reason></reason></test>
<test id="intl402/ch10/10.3/10.3.2_CS_a.js"><reason></reason></test>
<test id="intl402/ch10/10.3/10.3.2_CS_b_NN.js"><reason></reason></test>
<test id="intl402/ch10/10.3/10.3.2_CS_c_NN.js"><reason></reason></test>
<test id="intl402/ch10/10.3/10.3.2_CS_d_NN.js"><reason></reason></test>
<test id="intl402/ch10/10.3/10.3.2_L15.js"><reason></reason></test>
<test id="intl402/ch10/10.3/10.3.3.js"><reason></reason></test>
<test id="intl402/ch10/10.3/10.3.3_L15.js"><reason></reason></test>
<test id="intl402/ch10/10.3/10.3_L15.js"><reason></reason></test>
<test id="intl402/ch10/10.3/10.3_a.js"><reason></reason></test>
<test id="intl402/ch10/10.3/10.3_b.js"><reason></reason></test>
<test id="intl402/ch10/10.4/10.4_a.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1.1_1.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1.1_15.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1.1_17.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1.1_19.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1.1_20_c.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1.1_21.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1.1_32.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1.1_34.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1.1_6.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1.1_7.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1.1_a.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1.2.1_4.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1.2.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1.3.js"><reason></reason></test>
<test id="intl402/ch11/11.1/11.1_L15.js"><reason></reason></test>
<test id="intl402/ch11/11.2/11.2.1.js"><reason></reason></test>
<test id="intl402/ch11/11.2/11.2.2_L15.js"><reason></reason></test>
<test id="intl402/ch11/11.2/11.2.2_a.js"><reason></reason></test>
<test id="intl402/ch11/11.2/11.2.2_b.js"><reason></reason></test>
<test id="intl402/ch11/11.2/11.2.3_b.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3.1.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3.2_1_a_L15.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3.2_1_a_ii.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3.2_1_c.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3.2_FN_1.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3.2_FN_2.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3.2_FN_3_b.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3.2_L15.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3.2_TRF.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3.2_TRP.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3.3.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3.3_L15.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3_L15.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3_a.js"><reason></reason></test>
<test id="intl402/ch11/11.3/11.3_b.js"><reason></reason></test>
<test id="intl402/ch11/11.4/11.4_a.js"><reason></reason></test>
<test id="intl402/ch12/12.1/12.1.1_1.js"><reason></reason></test>
<test id="intl402/ch12/12.1/12.1.1_18.js"><reason></reason></test>
<test id="intl402/ch12/12.1/12.1.1_22.js"><reason></reason></test>
<test id="intl402/ch12/12.1/12.1.1_23.js"><reason></reason></test>
<test id="intl402/ch12/12.1/12.1.1_25.js"><reason></reason></test>
<test id="intl402/ch12/12.1/12.1.1_5.js"><reason></reason></test>
<test id="intl402/ch12/12.1/12.1.1_6.js"><reason></reason></test>
<test id="intl402/ch12/12.1/12.1.1_TDTO.js"><reason></reason></test>
<test id="intl402/ch12/12.1/12.1.1_a.js"><reason></reason></test>
<test id="intl402/ch12/12.1/12.1.2.1_4.js"><reason></reason></test>
<test id="intl402/ch12/12.1/12.1.2.js"><reason></reason></test>
<test id="intl402/ch12/12.1/12.1.3.js"><reason></reason></test>
<test id="intl402/ch12/12.1/12.1_L15.js"><reason></reason></test>
<test id="intl402/ch12/12.2/12.2.1.js"><reason></reason></test>
<test id="intl402/ch12/12.2/12.2.2_L15.js"><reason></reason></test>
<test id="intl402/ch12/12.2/12.2.2_a.js"><reason></reason></test>
<test id="intl402/ch12/12.2/12.2.2_b.js"><reason></reason></test>
<test id="intl402/ch12/12.2/12.2.3_b.js"><reason></reason></test>
<test id="intl402/ch12/12.2/12.2.3_c.js"><reason></reason></test>
<test id="intl402/ch12/12.3/12.3.1.js"><reason></reason></test>
<test id="intl402/ch12/12.3/12.3.2_1_a_L15.js"><reason></reason></test>
<test id="intl402/ch12/12.3/12.3.2_1_c.js"><reason></reason></test>
<test id="intl402/ch12/12.3/12.3.2_FDT_1.js"><reason></reason></test>
<test id="intl402/ch12/12.3/12.3.2_FDT_7_a_iv.js"><reason></reason></test>
<test id="intl402/ch12/12.3/12.3.2_L15.js"><reason></reason></test>
<test id="intl402/ch12/12.3/12.3.2_TLT_2.js"><reason></reason></test>
<test id="intl402/ch12/12.3/12.3.3.js"><reason></reason></test>
<test id="intl402/ch12/12.3/12.3.3_L15.js"><reason></reason></test>
<test id="intl402/ch12/12.3/12.3_L15.js"><reason></reason></test>
<test id="intl402/ch12/12.3/12.3_a.js"><reason></reason></test>
<test id="intl402/ch12/12.3/12.3_b.js"><reason></reason></test>
<test id="intl402/ch12/12.4/12.4_a.js"><reason></reason></test>
<test id="intl402/ch13/13.1/13.1.1_6_1.js"><reason></reason></test>
<test id="intl402/ch13/13.1/13.1.1_6_2.js"><reason></reason></test>
<test id="intl402/ch13/13.1/13.1.1_7.js"><reason></reason></test>
<test id="intl402/ch13/13.2/13.2.1_4_1.js"><reason></reason></test>
<test id="intl402/ch13/13.2/13.2.1_4_2.js"><reason></reason></test>
<test id="intl402/ch13/13.2/13.2.1_5.js"><reason></reason></test>
<test id="intl402/ch13/13.3/13.3.0_6_1.js"><reason></reason></test>
<test id="intl402/ch13/13.3/13.3.0_6_2.js"><reason></reason></test>
<test id="intl402/ch13/13.3/13.3.0_7.js"><reason></reason></test>
</excludeList>
9 changes: 6 additions & 3 deletions tools/run-tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,8 +198,10 @@ def get_arguments():
help='Run jerry-debugger tests')
parser.add_argument('--jerry-tests', action='store_true',
help='Run jerry-tests')
parser.add_argument('--test262', action='store_true',
help='Run test262 - ES5.1')
parser.add_argument('--test262', default=False, const='default',
nargs='?', choices=['default', 'all', 'update'],
help='Run test262 - ES5.1. default: all tests except excludelist, ' +
'all: all tests, update: all tests and update excludelist')
parser.add_argument('--test262-es2015', default=False, const='default',
nargs='?', choices=['default', 'all', 'update'],
help='Run test262 - ES2015. default: all tests except excludelist, ' +
Expand Down Expand Up @@ -437,7 +439,8 @@ def run_test262_test_suite(options):
test_cmd.append('--esnext')
test_cmd.append(options.test262_esnext)
else:
test_cmd.append('--es51')
test_cmd.append('--es5.1')
test_cmd.append(options.test262)

if job.test_args:
test_cmd.extend(job.test_args)
Expand Down
34 changes: 12 additions & 22 deletions tools/runners/run-test-suite-test262.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import argparse
import os
import re
import shutil
import subprocess
import sys

Expand All @@ -40,8 +39,10 @@ def get_arguments():
parser.add_argument('--test-dir', metavar='DIR', required=True,
help='Directory contains test262 test suite')
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--es51', action='store_true',
help='Run test262 ES5.1 version')
group.add_argument('--es5.1', dest='es5_1', default=False, const='default',
nargs='?', choices=['default', 'all', 'update'],
help='Run test262 - ES5.1. default: all tests except excludelist, ' +
'all: all tests, update: all tests and update excludelist')
group.add_argument('--es2015', default=False, const='default',
nargs='?', choices=['default', 'all', 'update'],
help='Run test262 - ES2015. default: all tests except excludelist, ' +
Expand All @@ -57,20 +58,18 @@ def get_arguments():

if args.es2015:
args.test_dir = os.path.join(args.test_dir, 'es2015')
args.test262_harness_dir = os.path.abspath(os.path.dirname(__file__))
args.test262_git_hash = 'fd44cd73dfbce0b515a2474b7cd505d6176a9eb5'
args.excludelist_path = os.path.join('tests', 'test262-es6-excludelist.xml')
elif args.esnext:
args.test_dir = os.path.join(args.test_dir, 'esnext')
args.test262_harness_dir = os.path.abspath(os.path.dirname(__file__))
args.test262_git_hash = '281eb10b2844929a7c0ac04527f5b42ce56509fd'
args.excludelist_path = os.path.join('tests', 'test262-esnext-excludelist.xml')
else:
args.test_dir = os.path.join(args.test_dir, 'es51')
args.test262_harness_dir = args.test_dir
args.test_dir = os.path.join(args.test_dir, 'es5.1')
args.test262_git_hash = 'es5-tests'
args.excludelist_path = os.path.join('tests', 'test262-es5.1-excludelist.xml')

args.mode = args.es2015 or args.esnext
args.mode = args.es2015 or args.esnext or args.es5_1

return args

Expand All @@ -88,15 +87,6 @@ def prepare_test262_test_suite(args):
return_code = subprocess.call(['git', 'checkout', args.test262_git_hash], cwd=args.test_dir)
assert not return_code, 'Cloning test262 repository failed - invalid git revision.'

if args.es51:
path_to_remove = os.path.join(args.test_dir, 'test', 'suite', 'bestPractice')
if os.path.isdir(path_to_remove):
shutil.rmtree(path_to_remove)

path_to_remove = os.path.join(args.test_dir, 'test', 'suite', 'intl402')
if os.path.isdir(path_to_remove):
shutil.rmtree(path_to_remove)

# Since ES2018 iterator's next method is called once during the prologue of iteration,
# rather than during each step. The test is incorrect and stuck in an infinite loop.
# https://github.com/tc39/test262/pull/1248 fixed the test and it passes on test262-esnext.
Expand Down Expand Up @@ -187,16 +177,16 @@ def main(args):
if sys.version_info.major >= 3:
kwargs['errors'] = 'ignore'

if args.es51:
test262_harness_path = os.path.join(args.test262_harness_dir, 'tools/packaging/test262.py')
else:
test262_harness_path = os.path.join(args.test262_harness_dir, 'test262-harness.py')

test262_harness_dir = os.path.abspath(os.path.dirname(__file__))
test262_harness_path = os.path.join(test262_harness_dir, 'test262-harness.py')
test262_command = get_platform_cmd_prefix() + \
[test262_harness_path,
'--command', command,
'--tests', args.test_dir,
'--summary']
if args.es5_1:
test262_command.extend(['--es5.1'])
test262_command.extend(['--unmarked_default', 'non_strict'])

if 'excludelist_path' in args and args.mode == 'default':
test262_command.extend(['--exclude-list', args.excludelist_path])
Expand Down
Loading