diff --git a/tools/run-tests.py b/tools/run-tests.py index ce96fee4c9..43a46b4cd9 100755 --- a/tools/run-tests.py +++ b/tools/run-tests.py @@ -286,7 +286,7 @@ def create_binary(job, options): subprocess.check_output(build_cmd) ret = 0 except subprocess.CalledProcessError as err: - print(err.output) + print(err.output.decode(errors="ignore")) ret = err.returncode BINARY_CACHE[binary_key] = (ret, build_dir_path) @@ -495,6 +495,7 @@ def run_buildoption_test(options): Check = collections.namedtuple('Check', ['enabled', 'runner', 'arg']) def main(options): + util.setup_stdio() checks = [ Check(options.check_signed_off, run_check, [settings.SIGNED_OFF_SCRIPT] + {'tolerant': ['--tolerant'], 'gh-actions': ['--gh-actions']}.get(options.check_signed_off, [])), diff --git a/tools/runners/run-test-suite-test262.py b/tools/runners/run-test-suite-test262.py index 4da81d4a09..1526dc2f38 100755 --- a/tools/runners/run-test-suite-test262.py +++ b/tools/runners/run-test-suite-test262.py @@ -24,12 +24,6 @@ import util -def get_platform_cmd_prefix(): - if sys.platform == 'win32': - return ['cmd', '/S', '/C'] - return ['python2'] # The official test262.py isn't python3 compatible, but has python shebang. - - def get_arguments(): execution_runtime = os.environ.get('RUNTIME', '') parser = argparse.ArgumentParser() @@ -172,6 +166,7 @@ def update_exclude_list(args): def main(args): + util.setup_stdio() return_code = prepare_test262_test_suite(args) if return_code: return return_code @@ -192,7 +187,7 @@ def main(args): else: test262_harness_path = os.path.join(args.test262_harness_dir, 'test262-harness.py') - test262_command = get_platform_cmd_prefix() + \ + test262_command = util.get_python_cmd_prefix() + \ [test262_harness_path, '--command', command, '--tests', args.test_dir, diff --git a/tools/runners/run-test-suite.py b/tools/runners/run-test-suite.py index 04ea68a828..ed583b253a 100755 --- a/tools/runners/run-test-suite.py +++ b/tools/runners/run-test-suite.py @@ -79,6 +79,7 @@ def execute_test_command(test_cmd): kwargs = {} if sys.version_info.major >= 3: kwargs['encoding'] = 'unicode_escape' + kwargs['errors'] = 'ignore' process = subprocess.Popen(test_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, **kwargs) stdout = process.communicate()[0] @@ -86,6 +87,7 @@ def execute_test_command(test_cmd): def main(args): + util.setup_stdio() tests = get_tests(args.test_dir, args.test_list, args.skip_list) total = len(tests) if total == 0: diff --git a/tools/runners/run-unittests.py b/tools/runners/run-unittests.py index 6a38e332e1..aca43aaea0 100755 --- a/tools/runners/run-unittests.py +++ b/tools/runners/run-unittests.py @@ -50,6 +50,7 @@ def get_unittests(path): def main(args): + util.setup_stdio() unittests = get_unittests(args.path) total = len(unittests) if total == 0: diff --git a/tools/runners/test262-harness.py b/tools/runners/test262-harness.py index 68f5bd6a1b..3fb1acd99c 100755 --- a/tools/runners/test262-harness.py +++ b/tools/runners/test262-harness.py @@ -42,6 +42,7 @@ from __future__ import print_function +import codecs import logging import optparse import os @@ -58,6 +59,8 @@ import threading import multiprocessing +import util + ####################################################################### # based on _monkeyYaml.py ####################################################################### @@ -402,13 +405,13 @@ def open_file(self): text=self.text) def write(self, string): - os.write(self.file_desc, string) + os.write(self.file_desc, string.encode("utf8", "ignore")) def read(self): - file_desc = file(self.name) + file_desc = open(self.name, "rb") result = file_desc.read() file_desc.close() - return result + return result.decode("utf8", "ignore") def close(self): if not self.is_closed: @@ -460,12 +463,12 @@ def report_outcome(self, long_format): def write_output(self, target): out = self.stdout.strip() if out: - target.write("--- output --- \n %s" % out) + target.write(u"--- output --- \n %s" % out) error = self.stderr.strip() if error: - target.write("--- errors --- \n %s" % error) + target.write(u"--- errors --- \n %s" % error) - target.write("\n--- exit code: %d ---\n" % self.exit_code) + target.write(u"\n--- exit code: %d ---\n" % self.exit_code) def has_failed(self): return self.exit_code != 0 @@ -495,8 +498,8 @@ def __init__(self, suite, name, full_path, strict_mode, command_template, module self.name = name self.full_path = full_path self.strict_mode = strict_mode - with open(self.full_path) as file_desc: - self.contents = file_desc.read() + with open(self.full_path, "rb") as file_desc: + self.contents = file_desc.read().decode("utf8", "ignore") test_record = parse_test_record(self.contents, name) self.test = test_record["test"] del test_record["test"] @@ -645,7 +648,7 @@ def run_test_in(self, tmp): return TestResult(code, out, err, self) def run(self): - tmp = TempFile(suffix=".js", prefix="test262-", text=True) + tmp = TempFile(suffix=".js", prefix="test262-") try: result = self.run_test_in(tmp) finally: @@ -762,8 +765,8 @@ def get_include(self, name): if not name in self.include_cache: static = path.join(self.lib_root, name) if path.exists(static): - with open(static) as file_desc: - contents = file_desc.read() + with open(static, "rb") as file_desc: + contents = file_desc.read().decode("utf8", "ignore") contents = re.sub(r'\r\n', '\n', contents) self.include_cache[name] = contents + "\n" else: @@ -851,7 +854,7 @@ def run(self, command_template, tests, print_summary, full_summary, logname, job report_error("No tests to run") progress = ProgressIndicator(len(cases)) if logname: - self.logf = open(logname, "w") + self.logf = codecs.open(logname, "w", encoding="utf8", errors="ignore") if job_count == 1: for case in cases: @@ -917,6 +920,7 @@ def list_includes(self, tests): def main(): + util.setup_stdio() code = 0 parser = build_options() (options, args) = parser.parse_args() diff --git a/tools/runners/util.py b/tools/runners/util.py index 78dea6e993..553aa9aadf 100755 --- a/tools/runners/util.py +++ b/tools/runners/util.py @@ -15,6 +15,7 @@ # limitations under the License. from __future__ import print_function +import codecs import signal import subprocess import sys @@ -45,6 +46,17 @@ def set_sighdl_to_reset_timezone(timezone): signal.signal(signal.SIGINT, lambda signal, frame: set_timezone_and_exit(timezone)) +def setup_stdio(): + (out_stream, err_stream) = (sys.stdout, sys.stderr) + if sys.version_info.major >= 3: + (out_stream, err_stream) = (sys.stdout.buffer, sys.stderr.buffer) + # For tty using native encoding, otherwise (pipe) use 'utf-8' + encoding = sys.stdout.encoding if sys.stdout.isatty() else 'utf-8' + # Always override it to anvoid encode error + sys.stdout = codecs.getwriter(encoding)(out_stream, 'xmlcharrefreplace') + sys.stderr = codecs.getwriter(encoding)(err_stream, 'xmlcharrefreplace') + + def print_test_summary(summary_string, total, passed, failed): print("\n[summary] %s\n" % summary_string) print("TOTAL: %d" % total)