Revise tools scripts to be python3 compatible

tools/run-tests.py

@@ -286,7 +286,7 @@ def create_binary(job, options):
         subprocess.check_output(build_cmd)
         ret = 0
     except subprocess.CalledProcessError as err:
-        print(err.output)
+        print(err.output.decode(errors="ignore"))
         ret = err.returncode
 
     BINARY_CACHE[binary_key] = (ret, build_dir_path)
@@ -495,6 +495,7 @@ def run_buildoption_test(options):
 Check = collections.namedtuple('Check', ['enabled', 'runner', 'arg'])
 
 def main(options):
+    util.setup_stdio()
     checks = [
         Check(options.check_signed_off, run_check, [settings.SIGNED_OFF_SCRIPT]
               + {'tolerant': ['--tolerant'], 'gh-actions': ['--gh-actions']}.get(options.check_signed_off, [])),

tools/runners/run-test-suite-test262.py

@@ -24,12 +24,6 @@
 
 import util
 
-def get_platform_cmd_prefix():
-    if sys.platform == 'win32':
-        return ['cmd', '/S', '/C']
-    return ['python2']  # The official test262.py isn't python3 compatible, but has python shebang.
-
-
 def get_arguments():
     execution_runtime = os.environ.get('RUNTIME', '')
     parser = argparse.ArgumentParser()
@@ -172,6 +166,7 @@ def update_exclude_list(args):
 
 
 def main(args):
+    util.setup_stdio()
     return_code = prepare_test262_test_suite(args)
     if return_code:
         return return_code
@@ -192,7 +187,7 @@ def main(args):
     else:
         test262_harness_path = os.path.join(args.test262_harness_dir, 'test262-harness.py')
 
-    test262_command = get_platform_cmd_prefix() + \
+    test262_command = util.get_python_cmd_prefix() + \
                       [test262_harness_path,
                        '--command', command,
                        '--tests', args.test_dir,

tools/runners/run-test-suite.py

@@ -79,13 +79,15 @@ def execute_test_command(test_cmd):
     kwargs = {}
     if sys.version_info.major >= 3:
         kwargs['encoding'] = 'unicode_escape'
+        kwargs['errors'] = 'ignore'
     process = subprocess.Popen(test_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                                universal_newlines=True, **kwargs)
     stdout = process.communicate()[0]
     return (process.returncode, stdout)
 
 
 def main(args):
+    util.setup_stdio()
     tests = get_tests(args.test_dir, args.test_list, args.skip_list)
     total = len(tests)
     if total == 0:

tools/runners/run-unittests.py

@@ -50,6 +50,7 @@ def get_unittests(path):
 
 
 def main(args):
+    util.setup_stdio()
     unittests = get_unittests(args.path)
     total = len(unittests)
     if total == 0:

tools/runners/test262-harness.py

@@ -42,6 +42,7 @@
 
 from __future__ import print_function
 
+import codecs
 import logging
 import optparse
 import os
@@ -58,6 +59,8 @@
 import threading
 import multiprocessing
 
+import util
+
 #######################################################################
 # based on _monkeyYaml.py
 #######################################################################
@@ -402,13 +405,13 @@ def open_file(self):
             text=self.text)
 
     def write(self, string):
-        os.write(self.file_desc, string)
+        os.write(self.file_desc, string.encode("utf8", "ignore"))
 
     def read(self):
-        file_desc = file(self.name)
+        file_desc = open(self.name, "rb")
         result = file_desc.read()
         file_desc.close()
-        return result
+        return result.decode("utf8", "ignore")
 
     def close(self):
         if not self.is_closed:
@@ -460,12 +463,12 @@ def report_outcome(self, long_format):
     def write_output(self, target):
         out = self.stdout.strip()
         if out:
-            target.write("--- output --- \n %s" % out)
+            target.write(u"--- output --- \n %s" % out)
         error = self.stderr.strip()
         if error:
-            target.write("--- errors ---  \n %s" % error)
+            target.write(u"--- errors ---  \n %s" % error)
 
-        target.write("\n--- exit code: %d ---\n" % self.exit_code)
+        target.write(u"\n--- exit code: %d ---\n" % self.exit_code)
 
     def has_failed(self):
         return self.exit_code != 0
@@ -495,8 +498,8 @@ def __init__(self, suite, name, full_path, strict_mode, command_template, module
         self.name = name
         self.full_path = full_path
         self.strict_mode = strict_mode
-        with open(self.full_path) as file_desc:
-            self.contents = file_desc.read()
+        with open(self.full_path, "rb") as file_desc:
+            self.contents = file_desc.read().decode("utf8", "ignore")
         test_record = parse_test_record(self.contents, name)
         self.test = test_record["test"]
         del test_record["test"]
@@ -645,7 +648,7 @@ def run_test_in(self, tmp):
         return TestResult(code, out, err, self)
 
     def run(self):
-        tmp = TempFile(suffix=".js", prefix="test262-", text=True)
+        tmp = TempFile(suffix=".js", prefix="test262-")
         try:
             result = self.run_test_in(tmp)
         finally:
@@ -762,8 +765,8 @@ def get_include(self, name):
         if not name in self.include_cache:
             static = path.join(self.lib_root, name)
             if path.exists(static):
-                with open(static) as file_desc:
-                    contents = file_desc.read()
+                with open(static, "rb") as file_desc:
+                    contents = file_desc.read().decode("utf8", "ignore")
                     contents = re.sub(r'\r\n', '\n', contents)
                     self.include_cache[name] = contents + "\n"
             else:
@@ -851,7 +854,7 @@ def run(self, command_template, tests, print_summary, full_summary, logname, job
             report_error("No tests to run")
         progress = ProgressIndicator(len(cases))
         if logname:
-            self.logf = open(logname, "w")
+            self.logf = codecs.open(logname, "w", encoding="utf8", errors="ignore")
 
         if job_count == 1:
             for case in cases:
@@ -917,6 +920,7 @@ def list_includes(self, tests):
 
 
 def main():
+    util.setup_stdio()
     code = 0
     parser = build_options()
     (options, args) = parser.parse_args()

tools/runners/util.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 from __future__ import print_function
+import codecs
 import signal
 import subprocess
 import sys
@@ -45,6 +46,17 @@ def set_sighdl_to_reset_timezone(timezone):
     signal.signal(signal.SIGINT, lambda signal, frame: set_timezone_and_exit(timezone))
 
 
+def setup_stdio():
+    (out_stream, err_stream) = (sys.stdout, sys.stderr)
+    if sys.version_info.major >= 3:
+        (out_stream, err_stream) = (sys.stdout.buffer, sys.stderr.buffer)
+    # For tty using native encoding, otherwise (pipe) use 'utf-8'
+    encoding = sys.stdout.encoding if sys.stdout.isatty() else 'utf-8'
+    # Always override it to anvoid encode error
+    sys.stdout = codecs.getwriter(encoding)(out_stream, 'xmlcharrefreplace')
+    sys.stderr = codecs.getwriter(encoding)(err_stream, 'xmlcharrefreplace')
+
+
 def print_test_summary(summary_string, total, passed, failed):
     print("\n[summary] %s\n" % summary_string)
     print("TOTAL: %d" % total)