diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..62bca231 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,37 @@ +name: test-run + +on: [push, pull_request] + +jobs: + build: + if: ( github.event_name == 'push' || + github.event.pull_request.head.repo.full_name != github.repository ) && + ( github.repository == 'tarantool/test-run' ) + + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.8-dev] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: display python version + run: python -c "import sys; print(sys.version)" + - name: setup dependencies + run: | + sudo apt update -y + sudo apt-get -y install lua5.1 luarocks + sudo luarocks install luacheck + - name: setup python dependencies + run: | + pip install -r requirements.txt + pip install -r requirements-test.txt + - name: run static analysis + run: | + make lint diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d5c4344d..00000000 --- a/.travis.yml +++ /dev/null @@ -1,14 +0,0 @@ -language: python -python: 2.7 - -before_install: - - sudo apt update -y - - sudo apt-get -y install lua5.1 luarocks - - sudo luarocks install luacheck - -install: - - pip install -r requirements.txt - - pip install -r requirements-test.txt - -script: - - make lint diff --git a/Makefile b/Makefile index a2143d37..9e8b6f5c 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ default: lint: flake8 luacheck flake8: - python2 -m flake8 *.py lib/*.py + python -m flake8 *.py lib/*.py luacheck: luacheck --config .luacheckrc . diff --git a/dispatcher.py b/dispatcher.py index 9f9c0443..f878e9a4 100644 --- a/dispatcher.py +++ b/dispatcher.py @@ -7,13 +7,42 @@ import yaml import multiprocessing -from multiprocessing.queues import SimpleQueue -import listeners -import lib +# SimpleQueue is available from multiprocessing.queues on +# all Python versions known at the moment of writting the code +# (up to 3.9). +# +# It was additionally exposed directly from the multiprocessing +# module since Python 3.3 ([1]). +# +# However the mandatory argument 'ctx' +# (see multiprocessing.get_context()) was added to the constructor +# of SimpleQueue from multiprocessing.queues since Python 3.4 +# ([2]). +# +# So we should import SimpleQueue from multiprocessing on +# Python 3.3+ (and must to do so on Python 3.4+) to uniformly +# instantiate it (without constructor arguments). +# +# [1]: https://bugs.python.org/issue11836 +# [2]: https://bugs.python.org/issue18999 +try: + # Python 3.3+ + from multiprocessing import SimpleQueue +except ImportError: + # Python 2 + from multiprocessing.queues import SimpleQueue + +from lib import Options from lib.utils import set_fd_cloexec from lib.worker import WorkerTaskResult, WorkerDone from lib.colorer import color_stdout +from listeners import ArtifactsWatcher +from listeners import FailWatcher +from listeners import HangWatcher +from listeners import LogOutputWatcher +from listeners import OutputWatcher +from listeners import StatisticsWatcher class TcpPortDispatcher: @@ -122,30 +151,27 @@ def kill_all_workers(self): pass def init_listeners(self): - args = lib.Options().args + args = Options().args watch_hang = args.no_output_timeout >= 0 and \ not args.gdb and \ not args.gdbserver and \ not args.lldb and \ not args.valgrind - watch_fail = not lib.Options().args.is_force - - log_output_watcher = listeners.LogOutputWatcher() - self.statistics = listeners.StatisticsWatcher( - log_output_watcher.get_logfile) - self.artifacts = listeners.ArtifactsWatcher( - log_output_watcher.get_logfile) - output_watcher = listeners.OutputWatcher() + watch_fail = not Options().args.is_force + + log_output_watcher = LogOutputWatcher() + self.statistics = StatisticsWatcher(log_output_watcher.get_logfile) + self.artifacts = ArtifactsWatcher(log_output_watcher.get_logfile) + output_watcher = OutputWatcher() self.listeners = [self.statistics, log_output_watcher, output_watcher, self.artifacts] if watch_fail: - self.fail_watcher = listeners.FailWatcher( - self.terminate_all_workers) + self.fail_watcher = FailWatcher(self.terminate_all_workers) self.listeners.append(self.fail_watcher) if watch_hang: warn_timeout = 60.0 if args.long else 10.0 - hang_watcher = listeners.HangWatcher( - output_watcher.not_done_worker_ids, self.kill_all_workers, - warn_timeout, float(args.no_output_timeout)) + hang_watcher = HangWatcher(output_watcher.not_done_worker_ids, + self.kill_all_workers, warn_timeout, + float(args.no_output_timeout)) self.listeners.append(hang_watcher) def run_max_workers(self): @@ -315,8 +341,8 @@ def flush_ready(self, inputs): # leave only output listeners in self.listeners new_listeners = [] for listener in self.listeners: - if isinstance(listener, (listeners.LogOutputWatcher, - listeners.OutputWatcher)): + if isinstance(listener, (LogOutputWatcher, + OutputWatcher)): listener.report_at_timeout = False new_listeners.append(listener) self.listeners = new_listeners diff --git a/lib/__init__.py b/lib/__init__.py index c03a6d86..0dffc40f 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -5,7 +5,7 @@ from lib.options import Options from lib.tarantool_server import TarantoolServer from lib.unittest_server import UnittestServer -from utils import warn_unix_sockets_at_start +from lib.utils import warn_unix_sockets_at_start __all__ = ['Options'] diff --git a/lib/admin_connection.py b/lib/admin_connection.py index 1af31275..49ac1079 100644 --- a/lib/admin_connection.py +++ b/lib/admin_connection.py @@ -24,10 +24,12 @@ import re import sys -from tarantool_connection import TarantoolConnection -from tarantool_connection import TarantoolPool -from tarantool_connection import TarantoolAsyncConnection +from lib.tarantool_connection import TarantoolConnection +from lib.tarantool_connection import TarantoolPool +from lib.tarantool_connection import TarantoolAsyncConnection +from lib.utils import bytes_to_str +from lib.utils import str_to_bytes ADMIN_SEPARATOR = '\n' @@ -36,13 +38,13 @@ def get_handshake(sock, length=128, max_try=100): """ Correct way to get tarantool handshake """ - result = "" + result = b"" i = 0 while len(result) != length and i < max_try: - result = "%s%s" % (result, sock.recv(length-len(result))) + result = b"%s%s" % (result, sock.recv(length-len(result))) # max_try counter for tarantool/gh-1362 i += 1 - return result + return bytes_to_str(result) class AdminPool(TarantoolPool): @@ -54,19 +56,19 @@ def _new_connection(self): # tarantool/gh-1163 # 1. raise only if handshake is not full # 2. be silent on crashes or if it's server.stop() operation - print 'Handshake error {\n', handshake, '\n}' + print('Handshake error {\n', handshake, '\n}') raise RuntimeError('Broken tarantool console handshake') return s class ExecMixIn(object): def cmd(self, socket, cmd, silent): - socket.sendall(cmd) + socket.sendall(str_to_bytes(cmd)) bufsiz = 4096 res = "" while True: - buf = socket.recv(bufsiz) + buf = bytes_to_str(socket.recv(bufsiz)) if not buf: break res = res + buf diff --git a/lib/app_server.py b/lib/app_server.py index 00a82b8f..d4675c8a 100644 --- a/lib/app_server.py +++ b/lib/app_server.py @@ -21,8 +21,8 @@ from lib.utils import format_process from lib.utils import signame from lib.utils import warn_unix_socket -from test import TestRunGreenlet, TestExecutionError from threading import Timer +from lib.test import TestRunGreenlet, TestExecutionError def timeout_handler(server_process, test_timeout): @@ -38,8 +38,8 @@ def run_server(execs, cwd, server, logfile, retval): timer.start() stdout, stderr = server.process.communicate() timer.cancel() - sys.stdout.write(stdout) - with open(logfile, 'a') as f: + sys.stdout.write_bytes(stdout) + with open(logfile, 'ab') as f: f.write(stderr) retval['returncode'] = server.process.wait() server.process = None @@ -252,7 +252,7 @@ def is_correct(run): test_suite.ini, params=params, conf_name=conf_name - ) for conf_name, params in runs.iteritems() + ) for conf_name, params in runs.items() if is_correct(conf_name)]) else: tests.append(AppTest(test_name, diff --git a/lib/box_connection.py b/lib/box_connection.py index 6eb5121c..6a3617a7 100644 --- a/lib/box_connection.py +++ b/lib/box_connection.py @@ -25,7 +25,7 @@ import ctypes import socket -from tarantool_connection import TarantoolConnection +from lib.tarantool_connection import TarantoolConnection # monkey patch tarantool and msgpack from lib.utils import check_libs @@ -75,11 +75,11 @@ def execute_no_reconnect(self, command, silent=True): if not command: return if not silent: - print command + print(command) cmd = command.replace(SEPARATOR, ' ') + SEPARATOR response = self.py_con.call(cmd) if not silent: - print response + print(response) return response def execute(self, command, silent=True): @@ -88,8 +88,8 @@ def execute(self, command, silent=True): def call(self, command, *args): if not command: return - print 'call ', command, args + print('call {} {}'.format(command, args)) response = self.py_con.call(command, *args) result = str(response) - print result + print(result) return result diff --git a/lib/connpool.py b/lib/connpool.py index b54dfadf..b80ba584 100644 --- a/lib/connpool.py +++ b/lib/connpool.py @@ -10,7 +10,7 @@ from contextlib import contextmanager from functools import wraps -from test import TestRunGreenlet +from lib.test import TestRunGreenlet __all__ = ["ConnectionPool", "retry"] @@ -35,9 +35,9 @@ def __init__(self, size, exc_classes=DEFAULT_EXC_CLASSES, keepalive=None): self.keepalive = keepalive # Exceptions list must be in tuple form to be caught properly self.exc_classes = tuple(exc_classes) - for i in xrange(size): + for i in range(size): self.lock.acquire() - for i in xrange(size): + for i in range(size): greenlet = TestRunGreenlet(self._addOne) greenlet.start_later(self.SPAWN_FREQUENCY * i) if self.keepalive: @@ -134,7 +134,7 @@ def deco(*args, **kwargs): except exc_classes as e: if logger is not None: logger.log(retry_log_level, - retry_log_message.format(f=f.func_name, e=e)) + retry_log_message.format(f=f.__name__, e=e)) gevent.sleep(interval) failures += 1 if max_failures is not None \ @@ -142,6 +142,6 @@ def deco(*args, **kwargs): if logger is not None: logger.log(max_failure_log_level, max_failure_log_message.format( - f=f.func_name, e=e)) + f=f.__name__, e=e)) raise return deco diff --git a/lib/inspector.py b/lib/inspector.py index 1d68deaa..9733cc1f 100644 --- a/lib/inspector.py +++ b/lib/inspector.py @@ -6,8 +6,10 @@ from gevent.lock import Semaphore from gevent.server import StreamServer +from lib.utils import bytes_to_str from lib.utils import find_port from lib.utils import prefix_each_line +from lib.utils import str_to_bytes from lib.colorer import color_stdout from lib.colorer import color_log from lib.colorer import qa_notice @@ -77,7 +79,7 @@ def readline(socket, delimiter='\n', size=4096): while data: try: - data = socket.recv(size) + data = bytes_to_str(socket.recv(size)) except IOError: # catch instance halt connection refused errors data = '' @@ -119,7 +121,7 @@ def handle(self, socket, addr): color_log("DEBUG: test-run's response for [{}]\n{}\n".format( line, prefix_each_line(' | ', result)), schema='test-run command') - socket.sendall(result) + socket.sendall(str_to_bytes(result)) self.sem.release() diff --git a/lib/preprocessor.py b/lib/preprocessor.py index 29d6c517..a2c1232b 100644 --- a/lib/preprocessor.py +++ b/lib/preprocessor.py @@ -6,12 +6,13 @@ import yaml from gevent import socket -import six from lib.admin_connection import AdminAsyncConnection from lib.colorer import color_log from lib.utils import signum from lib.utils import signame +from lib.utils import integer_types +from lib.utils import string_types class Namespace(object): @@ -216,7 +217,7 @@ def server_start(self, ctype, sname, opts): self.connections[sname] = self.servers[sname].admin try: self.connections[sname]('return true', silent=True) - except socket.error as e: + except socket.error: LuaPreprocessorException( 'Can\'t start server {0}'.format(repr(sname))) return not crash_occured @@ -412,9 +413,9 @@ def variable(self, ctype, ref, ret): if ctype == 'set': self.curcon[0].reconnect() result = eval(ret[1:-1], {}, self.environ.__dict__) - if isinstance(result, six.integer_types): + if isinstance(result, integer_types): cmd = '{0}={1}'.format(ref, result) - elif isinstance(result, six.string_types): + elif isinstance(result, string_types): cmd = '{0}="{1}"'.format(ref, result) else: raise LuaPreprocessorException( @@ -435,7 +436,7 @@ def stop_nondefault(self, signal=signal.SIGTERM): names), schema='info') if sys.stdout.__class__.__name__ == 'FilteredStream': sys.stdout.clear_all_filters() - for k, v in self.servers.iteritems(): + for k, v in self.servers.items(): # don't stop the default server if k == 'default': continue @@ -448,7 +449,7 @@ def cleanup_nondefault(self): names = [k for k in self.servers.keys() if k != 'default'] color_log('DEBUG: Cleanup non-default servers: {}\n'.format(names), schema='info') - for k, v in self.servers.iteritems(): + for k, v in self.servers.items(): # don't cleanup the default server if k == 'default': continue diff --git a/lib/pytap13.py b/lib/pytap13.py index 295a0d15..5aef4313 100644 --- a/lib/pytap13.py +++ b/lib/pytap13.py @@ -17,12 +17,16 @@ # Author: Josef Skladanka import re + try: - from CStringIO import StringIO -except ImportError: + # Python 2 from StringIO import StringIO +except ImportError: + # Python 3 + from io import StringIO import yaml +from lib.utils import string_types RE_VERSION = re.compile(r"^\s*TAP version 13\s*$") @@ -173,7 +177,7 @@ def _parse(self, source): self.tests.append(Test('not ok', len(self.tests), comment=comment)) def parse(self, source): - if isinstance(source, (str, unicode)): + if isinstance(source, string_types): self._parse(StringIO(source)) elif hasattr(source, "__iter__"): self._parse(source) diff --git a/lib/server.py b/lib/server.py index c699997a..d13aa024 100644 --- a/lib/server.py +++ b/lib/server.py @@ -2,6 +2,7 @@ import os import shutil from itertools import product + from lib.server_mixins import ValgrindMixin from lib.server_mixins import GdbMixin from lib.server_mixins import GdbServerMixin diff --git a/lib/server_mixins.py b/lib/server_mixins.py index 44da01fe..b2ba8a69 100644 --- a/lib/server_mixins.py +++ b/lib/server_mixins.py @@ -1,11 +1,13 @@ import os import glob import shlex +from six.moves import shlex_quote + from lib.utils import find_in_path from lib.utils import print_tail_n from lib.utils import non_empty_valgrind_logs -from lib.colorer import color_stdout, color_log -from six.moves import shlex_quote +from lib.colorer import color_log +from lib.colorer import color_stdout def shlex_join(strings): diff --git a/lib/tarantool_connection.py b/lib/tarantool_connection.py index 019ed990..b246d4f4 100644 --- a/lib/tarantool_connection.py +++ b/lib/tarantool_connection.py @@ -30,10 +30,10 @@ import gevent from gevent import socket as gsocket -from connpool import ConnectionPool -from test import TestRunGreenlet -from utils import warn_unix_socket -from utils import set_fd_cloexec +from lib.connpool import ConnectionPool +from lib.test import TestRunGreenlet +from lib.utils import warn_unix_socket +from lib.utils import set_fd_cloexec class TarantoolPool(ConnectionPool): @@ -146,7 +146,7 @@ def opt_reconnect(self): """ try: if not self.is_connected or self.socket.recv( - 1, socket.MSG_DONTWAIT | socket.MSG_PEEK) == '': + 1, socket.MSG_DONTWAIT | socket.MSG_PEEK) == b'': self.reconnect() except socket.error as e: if e.errno == errno.EAGAIN: diff --git a/lib/tarantool_server.py b/lib/tarantool_server.py index 4dc640a6..f611dbaf 100644 --- a/lib/tarantool_server.py +++ b/lib/tarantool_server.py @@ -19,9 +19,11 @@ from threading import Timer try: - from cStringIO import StringIO -except ImportError: + # Python 2 from StringIO import StringIO +except ImportError: + # Python 3 + from io import StringIO from lib.admin_connection import AdminConnection, AdminAsyncConnection from lib.box_connection import BoxConnection @@ -33,6 +35,7 @@ from lib.server import Server from lib.server import DEFAULT_SNAPSHOT_NAME from lib.test import Test +from lib.utils import bytes_to_str from lib.utils import find_port from lib.utils import extract_schema_from_snapshot from lib.utils import format_process @@ -40,7 +43,7 @@ from lib.utils import signame from lib.utils import warn_unix_socket from lib.utils import prefix_each_line -from test import TestRunGreenlet, TestExecutionError +from lib.test import TestRunGreenlet, TestExecutionError def save_join(green_obj, timeout=None): @@ -396,8 +399,10 @@ class PythonTest(Test): def execute(self, server): super(PythonTest, self).execute(server) - execfile(self.name, dict(locals(), test_run_current_test=self, - **server.__dict__)) + new_globals = dict(locals(), test_run_current_test=self, **server.__dict__) + with open(self.name) as f: + code = compile(f.read(), self.name, 'exec') + exec(code, new_globals) # crash was detected (possibly on non-default server) if server.current_test.is_crash_reported: raise TestExecutionError @@ -573,7 +578,7 @@ def _iproto(self, port): @property def log_des(self): if not hasattr(self, '_log_des'): - self._log_des = open(self.logfile, 'a') + self._log_des = open(self.logfile, 'ab') return self._log_des @log_des.deleter @@ -659,7 +664,7 @@ def __del__(self): @classmethod def version(cls): p = subprocess.Popen([cls.binary, "--version"], stdout=subprocess.PIPE) - version = p.stdout.read().rstrip() + version = bytes_to_str(p.stdout.read()).rstrip() p.wait() return version @@ -759,7 +764,7 @@ def deploy(self, silent=True, **kwargs): def copy_files(self): if self.script: shutil.copy(self.script, self.script_dst) - os.chmod(self.script_dst, 0777) + os.chmod(self.script_dst, 0o777) if self.lua_libs: for i in self.lua_libs: source = os.path.join(self.testdir, i) @@ -1153,15 +1158,15 @@ def read_pidfile(self): def test_option_get(self, option_list_str, silent=False): args = [self.binary] + shlex.split(option_list_str) if not silent: - print " ".join([os.path.basename(self.binary)] + args[1:]) + print(" ".join([os.path.basename(self.binary)] + args[1:])) output = subprocess.Popen(args, cwd=self.vardir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.read() - return output + return bytes_to_str(output) def test_option(self, option_list_str): - print self.test_option_get(option_list_str) + print(self.test_option_get(option_list_str)) @staticmethod def find_tests(test_suite, suite_path): diff --git a/lib/test.py b/lib/test.py index ad22243d..0f002802 100644 --- a/lib/test.py +++ b/lib/test.py @@ -2,25 +2,22 @@ import gevent import os import pprint -import pytap13 import re import shutil import sys import traceback from functools import partial from hashlib import md5 -from utils import safe_makedirs -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - -import lib +from lib import Options from lib.colorer import color_stdout +from lib.utils import assert_bytes from lib.utils import non_empty_valgrind_logs from lib.utils import print_tail_n from lib.utils import print_unidiff as utils_print_unidiff +from lib.utils import safe_makedirs +from lib.utils import str_to_bytes +from lib import pytap13 class TestExecutionError(OSError): @@ -46,21 +43,17 @@ def __repr__(self): class FilteredStream: """Helper class to filter .result file output""" def __init__(self, filename): - # - # always open the output stream in line-buffered mode, - # to see partial results of a failed test - # - self.stream = open(filename, "w+", 1) + self.stream = open(filename, "wb+") self.filters = [] self.inspector = None - def write(self, fragment): - """Apply all filters, then write result to the undelrying stream. - Do line-oriented filtering: the fragment doesn't have to represent - just one line.""" - fragment_stream = StringIO(fragment) + def write_bytes(self, fragment): + """ The same as ``write()``, but accepts ```` as + input. + """ + assert_bytes(fragment) skipped = False - for line in fragment_stream: + for line in fragment.splitlines(True): original_len = len(line.strip()) for pattern, replacement in self.filters: line = re.sub(pattern, replacement, line) @@ -71,8 +64,20 @@ def write(self, fragment): if not skipped: self.stream.write(line) + def write(self, fragment): + """ Apply all filters, then write result to the underlying + stream. + + Do line-oriented filtering: the fragment doesn't have + to represent just one line. + + Accepts ```` as input, just like the standard + ``sys.stdout.write()``. + """ + self.write_bytes(str_to_bytes(fragment)) + def push_filter(self, pattern, replacement): - self.filters.append([pattern, replacement]) + self.filters.append([str_to_bytes(pattern), str_to_bytes(replacement)]) def pop_filter(self): self.filters.pop() @@ -173,7 +178,10 @@ def run(self, server): if os.path.exists(self.skip_cond): sys.stdout = FilteredStream(self.tmp_result) stdout_fileno = sys.stdout.stream.fileno() - execfile(self.skip_cond, dict(locals(), **server.__dict__)) + new_globals = dict(locals(), **server.__dict__) + with open(self.skip_cond, 'r') as f: + code = compile(f.read(), self.skip_cond, 'exec') + exec(code, new_globals) sys.stdout.close() sys.stdout = save_stdout if not self.skip: @@ -208,7 +216,7 @@ def run(self, server): self.is_equal_result = filecmp.cmp(self.result, self.tmp_result) elif self.is_executed_ok: - if lib.Options().args.is_verbose: + if Options().args.is_verbose: color_stdout('\n') with open(self.tmp_result, 'r') as f: color_stdout(f.read(), schema='log') @@ -241,7 +249,7 @@ def run(self, server): not self.is_equal_result and not os.path.isfile(self.result) and not is_tap and - lib.Options().args.update_result): + Options().args.update_result): shutil.copy(self.tmp_result, self.result) short_status = 'new' color_stdout("[ new ]\n", schema='test_new') @@ -249,7 +257,7 @@ def run(self, server): not self.is_equal_result and os.path.isfile(self.result) and not is_tap and - lib.Options().args.update_result): + Options().args.update_result): shutil.copy(self.tmp_result, self.result) short_status = 'updated' color_stdout("[ updated ]\n", schema='test_new') @@ -343,7 +351,7 @@ def check_tap_output(self): schema='error') color_stdout('\nNo result file (%s) found.\n' % self.result, schema='error') - if not lib.Options().args.update_result: + if not Options().args.update_result: msg = 'Run the test with --update-result option to write the new result file.\n' color_stdout(msg, schema='error') self.is_crash_reported = True diff --git a/lib/test_suite.py b/lib/test_suite.py index 91f8088b..67cdadff 100644 --- a/lib/test_suite.py +++ b/lib/test_suite.py @@ -1,9 +1,16 @@ -import ConfigParser +try: + # Python 2 + import ConfigParser as configparser +except ImportError: + # Python 3 + import configparser + import json import os import re +import sys -import lib +from lib import Options from lib.app_server import AppServer from lib.colorer import color_stdout from lib.inspector import TarantoolInspector @@ -90,7 +97,11 @@ def __init__(self, suite_path, args): raise RuntimeError("Suite %s doesn't exist" % repr(suite_path)) # read the suite config - config = ConfigParser.ConfigParser() + parser_kwargs = dict() + if sys.version_info[0] == 3: + parser_kwargs['inline_comment_prefixes'] = (';',) + parser_kwargs['strict'] = True + config = configparser.ConfigParser(**parser_kwargs) config.read(os.path.join(suite_path, "suite.ini")) self.ini.update(dict(config.items("default"))) self.ini.update(self.args.__dict__) @@ -153,7 +164,7 @@ def collect_tests(self): else: raise ValueError('Cannot collect tests of unknown type') - if not lib.Options().args.reproduce: + if not Options().args.reproduce: color_stdout("Collecting tests in ", schema='ts_text') color_stdout( '%s (Found %s tests)' % ( @@ -196,7 +207,7 @@ def gen_server(self): try: return Server(self.ini, test_suite=self) except Exception as e: - print e + print(e) raise RuntimeError("Unknown server: core = {0}".format( self.ini["core"])) @@ -265,7 +276,7 @@ def run_test(self, test, server, inspector): result_checksum = None # cleanup only if test passed or if --force mode enabled - if lib.Options().args.is_force or short_status == 'pass': + if Options().args.is_force or short_status == 'pass': inspector.cleanup_nondefault() return short_status, result_checksum diff --git a/lib/unittest_server.py b/lib/unittest_server.py index a7cc20e5..4df8f0b0 100644 --- a/lib/unittest_server.py +++ b/lib/unittest_server.py @@ -16,7 +16,7 @@ def execute(self, server): server.current_test = self execs = server.prepare_args() proc = Popen(execs, cwd=server.vardir, stdout=PIPE, stderr=STDOUT) - sys.stdout.write(proc.communicate()[0]) + sys.stdout.write_bytes(proc.communicate()[0]) class UnittestServer(Server): @@ -48,7 +48,7 @@ def binary(self): def prepare_args(self, args=[]): executable_path = os.path.join(self.builddir, "test", self.current_test.name) - return [executable_path] + args + return [os.path.abspath(executable_path)] + args def deploy(self, vardir=None, silent=True, wait=True): self.vardir = vardir diff --git a/lib/utils.py b/lib/utils.py index 2fb12ed9..a92db07c 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -1,6 +1,5 @@ import os import sys -import six import collections import signal import random @@ -21,6 +20,17 @@ UNIX_SOCKET_LEN_LIMIT = 107 +# Useful for very coarse version differentiation. +PY3 = sys.version_info[0] == 3 +PY2 = sys.version_info[0] == 2 + +if PY3: + string_types = str, + integer_types = int, +else: + string_types = basestring, # noqa: F821 + integer_types = (int, long) # noqa: F821 + def check_libs(): deps = [ @@ -68,7 +78,7 @@ def check_port(port, rais=True, ipv4=True, ipv6=True): connections (UNIX Sockets in case of file path). False -- otherwise. """ try: - if isinstance(port, (int, long)): + if isinstance(port, integer_types): if ipv4: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('127.0.0.1', port)) @@ -139,22 +149,22 @@ def find_in_path(name): def signame(signal): - if isinstance(signal, six.integer_types): + if isinstance(signal, integer_types): return SIGNAMES[signal] if Signals and isinstance(signal, Signals): return SIGNAMES[int(signal)] - if isinstance(signal, six.string_types): + if isinstance(signal, string_types): return signal raise TypeError('signame(): signal argument of unexpected type: {}'.format( str(type(signal)))) def signum(signal): - if isinstance(signal, six.integer_types): + if isinstance(signal, integer_types): return signal if Signals and isinstance(signal, Signals): return int(signal) - if isinstance(signal, six.string_types): + if isinstance(signal, string_types): if not signal.startswith('SIG'): signal = 'SIG' + signal return SIGNUMS[signal] @@ -303,3 +313,41 @@ def extract_schema_from_snapshot(snapshot_path): if res[0] == 'version': return res return None + + +def assert_bytes(b): + """ Ensure given value is . + """ + if type(b) != bytes: + raise ValueError('Internal error: expected {}, got {}: {}'.format( + str(bytes), str(type(b)), repr(b))) + + +def assert_str(s): + """ Ensure given value is . + """ + if type(s) != str: + raise ValueError('Internal error: expected {}, got {}: {}'.format( + str(str), str(type(s)), repr(s))) + + +def bytes_to_str(b): + """ Convert to . + + No-op on Python 2. + """ + assert_bytes(b) + if PY2: + return b + return b.decode('utf-8') + + +def str_to_bytes(s): + """ Convert to . + + No-op on Python 2. + """ + assert_str(s) + if PY2: + return s + return s.encode('utf-8') diff --git a/lib/worker.py b/lib/worker.py index a8643fe8..9ed9b812 100644 --- a/lib/worker.py +++ b/lib/worker.py @@ -7,7 +7,7 @@ import yaml from datetime import datetime -import lib +from lib import Options from lib.colorer import color_log from lib.colorer import color_stdout from lib.tarantool_server import TarantoolServer @@ -20,13 +20,13 @@ def find_suites(): - suite_names = lib.Options().args.suites + suite_names = Options().args.suites if suite_names == []: for root, dirs, names in os.walk(os.getcwd(), followlinks=True): if "suite.ini" in names: suite_names.append(os.path.basename(root)) - suites = [TestSuite(suite_name, lib.Options().args) + suites = [TestSuite(suite_name, Options().args) for suite_name in sorted(suite_names)] return suites @@ -48,7 +48,7 @@ def parse_reproduce_file(filepath): def get_reproduce_file(worker_name): - main_vardir = os.path.realpath(lib.Options().args.vardir) + main_vardir = os.path.realpath(Options().args.vardir) reproduce_dir = os.path.join(main_vardir, 'reproduce') return os.path.join(reproduce_dir, '%s.list.yaml' % worker_name) @@ -100,7 +100,7 @@ def reproduce_task_groups(task_groups): this group as in the reproduce file. """ found_keys = [] - reproduce = parse_reproduce_file(lib.Options().args.reproduce) + reproduce = parse_reproduce_file(Options().args.reproduce) if not reproduce: raise ValueError('[reproduce] Tests list cannot be empty') for i, task_id in enumerate(reproduce): @@ -362,7 +362,7 @@ def run_loop(self, task_queue, result_queue): result_queue.put(self.wrap_result(task_id, short_status, result_checksum)) if short_status == 'fail': - if lib.Options().args.is_force: + if Options().args.is_force: self.restart_server() color_stdout( 'Worker "%s" got failed test; restarted the server\n' diff --git a/listeners.py b/listeners.py index 54f8dddc..b6a9b79b 100644 --- a/listeners.py +++ b/listeners.py @@ -3,7 +3,7 @@ import yaml import shutil -import lib +from lib import Options from lib.colorer import color_stdout from lib.colorer import decolor from lib.worker import WorkerCurrentTask @@ -13,6 +13,8 @@ from lib.worker import get_reproduce_file from lib.utils import prefix_each_line from lib.utils import safe_makedirs +from lib.utils import print_tail_n +from lib.utils import print_unidiff class BaseWatcher(object): @@ -69,7 +71,7 @@ def print_statistics(self): color_stdout('# reproduce file: %s\n' % reproduce_file_path) if show_reproduce_content: color_stdout("---\n", schema='separator') - lib.utils.print_tail_n(reproduce_file_path) + print_tail_n(reproduce_file_path) color_stdout("...\n", schema='separator') return True @@ -97,7 +99,7 @@ def save_artifacts(self): if not self.failed_workers: return - vardir = lib.Options().args.vardir + vardir = Options().args.vardir artifacts_dir = os.path.join(vardir, 'artifacts') artifacts_log_dir = os.path.join(artifacts_dir, 'log') artifacts_reproduce_dir = os.path.join(artifacts_dir, 'reproduce') @@ -124,7 +126,7 @@ def save_artifacts(self): class LogOutputWatcher(BaseWatcher): def __init__(self): self.fds = dict() - self.logdir = os.path.join(lib.Options().args.vardir, 'log') + self.logdir = os.path.join(Options().args.vardir, 'log') try: os.makedirs(self.logdir) except OSError: @@ -205,7 +207,7 @@ def process_result(self, obj): return # Skip color_log() events if --debug is not passed. - if obj.log_only and not lib.Options().args.debug: + if obj.log_only and not Options().args.debug: return # Prepend color_log() messages with a timestamp. @@ -267,7 +269,7 @@ def process_result(self, obj): return # Skip color_log() events if --debug is not passed. - if obj.log_only and not lib.Options().args.debug: + if obj.log_only and not Options().args.debug: return self.warned_seconds_ago = 0.0 @@ -291,7 +293,7 @@ def process_timeout(self, delta_seconds): schema=color_schema) hung_tasks = [task for worker_id, task - in self.worker_current_task.iteritems() + in self.worker_current_task.items() if worker_id in worker_ids] for task in hung_tasks: result_file = task.task_tmp_result @@ -313,7 +315,7 @@ def process_timeout(self, delta_seconds): for task in hung_tasks: color_stdout("Test hung! Result content mismatch:\n", schema='error') - lib.utils.print_unidiff(task.task_result, task.task_tmp_result) + print_unidiff(task.task_result, task.task_tmp_result) color_stdout('\n[Main process] No output from workers. ' 'It seems that we hang. Send SIGKILL to workers; ' 'exiting...\n', schema='error') diff --git a/requirements.txt b/requirements.txt index a1c72dcb..365d1b1f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML==5.1 argparse==1.1 msgpack-python==0.4.6 -gevent==1.1b5 +gevent==21.1.2 six>=1.8.0 diff --git a/test-run.py b/test-run.py index d63f5ed8..0b90494e 100755 --- a/test-run.py +++ b/test-run.py @@ -1,7 +1,6 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 """Tarantool regression test suite front-end.""" - # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: @@ -54,14 +53,17 @@ import sys import time -import lib -import lib.worker - -from dispatcher import Dispatcher +from lib import Options from lib.colorer import color_stdout +from lib.utils import print_tail_n +from lib.utils import PY3 +from lib.worker import get_task_groups +from lib.worker import get_reproduce_file +from lib.worker import reproduce_task_groups +from lib.worker import print_greetings +from dispatcher import Dispatcher from listeners import HangError - EXIT_SUCCESS = 0 EXIT_HANG = 1 EXIT_INTERRUPTED = 2 @@ -73,7 +75,7 @@ def main_loop_parallel(): color_stdout("Started {0}\n".format(" ".join(sys.argv)), schema='tr_text') - args = lib.Options().args + args = Options().args jobs = args.jobs if jobs < 1: # faster result I got was with 2 * cpu_count @@ -94,16 +96,16 @@ def main_loop_parallel(): format(args.no_output_timeout), schema='tr_text') color_stdout("\n", schema='tr_text') - task_groups = lib.worker.get_task_groups() - if lib.Options().args.reproduce: - task_groups = lib.worker.reproduce_task_groups(task_groups) + task_groups = get_task_groups() + if Options().args.reproduce: + task_groups = reproduce_task_groups(task_groups) jobs = 1 randomize = False dispatcher = Dispatcher(task_groups, jobs, randomize) dispatcher.start() - lib.worker.print_greetings() + print_greetings() color_stdout("\n", '=' * 86, "\n", schema='separator') color_stdout("WORKR".ljust(6), schema='t_name') @@ -113,7 +115,7 @@ def main_loop_parallel(): color_stdout('-' * 81, "\n", schema='separator') try: - is_force = lib.Options().args.is_force + is_force = Options().args.is_force dispatcher.wait() dispatcher.wait_processes() color_stdout('-' * 81, "\n", schema='separator') @@ -152,8 +154,8 @@ def main_parallel(): def main_loop_consistent(failed_test_ids): # find and prepare all tasks/groups, print information - task_groups = lib.worker.get_task_groups().items() - lib.worker.print_greetings() + task_groups = get_task_groups().items() + print_greetings() for name, task_group in task_groups: # print information about current test suite @@ -173,15 +175,15 @@ def main_loop_consistent(failed_test_ids): short_status = worker.run_task(task_id) if short_status == 'fail': reproduce_file_path = \ - lib.worker.get_reproduce_file(worker.name) + get_reproduce_file(worker.name) color_stdout('Reproduce file %s\n' % reproduce_file_path, schema='error') if show_reproduce_content: color_stdout("---\n", schema='separator') - lib.utils.print_tail_n(reproduce_file_path) + print_tail_n(reproduce_file_path) color_stdout("...\n", schema='separator') failed_test_ids.append(task_id) - if not lib.Options().args.is_force: + if not Options().args.is_force: worker.stop_server(cleanup=False) return @@ -203,11 +205,11 @@ def main_consistent(): except RuntimeError as e: color_stdout("\nFatal error: %s. Execution aborted.\n" % e, schema='error') - if lib.Options().args.gdb: + if Options().args.gdb: time.sleep(100) return -1 - if failed_test_ids and lib.Options().args.is_force: + if failed_test_ids and Options().args.is_force: color_stdout("\n===== %d tests failed:\n" % len(failed_test_ids), schema='error') for test_id in failed_test_ids: @@ -217,14 +219,61 @@ def main_consistent(): if __name__ == "__main__": + # In Python 3 start method 'spawn' in multiprocessing module becomes + # default on Mac OS. + # + # The 'spawn' method causes re-execution of some code, which is already + # executed in the main process. At least it is seen on the + # lib/__init__.py code, which removes the 'var' directory. Some other + # code may have side effects too, it requires investigation. + # + # The method also requires object serialization that doesn't work when + # objects use lambdas, whose for example used in class TestSuite + # (lib/test_suite.py). + # + # The latter problem is easy to fix, but the former looks more + # fundamental. So we stick to the 'fork' method now. + if PY3: + multiprocessing.set_start_method('fork') + + # test-run assumes that text file streams are UTF-8 (as + # contrary to ASCII) on Python 3. It is necessary to process + # non ASCII symbols in test files, result files and so on. + # + # Default text file stream encoding depends on a system + # locale with exception for the POSIX locale (C locale): in + # this case UTF-8 is used (see PEP-0540). Sadly, this + # behaviour is in effect since Python 3.7. + # + # We want to achieve the same behaviour on lower Python + # versions, at least on 3.6.8, which is provided by CentOS 7 + # and CentOS 8. + # + # So we hack the open() builtin. + # + # https://stackoverflow.com/a/53347548/1598057 + if PY3 and sys.version_info[0:2] < (3, 7): + std_open = __builtins__.open + + def open_as_utf8(*args, **kwargs): + if len(args) >= 2: + mode = args[1] + else: + mode = kwargs.get('mode', '') + if 'b' not in mode: + kwargs.setdefault('encoding', 'utf-8') + return std_open(*args, **kwargs) + + __builtins__.open = open_as_utf8 + # don't sure why, but it values 1 or 2 gives 1.5x speedup for parallel # test-run (and almost doesn't affect consistent test-run) os.environ['OMP_NUM_THREADS'] = '2' status = 0 - force_parallel = bool(lib.Options().args.reproduce) - if not force_parallel and lib.Options().args.jobs == -1: + force_parallel = bool(Options().args.reproduce) + if not force_parallel and Options().args.jobs == -1: status = main_consistent() else: status = main_parallel()