Skip to content
This repository was archived by the owner on Nov 17, 2023. It is now read-only.

[WIP] This fix enables running tests on the CI with pytest-parallel #20352

Draft
akarbown wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

[WIP] This fix enables running tests on the CI with pytest-parallel #20352

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ci/docker/install/requirements
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ pytest==6.1.2
pytest-env==0.6.2
pytest-cov==2.10.1
pytest-xdist==2.1.0
pytest-parallel==0.1.0
pytest-timeout==1.4.2
flaky==3.7.0
setuptools==49.6.0 # https://github.com/pypa/setuptools/issues/2352
Expand Down
46 changes: 23 additions & 23 deletions ci/docker/runtime_functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -720,7 +720,7 @@ sanity_python() {
set -ex
export DMLC_LOG_STACK_TRACE_DEPTH=100
python3 -m pylint --rcfile=ci/other/pylintrc --ignore-patterns=".*\.so$$,.*\.dll$$,.*\.dylib$$" python/mxnet
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/tutorials/test_sanity_tutorials.py
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest --timeout-method=thread --workers 4 tests/tutorials/test_sanity_tutorials.py
}

# Tests libmxnet
Expand All @@ -739,7 +739,7 @@ cd_unittest_ubuntu() {

local mxnet_variant=${1:?"This function requires a mxnet variant as the first argument"}

OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -n 4 --durations=50 --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' --timeout-method=thread --workers 4 --durations=50 --verbose tests/python/unittest
pytest -m 'serial' --durations=50 --verbose tests/python/unittest

# https://github.com/apache/incubator-mxnet/issues/11801
Expand All @@ -750,9 +750,9 @@ cd_unittest_ubuntu() {
if [[ ${mxnet_variant} = cu* ]]; then
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
MXNET_ENGINE_TYPE=NaiveEngine \
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' --timeout-method=thread --workers 4 --durations=50 --verbose tests/python/gpu
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations=50 --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' --timeout-method=thread --workers 4 --durations=50 --verbose tests/python/gpu
pytest -m 'serial' --durations=50 --verbose tests/python/gpu
pytest --durations=50 --verbose tests/python/gpu/test_amp_init.py

Expand All @@ -763,7 +763,7 @@ cd_unittest_ubuntu() {
fi

if [[ ${mxnet_variant} = *mkl ]]; then
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 --durations=50 --verbose tests/python/mkl
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest --timeout-method=thread --workers 4 --durations=50 --verbose tests/python/mkl
fi
}

Expand All @@ -775,9 +775,9 @@ unittest_ubuntu_python3_cpu() {
export MXNET_SUBGRAPH_VERBOSE=0
export MXNET_ENABLE_CYTHON=0
export DMLC_LOG_STACK_TRACE_DEPTH=100
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
MXNET_ENGINE_TYPE=NaiveEngine \
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
}

Expand All @@ -789,9 +789,9 @@ unittest_ubuntu_python3_cpu_onednn() {
export MXNET_SUBGRAPH_VERBOSE=0
export MXNET_ENABLE_CYTHON=0
export DMLC_LOG_STACK_TRACE_DEPTH=100
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
MXNET_ENGINE_TYPE=NaiveEngine \
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
pytest --durations=50 --cov-report xml:tests_mkl.xml --verbose tests/python/mkl
}
Expand All @@ -806,10 +806,10 @@ unittest_ubuntu_python3_gpu() {
export MXNET_ENABLE_CYTHON=0
export DMLC_LOG_STACK_TRACE_DEPTH=100
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
MXNET_ENGINE_TYPE=NaiveEngine \
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
pytest --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu/test_amp_init.py
}
Expand All @@ -826,10 +826,10 @@ unittest_ubuntu_python3_gpu_cython() {
export DMLC_LOG_STACK_TRACE_DEPTH=100
check_cython
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
MXNET_ENGINE_TYPE=NaiveEngine \
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
pytest --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu/test_amp_init.py
}
Expand All @@ -843,10 +843,10 @@ unittest_ubuntu_python3_gpu_nocudnn() {
export MXNET_ENABLE_CYTHON=0
export DMLC_LOG_STACK_TRACE_DEPTH=100
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
MXNET_ENGINE_TYPE=NaiveEngine \
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
pytest --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu/test_amp_init.py
}
Expand All @@ -862,11 +862,11 @@ unittest_centos7_cpu() {
source /opt/rh/rh-python36/enable
cd /work/mxnet
export DMLC_LOG_STACK_TRACE_DEPTH=100
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -m 'not serial' -k 'not test_operator' --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
MXNET_ENGINE_TYPE=NaiveEngine \
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -m 'not serial' -k 'test_operator' --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
python -m pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -n 4 --durations=50 --cov-report xml:tests_train.xml --verbose tests/python/train
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_train.xml --verbose tests/python/train
}

unittest_centos7_gpu() {
Expand All @@ -876,10 +876,10 @@ unittest_centos7_gpu() {
export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3}
export DMLC_LOG_STACK_TRACE_DEPTH=100
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
MXNET_ENGINE_TYPE=NaiveEngine \
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' --timeout-method=thread --workers 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
pytest --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu/test_amp_init.py
}
Expand Down Expand Up @@ -978,9 +978,9 @@ test_ubuntu_cpu_python3() {
cd /work/mxnet/python
pip3 install -e .
cd /work/mxnet
OMP_NUM_THREADS=$(expr $(nproc) / 4) python3 -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) python3 -m pytest -m 'not serial' -k 'not test_operator' --timeout-method=thread --workers 4 --durations=50 --verbose tests/python/unittest
MXNET_ENGINE_TYPE=NaiveEngine \
OMP_NUM_THREADS=$(expr $(nproc) / 4) python3 -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) python3 -m pytest -m 'not serial' -k 'test_operator' --timeout-method=thread --workers 4 --durations=50 --verbose tests/python/unittest
python3 -m pytest -m 'serial' --durations=50 --verbose tests/python/unittest

popd
Expand Down Expand Up @@ -1390,7 +1390,7 @@ test_artifact_repository() {
set -ex
pushd .
cd cd/utils/
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 test_artifact_repository.py
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest --timeout-method=thread --workers 4 test_artifact_repository.py
popd
}

Expand Down
11 changes: 7 additions & 4 deletions python/mxnet/gluon/data/dataloader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -660,13 +660,16 @@ def __init__(self, dataset, batch_size=None, shuffle=False, sampler=None,
initializer=_thread_worker_initializer,
initargs=(is_np_shape(), is_np_array()))
else:
# set ignore keyboard interupt signal before forking processes
original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
# according to the: https://bugs.python.org/issue39042
if threading.current_thread() == threading.main_thread():
# set ignore keyboard interupt signal before forking processes
original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
self._worker_pool = multiprocessing.Pool(
self._num_workers, initializer=_worker_initializer,
initargs=[self._dataset, is_np_shape(), is_np_array()])
# resume keyboard interupt signal in main process
signal.signal(signal.SIGINT, original_sigint_handler)
if threading.current_thread() == threading.main_thread():
# resume keyboard interupt signal in main process
signal.signal(signal.SIGINT, original_sigint_handler)

def __iter__(self):
if self._mx_iter is not None:
Expand Down
1 change: 1 addition & 0 deletions tests/python/unittest/test_ndarray.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,7 @@ def test_ndarray_pickle():
assert np.sum(a.asnumpy() != a2.asnumpy()) == 0


@pytest.mark.serial
@pytest.mark.parametrize('save_fn', [mx.nd.save, mx.npx.savez])
def test_ndarray_saveload(save_fn):
nrepeat = 10
Expand Down