From 9d47eed432fe8435e9bd368c14b9c687e3219de4 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 15 May 2018 23:22:39 +0900 Subject: [PATCH 01/84] Fix broken build with cython 0.28 --- python/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index 980ce6393701..c4106d06f859 100644 --- a/python/setup.py +++ b/python/setup.py @@ -69,7 +69,7 @@ def config_cython(): try: from Cython.Build import cythonize - # from setuptools.extension import Extension + from setuptools.extension import Extension if sys.version_info >= (3, 0): subdir = "_cy3" else: @@ -87,7 +87,7 @@ def config_cython(): if not fn.endswith(".pyx"): continue ret.append(Extension( - "mxnet/%s/.%s" % (subdir, fn[:-4]), + "mxnet.%s.%s" % (subdir, fn[:-4]), ["mxnet/cython/%s" % fn], include_dirs=["../include/", "../3rdparty/nnvm/include"], library_dirs=library_dirs, From b3a9784592395e6aaf8058e5deb26c80e6aeea50 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 26 May 2018 17:24:54 +0900 Subject: [PATCH 02/84] Fix setup.py to be compatible with cython 0.28 --- python/setup.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/python/setup.py b/python/setup.py index c4106d06f859..652fa706f7e6 100644 --- a/python/setup.py +++ b/python/setup.py @@ -20,6 +20,7 @@ from __future__ import absolute_import import os import sys +from setuptools import find_packages # need to use distutils.core for correct placement of cython dll kwargs = {} if "--inplace" in sys.argv: @@ -29,7 +30,6 @@ from setuptools import setup from setuptools.extension import Extension kwargs = {'install_requires': ['numpy<=1.15.0,>=1.8.2', 'requests<2.19.0,>=2.18.4', 'graphviz<0.9.0,>=0.8.1'], 'zip_safe': False} -from setuptools import find_packages with_cython = False if '--with-cython' in sys.argv: @@ -69,7 +69,6 @@ def config_cython(): try: from Cython.Build import cythonize - from setuptools.extension import Extension if sys.version_info >= (3, 0): subdir = "_cy3" else: @@ -80,8 +79,11 @@ def config_cython(): library_dirs = ['mxnet', '../build/Release', '../build'] libraries = ['libmxnet'] else: - library_dirs = None - libraries = None + library_dirs = ['../lib'] + libraries = ['mxnet'] + # Default paths to libmxnet.so relative to the shared library file generated by cython. + # These precede LD_LIBRARY_PATH. + extra_link_args = ["-Wl,-rpath=$ORIGIN/..:$ORIGIN/../../../lib"] for fn in os.listdir(path): if not fn.endswith(".pyx"): @@ -92,6 +94,7 @@ def config_cython(): include_dirs=["../include/", "../3rdparty/nnvm/include"], library_dirs=library_dirs, libraries=libraries, + extra_link_args=extra_link_args, language="c++")) return cythonize(ret) except ImportError: From feecd35177ddc874d7952dc8f16617575c3c1127 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 26 May 2018 17:30:59 +0900 Subject: [PATCH 03/84] Fix broken cython ndarray module --- python/mxnet/cython/base.pyi | 36 ++++++++++------ python/mxnet/cython/ndarray.pyx | 75 ++++++++++++++++++++++++--------- 2 files changed, 77 insertions(+), 34 deletions(-) diff --git a/python/mxnet/cython/base.pyi b/python/mxnet/cython/base.pyi index d73e1a7d0194..56605299b5ff 100644 --- a/python/mxnet/cython/base.pyi +++ b/python/mxnet/cython/base.pyi @@ -90,20 +90,30 @@ cdef extern from "mxnet/c_api.h": int MXSymbolSetAttr(SymbolHandle symbol, const char* key, const char* value); - int MXImperativeInvoke(OpHandle creator, + int MXImperativeInvokeEx(OpHandle creator, + int num_inputs, + NDArrayHandle *inputs, + int *num_outputs, + NDArrayHandle **outputs, + int num_params, + const char **param_keys, + const char **param_vals, + const int **out_stypes); + int MXNDArrayFree(NDArrayHandle handle); + int MXCreateCachedOpEx(SymbolHandle handle, + int num_flags, + const char** keys, + const char** vals, + int num_inputs, + const char** input_names, + int num_params, + const char** param_names, + NDArrayHandle* params, + CachedOpHandle *out); + int MXFreeCachedOp(CachedOpHandle handle); + int MXInvokeCachedOpEx(CachedOpHandle handle, int num_inputs, NDArrayHandle *inputs, int *num_outputs, NDArrayHandle **outputs, - int num_params, - const char **param_keys, - const char **param_vals); - int MXNDArrayFree(NDArrayHandle handle); - int MXCreateCachedOp(SymbolHandle handle, - CachedOpHandle *out); - int MXFreeCachedOp(CachedOpHandle handle); - int MXInvokeCachedOp(CachedOpHandle handle, - int num_inputs, - NDArrayHandle *inputs, - int *num_outputs, - NDArrayHandle **outputs); + const int **out_stypes); diff --git a/python/mxnet/cython/ndarray.pyx b/python/mxnet/cython/ndarray.pyx index 319dc492dbb8..f91127d8fb65 100644 --- a/python/mxnet/cython/ndarray.pyx +++ b/python/mxnet/cython/ndarray.pyx @@ -52,7 +52,7 @@ cdef class NDArrayBase: def __get__(self): return bool(self.cwritable) - def __init__(self, handle, writable=True): + def __init__(self, handle, writable=True, stype=-1): self._set_handle(handle) self.cwritable = writable @@ -70,12 +70,9 @@ def _set_ndarray_class(cls): _ndarray_cls = cls -cdef NewArray(NDArrayHandle handle): +cdef NewArray(NDArrayHandle handle, int stype=-1): """Create a new array given handle""" - nd = _ndarray_cls(None) - (nd).chandle = handle - (nd).cwritable = True - return nd + return _ndarray_cls(_ctypes.cast(handle, _ctypes.c_void_p), stype=stype) cdef class CachedOp: @@ -99,10 +96,44 @@ cdef class CachedOp: def __set__(self, value): self._set_handle(value) - def __init__(self, sym): - cdef unsigned long long ptr = sym.handle.value - CALL(MXCreateCachedOp( - (ptr), + def __init__(self, sym, flags=(), inputs=None, params=None): + cdef vector[string] s_flag_keys + cdef vector[string] s_flag_vals + if flags is not None: + for k, v in flags: + s_flag_keys.push_back(c_str(k)) + s_flag_vals.push_back(c_str(str(v))) + cdef vector[const char*] c_flag_keys = SVec2Ptr(s_flag_keys) + cdef vector[const char*] c_flag_vals = SVec2Ptr(s_flag_vals) + + cdef vector[string] s_param_names + cdef vector[NDArrayHandle] param_arrays + if inputs is None: + assert params is None, "When inputs is None params must also be None." + inputs = sym.list_inputs() + elif params is not None: + for name, arrs in params.items(): + c_name = c_str(name) + for a in arrs: + param_arrays.push_back((a).chandle) + s_param_names.push_back(c_name) + cdef vector[const char*] param_names = SVec2Ptr(s_param_names) + + cdef vector[string] s_inputs + for i in inputs: + s_inputs.push_back(c_str(i)) + cdef vector[const char*] c_inputs = SVec2Ptr(s_inputs) + + CALL(MXCreateCachedOpEx( + (sym.handle.value), + len(flags), + CBeginPtr(c_flag_keys), + CBeginPtr(c_flag_vals), + len(inputs), + CBeginPtr(c_inputs), + len(param_names), + CBeginPtr(param_names), + ¶m_arrays[0] if param_arrays.size() != 0 else NULL, &self.chandle)) def __del__(self): @@ -115,6 +146,7 @@ cdef class CachedOp: cdef NDArrayHandle* p_output_vars cdef NDArrayHandle ret_handle cdef int num_output + cdef const int* p_output_stypes for i in args: ndvars.push_back((i).chandle) @@ -130,24 +162,24 @@ cdef class CachedOp: num_output = output_vars.size() if output_vars.size() == 0: - output_vars.resize(1) p_output_vars = NULL else: p_output_vars = &output_vars[0] - CALL(MXInvokeCachedOp( - (self).chandle, + CALL(MXInvokeCachedOpEx( + self.chandle, len(args), &ndvars[0] if ndvars.size() != 0 else NULL, &num_output, - &p_output_vars)) + &p_output_vars, + &p_output_stypes)) if original_output is not None: return original_output if num_output == 1: - return NewArray(p_output_vars[0]) + return NewArray(p_output_vars[0], p_output_stypes[0]) else: - return tuple(NewArray(p_output_vars[i]) for i in range(num_output)) + return [NewArray(p_output_vars[i], p_output_stypes[i]) for i in range(num_output)] def _imperative_invoke(handle, ndargs, keys, vals, out): @@ -161,6 +193,7 @@ def _imperative_invoke(handle, ndargs, keys, vals, out): cdef NDArrayHandle* p_output_vars cdef NDArrayHandle ret_handle cdef int num_output + cdef const int* p_output_stypes for i in ndargs: ndvars.push_back((i).chandle) @@ -180,7 +213,6 @@ def _imperative_invoke(handle, ndargs, keys, vals, out): num_output = output_vars.size() if output_vars.size() == 0: - output_vars.resize(1) p_output_vars = NULL else: p_output_vars = &output_vars[0] @@ -188,7 +220,7 @@ def _imperative_invoke(handle, ndargs, keys, vals, out): cdef vector[const char*] param_keys = SVec2Ptr(ckeys) cdef vector[const char*] param_vals = SVec2Ptr(cvals) - CALL(MXImperativeInvoke( + CALL(MXImperativeInvokeEx( chandle, ndvars.size(), &ndvars[0] if ndvars.size() != 0 else NULL, @@ -196,11 +228,12 @@ def _imperative_invoke(handle, ndargs, keys, vals, out): &p_output_vars, param_keys.size(), CBeginPtr(param_keys), - CBeginPtr(param_vals))) + CBeginPtr(param_vals), + &p_output_stypes)) if original_output is not None: return original_output if num_output == 1: - return NewArray(p_output_vars[0]) + return NewArray(p_output_vars[0], p_output_stypes[0]) else: - return tuple(NewArray(p_output_vars[i]) for i in range(num_output)) + return [NewArray(p_output_vars[i], p_output_stypes[i]) for i in range(num_output)] From fcc41a04f52e45af854b277e1859dee3a9f497bb Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 26 May 2018 17:31:47 +0900 Subject: [PATCH 04/84] Revised comments --- python/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 652fa706f7e6..1c0ea1d65517 100644 --- a/python/setup.py +++ b/python/setup.py @@ -20,7 +20,8 @@ from __future__ import absolute_import import os import sys -from setuptools import find_packages +from setuptools import find_packages # This must precede distutils + # need to use distutils.core for correct placement of cython dll kwargs = {} if "--inplace" in sys.argv: From 771a9b6705dd389c5715a5e223b0b3fb1ec6d57e Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sun, 27 May 2018 04:08:26 +0900 Subject: [PATCH 05/84] Replace hard coded library path with one obtained by find_lib_path --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 1c0ea1d65517..17a2dca8ecc9 100644 --- a/python/setup.py +++ b/python/setup.py @@ -80,7 +80,7 @@ def config_cython(): library_dirs = ['mxnet', '../build/Release', '../build'] libraries = ['libmxnet'] else: - library_dirs = ['../lib'] + library_dirs = [os.path.dirname(p) for p in LIB_PATH] libraries = ['mxnet'] # Default paths to libmxnet.so relative to the shared library file generated by cython. # These precede LD_LIBRARY_PATH. From df1a291144e5ceabd5d3815aaef0651e1d8921d3 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sun, 27 May 2018 23:39:56 +0900 Subject: [PATCH 06/84] Add documentation for MXNET_ENABLE_CYTHON and MXNET_ENFORCE_CYTHON --- docs/faq/env_var.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/faq/env_var.md b/docs/faq/env_var.md index f29301dec7aa..b9d743b836e9 100644 --- a/docs/faq/env_var.md +++ b/docs/faq/env_var.md @@ -111,6 +111,19 @@ When USE_PROFILER is enabled in Makefile or CMake, the following environments ca - If set to '0', profiler records the events of the symbolic operators. - If set to '1', profiler records the events of all operators. +## Interface between Python and the C API + +* MXNET_ENABLE_CYTHON + - Values: 0(false), 1(true) ```(default=1)``` + - If set to 0, MXNet uses the ctypes to interface with the C API. + - If set to 1, MXNet tries to use the cython modules for the ndarray and symbol. If it fails, the ctypes is used or an error occurs depending on MXNET_ENFORCE_CYTHON. + +* MXNET_ENFORCE_CYTHON + - Values: 0(false) or 1(true) ```(default=0)``` + - This has an effect only if MXNET_ENABLE_CYTHON is 1. + - If set to 0, MXNet fallbacks to the ctypes if importing the cython modules fails. + - If set to 1, MXNet raises an error if importing the cython modules fails. + ## Other Environment Variables * MXNET_CUDNN_AUTOTUNE_DEFAULT @@ -120,7 +133,7 @@ When USE_PROFILER is enabled in Makefile or CMake, the following environments ca - Performance tests are run to pick the convolution algo when value is 1 or 2 - Value of 1 chooses the best algo in a limited workspace - Value of 2 chooses the fastest algo whose memory requirements may be larger than the default workspace threshold - + * MXNET_GLUON_REPO - Values: String ```(default='https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/'``` From 00009503a25a9fd3dea95b4873908ff37d393d5d Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 16 Jun 2018 15:28:01 +0900 Subject: [PATCH 07/84] Add cython build to CI --- Jenkinsfile | 10 ++++++---- ci/docker/runtime_functions.sh | 4 ++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 56fbf3d74af9..93fc81f03b38 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,6 +22,8 @@ // mxnet libraries mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' +// mxnet libraries with cython +mx_cython_lib = 'lib/libmxnet.so, lib/libmxnet.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' // for scala build, need to pass extra libs when run with dist_kvstore mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. @@ -170,7 +172,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() docker_run('centos7_cpu', 'build_centos7_cpu', false) - pack_lib('centos7_cpu') + pack_lib('centos7_cpu', mx_cython_lib) } } } @@ -192,7 +194,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() docker_run('centos7_gpu', 'build_centos7_gpu', false) - pack_lib('centos7_gpu') + pack_lib('centos7_gpu', mx_cython_lib) } } } @@ -632,7 +634,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { try { init_git() - unpack_lib('centos7_cpu') + unpack_lib('centos7_cpu', mx_cython_lib) docker_run('centos7_cpu', 'unittest_centos7_cpu', false) } finally { collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_centos7_cpu_unittest.xml') @@ -648,7 +650,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { try { init_git() - unpack_lib('centos7_gpu') + unpack_lib('centos7_gpu', mx_cython_lib) docker_run('centos7_gpu', 'unittest_centos7_gpu', true) } finally { collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_centos7_gpu.xml') diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 7170b8b22cc6..4aa9a03ed413 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -267,6 +267,7 @@ build_centos7_cpu() { USE_BLAS=openblas \ USE_DIST_KVSTORE=1 \ -j$(nproc) + make cython } build_centos7_mkldnn() { @@ -298,6 +299,7 @@ build_centos7_gpu() { USE_CUDNN=1 \ USE_DIST_KVSTORE=1 \ -j$(nproc) + make cython } build_ubuntu_cpu() { @@ -633,6 +635,7 @@ unittest_ubuntu_gpu_R() { unittest_centos7_cpu() { set -ex cd /work/mxnet + export MXNET_ENFORCE_CYTHON=1 python3.6 -m "nose" --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest python3.6 -m "nose" --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train } @@ -640,6 +643,7 @@ unittest_centos7_cpu() { unittest_centos7_gpu() { set -ex cd /work/mxnet + export MXNET_ENFORCE_CYTHON=1 python3.6 -m "nose" --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } From 86d1c903d654d8fc42f6c77b820eba7b50cb457b Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 16 Jun 2018 15:45:32 +0900 Subject: [PATCH 08/84] Fix for cython CI --- Jenkinsfile | 2 +- ci/docker/install/centos7_python.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 93fc81f03b38..915baa7eb9c6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,7 +23,7 @@ // mxnet libraries mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' // mxnet libraries with cython -mx_cython_lib = 'lib/libmxnet.so, lib/libmxnet.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' +mx_cython_lib = 'lib/libmxnet.so, lib/libmxnet.a, python/mxnet/_cy3/*.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' // for scala build, need to pass extra libs when run with dist_kvstore mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. diff --git a/ci/docker/install/centos7_python.sh b/ci/docker/install/centos7_python.sh index 154e3b8e4f56..30f79c2d2284 100755 --- a/ci/docker/install/centos7_python.sh +++ b/ci/docker/install/centos7_python.sh @@ -31,5 +31,5 @@ curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py" python2.7 get-pip.py python3.6 get-pip.py -pip2 install nose pylint numpy nose-timer requests h5py scipy==1.0.1 -pip3 install nose pylint numpy nose-timer requests h5py scipy==1.0.1 +pip2 install nose pylint numpy nose-timer requests h5py scipy==1.0.1 Cython +pip3 install nose pylint numpy nose-timer requests h5py scipy==1.0.1 Cython From f8a7b8c1e401e0fba5590f0c5f52c49aeac97bda Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 16 Jun 2018 16:00:25 +0900 Subject: [PATCH 09/84] Adjust python environment for cython CI --- ci/docker/install/centos7_python.sh | 2 +- ci/docker/runtime_functions.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/docker/install/centos7_python.sh b/ci/docker/install/centos7_python.sh index 30f79c2d2284..3bc485fd81e5 100755 --- a/ci/docker/install/centos7_python.sh +++ b/ci/docker/install/centos7_python.sh @@ -24,7 +24,7 @@ set -ex # Python 2.7 is installed by default, install 3.6 on top yum -y install https://centos7.iuscommunity.org/ius-release.rpm -yum -y install python36u +yum -y install python36u python36u-devel python27-devel # Install PIP curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py" diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 4aa9a03ed413..2ae7721b5789 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -267,7 +267,7 @@ build_centos7_cpu() { USE_BLAS=openblas \ USE_DIST_KVSTORE=1 \ -j$(nproc) - make cython + make cython3 } build_centos7_mkldnn() { @@ -299,7 +299,7 @@ build_centos7_gpu() { USE_CUDNN=1 \ USE_DIST_KVSTORE=1 \ -j$(nproc) - make cython + make cython3 } build_ubuntu_cpu() { From 646ea9123d1a553205b791d14b6a32570cd5a8cf Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 16 Jun 2018 16:34:04 +0900 Subject: [PATCH 10/84] Add make variables to set python executable --- Makefile | 2 +- ci/docker/runtime_functions.sh | 4 ++-- make/config.mk | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index ff4446ab80c7..07cfcde3f545 100644 --- a/Makefile +++ b/Makefile @@ -528,7 +528,7 @@ doxygen: # Cython build cython: - cd python; python setup.py build_ext --inplace --with-cython + cd python; $(PYTHON) setup.py build_ext --inplace --with-cython cython2: cd python; python2 setup.py build_ext --inplace --with-cython diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 2ae7721b5789..448db5c405f1 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -267,7 +267,7 @@ build_centos7_cpu() { USE_BLAS=openblas \ USE_DIST_KVSTORE=1 \ -j$(nproc) - make cython3 + make cython3 PYTHON=python3.6 } build_centos7_mkldnn() { @@ -299,7 +299,7 @@ build_centos7_gpu() { USE_CUDNN=1 \ USE_DIST_KVSTORE=1 \ -j$(nproc) - make cython3 + make cython PYTHON=python3.6 } build_ubuntu_cpu() { diff --git a/make/config.mk b/make/config.mk index b65f77c605f7..ce2b19c268fa 100644 --- a/make/config.mk +++ b/make/config.mk @@ -196,6 +196,9 @@ EXTRA_OPERATORS = # Create C++ interface package USE_CPP_PACKAGE = 0 +# Python executable. Needed for cython target +PYTHON = python + #---------------------------- # plugins #---------------------------- From 64057b3402c90e8c7392e40dad6b932dcda20477 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 16 Jun 2018 16:44:56 +0900 Subject: [PATCH 11/84] Fix typo --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 448db5c405f1..f2b082c0bf0e 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -267,7 +267,7 @@ build_centos7_cpu() { USE_BLAS=openblas \ USE_DIST_KVSTORE=1 \ -j$(nproc) - make cython3 PYTHON=python3.6 + make cython PYTHON=python3.6 } build_centos7_mkldnn() { From 5aaa8adc74276a94d3e25c9991621021fbbba7aa Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 16 Jun 2018 17:07:40 +0900 Subject: [PATCH 12/84] Fix nnvm include path --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 17a2dca8ecc9..dc7c63c99b77 100644 --- a/python/setup.py +++ b/python/setup.py @@ -92,7 +92,7 @@ def config_cython(): ret.append(Extension( "mxnet.%s.%s" % (subdir, fn[:-4]), ["mxnet/cython/%s" % fn], - include_dirs=["../include/", "../3rdparty/nnvm/include"], + include_dirs=["../include/", "../3rdparty/tvm/nnvm/include"], library_dirs=library_dirs, libraries=libraries, extra_link_args=extra_link_args, From 1f4d53aaec404a77478da811592b26f415e81764 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 16 Jun 2018 17:24:44 +0900 Subject: [PATCH 13/84] Does not use ccache for cython --- ci/docker/runtime_functions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index f2b082c0bf0e..90dbcfea6443 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -267,6 +267,8 @@ build_centos7_cpu() { USE_BLAS=openblas \ USE_DIST_KVSTORE=1 \ -j$(nproc) + export CC="gcc" + export CXX="g++" make cython PYTHON=python3.6 } From d252beafae4d62f11f4d96d3d7b42f84299f859f Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 16 Jun 2018 17:58:47 +0900 Subject: [PATCH 14/84] Fix issues with the wildcards in the library list in Jenkinsfile --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 915baa7eb9c6..7d0fe37bf878 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -80,7 +80,7 @@ def init_git_win() { def pack_lib(name, libs=mx_lib) { sh """ echo "Packing ${libs} into ${name}" -echo ${libs} | sed -e 's/,/ /g' | xargs md5sum +for i in $(echo ${libs} | sed -e 's/,/ /g'); do md5sum $i; done """ stash includes: libs, name: name } @@ -90,7 +90,7 @@ def unpack_lib(name, libs=mx_lib) { unstash name sh """ echo "Unpacked ${libs} from ${name}" -echo ${libs} | sed -e 's/,/ /g' | xargs md5sum +for i in $(echo ${libs} | sed -e 's/,/ /g'); do md5sum $i; done """ } From cbae43dea44123ae932ba9fd3cad31fce7909b07 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 16 Jun 2018 18:07:28 +0900 Subject: [PATCH 15/84] Fix issues with the wildcards in the library list in Jenkinsfile (continued) --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7d0fe37bf878..b3473f3d89f9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -80,7 +80,7 @@ def init_git_win() { def pack_lib(name, libs=mx_lib) { sh """ echo "Packing ${libs} into ${name}" -for i in $(echo ${libs} | sed -e 's/,/ /g'); do md5sum $i; done +for i in $(echo ${libs} | sed -e 's/,/ /g'); do md5sum \$i; done """ stash includes: libs, name: name } @@ -90,7 +90,7 @@ def unpack_lib(name, libs=mx_lib) { unstash name sh """ echo "Unpacked ${libs} from ${name}" -for i in $(echo ${libs} | sed -e 's/,/ /g'); do md5sum $i; done +for i in $(echo ${libs} | sed -e 's/,/ /g'); do md5sum \$i; done """ } From be0744a9490a8c43d74b199568dac2e040fbfe32 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 16 Jun 2018 18:13:46 +0900 Subject: [PATCH 16/84] Fix issues with the wildcards in the library list in Jenkinsfile (continued) --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b3473f3d89f9..fc34f8fd9da6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -80,7 +80,7 @@ def init_git_win() { def pack_lib(name, libs=mx_lib) { sh """ echo "Packing ${libs} into ${name}" -for i in $(echo ${libs} | sed -e 's/,/ /g'); do md5sum \$i; done +for i in \$(echo ${libs} | sed -e 's/,/ /g'); do md5sum \$i; done """ stash includes: libs, name: name } @@ -90,7 +90,7 @@ def unpack_lib(name, libs=mx_lib) { unstash name sh """ echo "Unpacked ${libs} from ${name}" -for i in $(echo ${libs} | sed -e 's/,/ /g'); do md5sum \$i; done +for i in \$(echo ${libs} | sed -e 's/,/ /g'); do md5sum \$i; done """ } From 40ba9de94bd0cc62ced43143eddf860341776f22 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 16 Jun 2018 20:56:22 +0900 Subject: [PATCH 17/84] Intentionally introduce a bug to check that the tests actually runwith cython --- python/mxnet/cython/ndarray.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/mxnet/cython/ndarray.pyx b/python/mxnet/cython/ndarray.pyx index f91127d8fb65..8e47a39530f6 100644 --- a/python/mxnet/cython/ndarray.pyx +++ b/python/mxnet/cython/ndarray.pyx @@ -195,8 +195,8 @@ def _imperative_invoke(handle, ndargs, keys, vals, out): cdef int num_output cdef const int* p_output_stypes - for i in ndargs: - ndvars.push_back((i).chandle) + #for i in ndargs: + # ndvars.push_back((i).chandle) for i in keys: ckeys.push_back(c_str(i)) for i in vals: From b5b0a54465683780098ac54c42d51da2d0fa9839 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 16 Jun 2018 21:46:00 +0900 Subject: [PATCH 18/84] Remove the intentionally introduced bug --- python/mxnet/cython/ndarray.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/mxnet/cython/ndarray.pyx b/python/mxnet/cython/ndarray.pyx index 8e47a39530f6..f91127d8fb65 100644 --- a/python/mxnet/cython/ndarray.pyx +++ b/python/mxnet/cython/ndarray.pyx @@ -195,8 +195,8 @@ def _imperative_invoke(handle, ndargs, keys, vals, out): cdef int num_output cdef const int* p_output_stypes - #for i in ndargs: - # ndvars.push_back((i).chandle) + for i in ndargs: + ndvars.push_back((i).chandle) for i in keys: ckeys.push_back(c_str(i)) for i in vals: From c798d38913e380738f5cde149d6a32697a5ae90c Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 16 Jun 2018 22:49:21 +0900 Subject: [PATCH 19/84] Update installation doc --- docs/install/index.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/install/index.md b/docs/install/index.md index cd8a8d690cc3..fae926728de8 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -459,7 +459,16 @@ $ make -j $(nproc) USE_OPENCV=1 USE_BLAS=openblas $ sudo apt-get install -y python-dev python-setuptools python-pip libgfortran3 ``` -**Step 2** Install the MXNet Python binding. +**Step 2** Build cython modules (optional). + +```bash +$ sudo apt-get install -y cython +$ make cython # You can set the python executable with `PYTHON` flag, e.g., make cython PYTHON=python3 +``` + +Note that you can control the use of the cython modules at runtime via the environment variables `MXNET_ENABLE_CYTHON` and `MXNET_ENFORCE_CYTHON`. See [here](/faq/env_var.html) for details. + +**Step 3** Install the MXNet Python binding. ```bash $ cd python @@ -468,13 +477,13 @@ $ pip install -e . Note that the `-e` flag is optional. It is equivalent to `--editable` and means that if you edit the source files, these changes will be reflected in the package installed. -**Step 3** Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package). +**Step 4** Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package). ```bash sudo apt-get install graphviz pip install graphviz ``` -**Step 4** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation). +**Step 5** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation). From 206e706fe791c1e985741d64a22dec3cca7c9fb1 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sun, 17 Jun 2018 00:39:21 +0900 Subject: [PATCH 20/84] Retrigger CI From 6df4e28a9cb1555d792de41bff596d49d7e9e0f2 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 19 Jun 2018 17:56:05 +0900 Subject: [PATCH 21/84] Run cython CI in ubuntu environment instead of CentOS environment --- Jenkinsfile | 26 ++++++++++++----------- ci/docker/install/centos7_python.sh | 6 +++--- ci/docker/runtime_functions.sh | 32 ++++++++++++++++++++++------- 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fc34f8fd9da6..1891a720ef3a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,10 +22,12 @@ // mxnet libraries mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' -// mxnet libraries with cython -mx_cython_lib = 'lib/libmxnet.so, lib/libmxnet.a, python/mxnet/_cy3/*.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' +// mx_lib with cython +mx_cython_lib = 'lib/libmxnet.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' // for scala build, need to pass extra libs when run with dist_kvstore mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' +// mx_dist_lib with cython +mx_cython_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' @@ -172,7 +174,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() docker_run('centos7_cpu', 'build_centos7_cpu', false) - pack_lib('centos7_cpu', mx_cython_lib) + pack_lib('centos7_cpu') } } } @@ -194,7 +196,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() docker_run('centos7_gpu', 'build_centos7_gpu', false) - pack_lib('centos7_gpu', mx_cython_lib) + pack_lib('centos7_gpu') } } } @@ -205,7 +207,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false) - pack_lib('cpu', mx_dist_lib) + pack_lib('cpu', mx_cython_dist_lib) } } } @@ -280,7 +282,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7', false) - pack_lib('gpu', mx_dist_lib) + pack_lib('gpu', mx_cython_dist_lib) stash includes: 'build/cpp-package/example/lenet', name: 'cpp_lenet' stash includes: 'build/cpp-package/example/alexnet', name: 'cpp_alexnet' stash includes: 'build/cpp-package/example/googlenet', name: 'cpp_googlenet' @@ -493,7 +495,7 @@ try { ws('workspace/ut-python2-cpu') { try { init_git() - unpack_lib('cpu') + unpack_lib('cpu', mx_cython_lib) python2_ut('ubuntu_cpu') } finally { collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_cpu_unittest.xml') @@ -508,7 +510,7 @@ try { ws('workspace/ut-python3-cpu') { try { init_git() - unpack_lib('cpu') + unpack_lib('cpu', mx_cython_lib) python3_ut('ubuntu_cpu') } finally { collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_unittest.xml') @@ -522,7 +524,7 @@ try { ws('workspace/ut-python2-gpu') { try { init_git() - unpack_lib('gpu', mx_lib) + unpack_lib('gpu', mx_cython_lib) python2_gpu_ut('ubuntu_gpu') } finally { collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_gpu.xml') @@ -535,7 +537,7 @@ try { ws('workspace/ut-python3-gpu') { try { init_git() - unpack_lib('gpu', mx_lib) + unpack_lib('gpu', mx_cython_lib) python3_gpu_ut('ubuntu_gpu') } finally { collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_gpu.xml') @@ -634,7 +636,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { try { init_git() - unpack_lib('centos7_cpu', mx_cython_lib) + unpack_lib('centos7_cpu') docker_run('centos7_cpu', 'unittest_centos7_cpu', false) } finally { collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_centos7_cpu_unittest.xml') @@ -650,7 +652,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { try { init_git() - unpack_lib('centos7_gpu', mx_cython_lib) + unpack_lib('centos7_gpu') docker_run('centos7_gpu', 'unittest_centos7_gpu', true) } finally { collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_centos7_gpu.xml') diff --git a/ci/docker/install/centos7_python.sh b/ci/docker/install/centos7_python.sh index 3bc485fd81e5..154e3b8e4f56 100755 --- a/ci/docker/install/centos7_python.sh +++ b/ci/docker/install/centos7_python.sh @@ -24,12 +24,12 @@ set -ex # Python 2.7 is installed by default, install 3.6 on top yum -y install https://centos7.iuscommunity.org/ius-release.rpm -yum -y install python36u python36u-devel python27-devel +yum -y install python36u # Install PIP curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py" python2.7 get-pip.py python3.6 get-pip.py -pip2 install nose pylint numpy nose-timer requests h5py scipy==1.0.1 Cython -pip3 install nose pylint numpy nose-timer requests h5py scipy==1.0.1 Cython +pip2 install nose pylint numpy nose-timer requests h5py scipy==1.0.1 +pip3 install nose pylint numpy nose-timer requests h5py scipy==1.0.1 diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 3a980f39add0..e4a8f7713e43 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -31,6 +31,15 @@ clean_repo() { git submodule update --init --recursive } +check_cython() { + set -ex + local python_ver=$1 + if [ "$(echo -e 'import mxnet as mx\nprint(mx.nd._internal.NDArrayBase)' | python)" != "" ]; then + echo "Error: cython is not used." + return 1 + fi +} + build_ccache_wrappers() { set -ex @@ -292,10 +301,6 @@ build_centos7_cpu() { -j$(nproc) report_ccache_usage - - export CC="gcc" - export CXX="g++" - make cython PYTHON=python3.6 } build_centos7_mkldnn() { @@ -330,7 +335,6 @@ build_centos7_gpu() { USE_CUDNN=1 \ USE_DIST_KVSTORE=1 \ -j$(nproc) - make cython PYTHON=python3.6 } build_ubuntu_cpu() { @@ -349,6 +353,11 @@ build_ubuntu_cpu_openblas() { -j$(nproc) report_ccache_usage + + export CC="gcc" + export CXX="g++" + make cython PYTHON=python2 + make cython PYTHON=python3 } build_ubuntu_cpu_clang39() { @@ -473,6 +482,9 @@ build_ubuntu_gpu_cuda91_cudnn7() { USE_CPP_PACKAGE=1 \ USE_DIST_KVSTORE=1 \ -j$(nproc) + + make cython PYTHON=python2 + make cython PYTHON=python3 } build_ubuntu_amalgamation() { @@ -548,6 +560,8 @@ unittest_ubuntu_python2_cpu() { # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export MXNET_ENFORCE_CYTHON=1 + check_cython 2 nosetests-2.7 --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest nosetests-2.7 --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train nosetests-2.7 --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization @@ -560,6 +574,8 @@ unittest_ubuntu_python3_cpu() { # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export MXNET_ENFORCE_CYTHON=1 + check_cython 3 nosetests-3.4 --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest nosetests-3.4 --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization } @@ -582,6 +598,8 @@ unittest_ubuntu_python2_gpu() { # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export MXNET_ENFORCE_CYTHON=1 + check_cython 2 nosetests-2.7 --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } @@ -616,6 +634,8 @@ unittest_ubuntu_python3_gpu() { # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export MXNET_ENFORCE_CYTHON=1 + check_cython 3 nosetests-3.4 --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } @@ -686,7 +706,6 @@ unittest_ubuntu_gpu_R() { unittest_centos7_cpu() { set -ex cd /work/mxnet - export MXNET_ENFORCE_CYTHON=1 python3.6 -m "nose" --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest python3.6 -m "nose" --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train } @@ -694,7 +713,6 @@ unittest_centos7_cpu() { unittest_centos7_gpu() { set -ex cd /work/mxnet - export MXNET_ENFORCE_CYTHON=1 python3.6 -m "nose" --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } From 741d0887200e6f3208bbe0c9e3e4fe89333ae8c0 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 19 Jun 2018 18:05:00 +0900 Subject: [PATCH 22/84] Commit a missed file --- ci/docker/install/ubuntu_python.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python.sh index da7c25697b6c..46cb30cf70a9 100755 --- a/ci/docker/install/ubuntu_python.sh +++ b/ci/docker/install/ubuntu_python.sh @@ -29,5 +29,5 @@ wget -nv https://bootstrap.pypa.io/get-pip.py python3 get-pip.py python2 get-pip.py -pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 -pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 +pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 Cython +pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 Cython From b32b8a6ec56f07cce982921f221a42400715c6fa Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 19 Jun 2018 19:26:29 +0900 Subject: [PATCH 23/84] Fix a bug in check_cython --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index e4a8f7713e43..05cf3b5ec74e 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -34,7 +34,7 @@ clean_repo() { check_cython() { set -ex local python_ver=$1 - if [ "$(echo -e 'import mxnet as mx\nprint(mx.nd._internal.NDArrayBase)' | python)" != "" ]; then + if [ "$(echo -e 'import mxnet as mx\nprint(mx.nd._internal.NDArrayBase.__module__)' | python${python_ver})" != "mxnet._cy${python_ver}.ndarray" ]; then echo "Error: cython is not used." return 1 fi From 9a526ecc56ea8583228d887e30ece699370e7766 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 19 Jun 2018 20:07:02 +0900 Subject: [PATCH 24/84] Refine environments for cython CI --- Jenkinsfile | 24 +++++++++++++++++------- ci/docker/runtime_functions.sh | 25 +++++++++++++++++++++---- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1891a720ef3a..74b7ff60b60c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,11 +22,9 @@ // mxnet libraries mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' -// mx_lib with cython -mx_cython_lib = 'lib/libmxnet.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' +mx_cython_lib = 'lib/libmxnet.so, lib/libmxnet.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' // for scala build, need to pass extra libs when run with dist_kvstore mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' -// mx_dist_lib with cython mx_cython_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' @@ -135,6 +133,12 @@ def python3_ut(docker_container_name) { } } +def python3_cython_ut(docker_container_name) { + timeout(time: max_time, unit: 'MINUTES') { + docker_run(docker_container_name, 'unittest_ubuntu_python3_cython_cpu', false) + } +} + def python3_ut_mkldnn(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu_mkldnn', false) @@ -150,6 +154,12 @@ def python2_gpu_ut(docker_container_name) { } } +def python2_cython_gpu_ut(docker_container_name) { + timeout(time: max_time, unit: 'MINUTES') { + docker_run(docker_container_name, 'unittest_ubuntu_python2_cython_gpu', true) + } +} + // Python 3 def python3_gpu_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { @@ -495,7 +505,7 @@ try { ws('workspace/ut-python2-cpu') { try { init_git() - unpack_lib('cpu', mx_cython_lib) + unpack_lib('cpu', mx_lib) python2_ut('ubuntu_cpu') } finally { collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_cpu_unittest.xml') @@ -511,7 +521,7 @@ try { try { init_git() unpack_lib('cpu', mx_cython_lib) - python3_ut('ubuntu_cpu') + python3_cython_ut('ubuntu_cpu') } finally { collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_unittest.xml') collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python3_cpu_quantization.xml') @@ -525,7 +535,7 @@ try { try { init_git() unpack_lib('gpu', mx_cython_lib) - python2_gpu_ut('ubuntu_gpu') + python2_cython_gpu_ut('ubuntu_gpu') } finally { collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_gpu.xml') } @@ -537,7 +547,7 @@ try { ws('workspace/ut-python3-gpu') { try { init_git() - unpack_lib('gpu', mx_cython_lib) + unpack_lib('gpu', mx_lib) python3_gpu_ut('ubuntu_gpu') } finally { collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_gpu.xml') diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 05cf3b5ec74e..dde1a27f6f43 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -560,14 +560,23 @@ unittest_ubuntu_python2_cpu() { # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 - export MXNET_ENFORCE_CYTHON=1 - check_cython 2 nosetests-2.7 --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest nosetests-2.7 --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train nosetests-2.7 --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization } unittest_ubuntu_python3_cpu() { + set -ex + export PYTHONPATH=./python/ + # MXNET_MKLDNN_DEBUG is buggy and produces false positives + # https://github.com/apache/incubator-mxnet/issues/10026 + #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present + export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + nosetests-3.4 --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest + nosetests-3.4 --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization +} + +unittest_ubuntu_python3_cython_cpu() { set -ex export PYTHONPATH=./python/ # MXNET_MKLDNN_DEBUG is buggy and produces false positives @@ -592,6 +601,16 @@ unittest_ubuntu_python3_cpu_mkldnn() { } unittest_ubuntu_python2_gpu() { + set -ex + export PYTHONPATH=./python/ + # MXNET_MKLDNN_DEBUG is buggy and produces false positives + # https://github.com/apache/incubator-mxnet/issues/10026 + #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present + export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + nosetests-2.7 --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu +} + +unittest_ubuntu_python2_cython_gpu() { set -ex export PYTHONPATH=./python/ # MXNET_MKLDNN_DEBUG is buggy and produces false positives @@ -634,8 +653,6 @@ unittest_ubuntu_python3_gpu() { # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 - export MXNET_ENFORCE_CYTHON=1 - check_cython 3 nosetests-3.4 --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } From 17286a49f5e95368d4be36907c54e316bb1480bc Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 19 Jun 2018 20:13:14 +0900 Subject: [PATCH 25/84] Restore unrelated changes --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 74b7ff60b60c..28e303dfd855 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -505,7 +505,7 @@ try { ws('workspace/ut-python2-cpu') { try { init_git() - unpack_lib('cpu', mx_lib) + unpack_lib('cpu') python2_ut('ubuntu_cpu') } finally { collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_cpu_unittest.xml') From 238f40cbf5f8d75ff0b24edd2b8d884110156b8d Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Wed, 20 Jun 2018 00:36:06 +0900 Subject: [PATCH 26/84] Fix a problem occurring when the cython modules for python 2 and 3 are built successively --- ci/docker/runtime_functions.sh | 5 ++++- python/setup.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index dde1a27f6f43..11d1d2cd1fd3 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -35,8 +35,11 @@ check_cython() { set -ex local python_ver=$1 if [ "$(echo -e 'import mxnet as mx\nprint(mx.nd._internal.NDArrayBase.__module__)' | python${python_ver})" != "mxnet._cy${python_ver}.ndarray" ]; then - echo "Error: cython is not used." + echo "ERROR: cython is not used." return 1 + else + echo "NOTE: cython is used." + return 0 fi } diff --git a/python/setup.py b/python/setup.py index dc7c63c99b77..8d992d916188 100644 --- a/python/setup.py +++ b/python/setup.py @@ -97,7 +97,9 @@ def config_cython(): libraries=libraries, extra_link_args=extra_link_args, language="c++")) - return cythonize(ret) + # If `force=True` is not used and you cythonize the modules for python2 and python3 + # successively, you need to delete `mxnet/cython/ndarray.cpp` after the first cythonize. + return cythonize(ret, force=True) except ImportError: print("WARNING: Cython is not installed, will compile without cython module") return [] From 7284615d414ed49008cf2d4f6be3b1c4f52de5eb Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Wed, 20 Jun 2018 01:13:03 +0900 Subject: [PATCH 27/84] Trigger CI From 57c341085d5d83067e8cddf84efe2de5e134113f Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Wed, 20 Jun 2018 14:53:12 +0900 Subject: [PATCH 28/84] Pin the cython version in the CI --- ci/docker/install/ubuntu_python.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python.sh index 46cb30cf70a9..422d956676ac 100755 --- a/ci/docker/install/ubuntu_python.sh +++ b/ci/docker/install/ubuntu_python.sh @@ -29,5 +29,5 @@ wget -nv https://bootstrap.pypa.io/get-pip.py python3 get-pip.py python2 get-pip.py -pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 Cython -pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 Cython +pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 Cython==0.28.2 +pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 Cython==0.28.2 From 96ed71fa1385c77fe2813c6676a93209e670412d Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 23 Jun 2018 01:17:30 +0900 Subject: [PATCH 29/84] Catch up #11320 --- python/mxnet/cython/base.pyi | 5 ----- python/mxnet/cython/ndarray.pyx | 23 ----------------------- 2 files changed, 28 deletions(-) diff --git a/python/mxnet/cython/base.pyi b/python/mxnet/cython/base.pyi index 56605299b5ff..548afc782763 100644 --- a/python/mxnet/cython/base.pyi +++ b/python/mxnet/cython/base.pyi @@ -104,11 +104,6 @@ cdef extern from "mxnet/c_api.h": int num_flags, const char** keys, const char** vals, - int num_inputs, - const char** input_names, - int num_params, - const char** param_names, - NDArrayHandle* params, CachedOpHandle *out); int MXFreeCachedOp(CachedOpHandle handle); int MXInvokeCachedOpEx(CachedOpHandle handle, diff --git a/python/mxnet/cython/ndarray.pyx b/python/mxnet/cython/ndarray.pyx index f91127d8fb65..2a401441659a 100644 --- a/python/mxnet/cython/ndarray.pyx +++ b/python/mxnet/cython/ndarray.pyx @@ -106,34 +106,11 @@ cdef class CachedOp: cdef vector[const char*] c_flag_keys = SVec2Ptr(s_flag_keys) cdef vector[const char*] c_flag_vals = SVec2Ptr(s_flag_vals) - cdef vector[string] s_param_names - cdef vector[NDArrayHandle] param_arrays - if inputs is None: - assert params is None, "When inputs is None params must also be None." - inputs = sym.list_inputs() - elif params is not None: - for name, arrs in params.items(): - c_name = c_str(name) - for a in arrs: - param_arrays.push_back((a).chandle) - s_param_names.push_back(c_name) - cdef vector[const char*] param_names = SVec2Ptr(s_param_names) - - cdef vector[string] s_inputs - for i in inputs: - s_inputs.push_back(c_str(i)) - cdef vector[const char*] c_inputs = SVec2Ptr(s_inputs) - CALL(MXCreateCachedOpEx( (sym.handle.value), len(flags), CBeginPtr(c_flag_keys), CBeginPtr(c_flag_vals), - len(inputs), - CBeginPtr(c_inputs), - len(param_names), - CBeginPtr(param_names), - ¶m_arrays[0] if param_arrays.size() != 0 else NULL, &self.chandle)) def __del__(self): From 991dd8ce4815b942e1a5359434f9bfdd40eb97d7 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 23 Jun 2018 11:24:16 +0900 Subject: [PATCH 30/84] Remove optional arguments unused after #11320 --- python/mxnet/cython/ndarray.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/cython/ndarray.pyx b/python/mxnet/cython/ndarray.pyx index 2a401441659a..3f05fc7e6a0f 100644 --- a/python/mxnet/cython/ndarray.pyx +++ b/python/mxnet/cython/ndarray.pyx @@ -96,7 +96,7 @@ cdef class CachedOp: def __set__(self, value): self._set_handle(value) - def __init__(self, sym, flags=(), inputs=None, params=None): + def __init__(self, sym, flags=()): cdef vector[string] s_flag_keys cdef vector[string] s_flag_vals if flags is not None: From d779ec9d0a32ebf0b79c2f48b6190fecb9474892 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 23 Jun 2018 15:29:07 +0900 Subject: [PATCH 31/84] Trigger CI From 5922f88612ff761b51c81136d85792edfe8cad58 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 26 Jun 2018 12:21:17 +0900 Subject: [PATCH 32/84] Trigger CI From ed1e1268aa5fc0cda611213f98ac2d8e516ad3a1 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Wed, 1 Aug 2018 13:41:46 +0900 Subject: [PATCH 33/84] Trigger CI From 430f38ded3ea1e86a6ff3dd2674fc940d23ad94a Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 2 Aug 2018 11:13:53 +0900 Subject: [PATCH 34/84] Remove unnecessary stype argument from the NDArrayBase constructor --- python/mxnet/cython/ndarray.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/cython/ndarray.pyx b/python/mxnet/cython/ndarray.pyx index 3f05fc7e6a0f..b352f04a242d 100644 --- a/python/mxnet/cython/ndarray.pyx +++ b/python/mxnet/cython/ndarray.pyx @@ -52,7 +52,7 @@ cdef class NDArrayBase: def __get__(self): return bool(self.cwritable) - def __init__(self, handle, writable=True, stype=-1): + def __init__(self, handle, writable=True): self._set_handle(handle) self.cwritable = writable From e3ad1caaca2983ce625f7079ce6a0068c05602be Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 2 Aug 2018 12:57:59 +0900 Subject: [PATCH 35/84] Revise confusing initialization of `_ndarray_cls` --- python/mxnet/cython/ndarray.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/cython/ndarray.pyx b/python/mxnet/cython/ndarray.pyx index b352f04a242d..f9279889b504 100644 --- a/python/mxnet/cython/ndarray.pyx +++ b/python/mxnet/cython/ndarray.pyx @@ -63,7 +63,7 @@ cdef class NDArrayBase: return (_ndarray_cls, (None,), self.__getstate__()) -_ndarray_cls = NDArrayBase +_ndarray_cls = None def _set_ndarray_class(cls): global _ndarray_cls From d0ef6fe2fb158f6f417c5e9314fa1875e13cfce6 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Fri, 3 Aug 2018 17:27:39 +0900 Subject: [PATCH 36/84] Add cython build for python3 in CI --- ci/docker/runtime_functions.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 9d00bf396d2e..15f7bf26e365 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -86,7 +86,8 @@ build_cython() { export MXNET_LIBRARY_PATH=${BUILD_DIR}/libmxnet.so cd ${PYTHON_DIR} - python setup.py build_ext --inplace --with-cython + python2 setup.py build_ext --inplace --with-cython + python3 setup.py build_ext --inplace --with-cython popd } From 8140a4d71c59612c66c00ecb43358a1811d94eb5 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Fri, 3 Aug 2018 17:32:11 +0900 Subject: [PATCH 37/84] Fix misplaced cython build in CI --- ci/docker/runtime_functions.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 15f7bf26e365..09ded070acec 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -343,6 +343,11 @@ build_ubuntu_cpu_openblas() { USE_BLAS=openblas \ USE_DIST_KVSTORE=1 \ -j$(nproc) + + export CC="gcc" + export CXX="g++" + make cython PYTHON=python2 + make cython PYTHON=python3 } build_ubuntu_cpu_cmake_debug() { @@ -487,9 +492,6 @@ build_ubuntu_gpu_cuda91_cudnn7() { USE_CPP_PACKAGE=1 \ USE_DIST_KVSTORE=1 \ -j$(nproc) - - make cython PYTHON=python2 - make cython PYTHON=python3 } build_ubuntu_amalgamation() { From 5093d83937ff35dcd62ca8ee6c3af400ee0a3f6c Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sun, 5 Aug 2018 13:46:34 +0900 Subject: [PATCH 38/84] Adjust CI environments for cython --- Jenkinsfile | 200 ++++++++------------------------- ci/docker/runtime_functions.sh | 58 ++++++---- 2 files changed, 84 insertions(+), 174 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 59ecdb983cea..9d672dbb3a02 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,14 +22,14 @@ // mxnet libraries mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' -mx_cython_lib = 'lib/libmxnet.so, lib/libmxnet.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' // for scala build, need to pass extra libs when run with dist_kvstore mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' -mx_cython_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' +mx_dist_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' +mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. -mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' +mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests' mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' // timeout in minutes @@ -136,16 +136,16 @@ def python2_ut(docker_container_name) { } } -// Python 3 -def python3_ut(docker_container_name) { +def python2_ut_cython(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu', false) + docker_run(docker_container_name, 'unittest_ubuntu_python2_cpu_cython', false) } } -def python3_cython_ut(docker_container_name) { +// Python 3 +def python3_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - docker_run(docker_container_name, 'unittest_ubuntu_python3_cython_cpu', false) + docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu', false) } } @@ -164,16 +164,16 @@ def python2_gpu_ut(docker_container_name) { } } -def python2_cython_gpu_ut(docker_container_name) { +// Python 3 +def python3_gpu_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - docker_run(docker_container_name, 'unittest_ubuntu_python2_cython_gpu', true) + docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu', true) } } -// Python 3 -def python3_gpu_ut(docker_container_name) { +def python3_gpu_ut_cython(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu', true) + docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu_cython', true) } } @@ -244,7 +244,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false) - pack_lib('cpu', mx_cython_dist_lib) + pack_lib('cpu', mx_dist_lib_cython) } } } @@ -341,7 +341,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7', false) - pack_lib('gpu', mx_cython_dist_lib) + pack_lib('gpu', mx_dist_lib) stash includes: 'build/cpp-package/example/lenet', name: 'cpp_lenet' stash includes: 'build/cpp-package/example/alexnet', name: 'cpp_alexnet' stash includes: 'build/cpp-package/example/googlenet', name: 'cpp_googlenet' @@ -394,7 +394,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false) - pack_lib('cmake_gpu', mx_cmake_lib) + pack_lib('cmake_gpu', mx_cmake_lib_cython) } } } @@ -405,28 +405,8 @@ try { ws('workspace/build-cpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { init_git_win() - bat """mkdir build_vc14_cpu - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" - cd build_vc14_cpu - cmake -G \"Visual Studio 14 2015 Win64\" -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DUSE_MKL_IF_AVAILABLE=0 ${env.WORKSPACE}""" - bat 'C:\\mxnet\\build_vc14_cpu.bat' - - bat '''rmdir /s/q pkg_vc14_cpu - mkdir pkg_vc14_cpu\\lib - mkdir pkg_vc14_cpu\\python - mkdir pkg_vc14_cpu\\include - mkdir pkg_vc14_cpu\\build - copy build_vc14_cpu\\Release\\libmxnet.lib pkg_vc14_cpu\\lib - copy build_vc14_cpu\\Release\\libmxnet.dll pkg_vc14_cpu\\build - xcopy python pkg_vc14_cpu\\python /E /I /Y - xcopy include pkg_vc14_cpu\\include /E /I /Y - xcopy 3rdparty\\dmlc-core\\include pkg_vc14_cpu\\include /E /I /Y - xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_cpu\\include\\mshadow /E /I /Y - xcopy 3rdparty\\nnvm\\include pkg_vc14_cpu\\nnvm\\include /E /I /Y - del /Q *.7z - 7z.exe a vc14_cpu.7z pkg_vc14_cpu\\ - ''' - stash includes: 'vc14_cpu.7z', name: 'vc14_cpu' + powershell 'python ci/build_windows.py -f WIN_CPU' + stash includes: 'windows_package.7z', name: 'windows_package_cpu' } } } @@ -438,28 +418,9 @@ try { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-gpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { - init_git_win() - bat """mkdir build_vc14_gpu - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" - cd build_vc14_gpu - cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN=52 -DCUDA_ARCH_PTX=52 -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release -DUSE_MKL_IF_AVAILABLE=0 ${env.WORKSPACE}""" - bat 'C:\\mxnet\\build_vc14_gpu.bat' - bat '''rmdir /s/q pkg_vc14_gpu - mkdir pkg_vc14_gpu\\lib - mkdir pkg_vc14_gpu\\python - mkdir pkg_vc14_gpu\\include - mkdir pkg_vc14_gpu\\build - copy build_vc14_gpu\\libmxnet.lib pkg_vc14_gpu\\lib - copy build_vc14_gpu\\libmxnet.dll pkg_vc14_gpu\\build - xcopy python pkg_vc14_gpu\\python /E /I /Y - xcopy include pkg_vc14_gpu\\include /E /I /Y - xcopy 3rdparty\\dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y - xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y - xcopy 3rdparty\\nnvm\\include pkg_vc14_gpu\\nnvm\\include /E /I /Y - del /Q *.7z - 7z.exe a vc14_gpu.7z pkg_vc14_gpu\\ - ''' - stash includes: 'vc14_gpu.7z', name: 'vc14_gpu' + init_git_win() + powershell 'python ci/build_windows.py -f WIN_GPU' + stash includes: 'windows_package.7z', name: 'windows_package_gpu' } } } @@ -470,37 +431,9 @@ try { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-gpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_gpu_mkldnn']) { - init_git_win() - bat """mkdir build_%BUILD_NAME% - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" - cd build_%BUILD_NAME% - copy ${env.WORKSPACE}\\3rdparty\\mkldnn\\config_template.vcxproj.user ${env.WORKSPACE}\\config_template.vcxproj.user /y - cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN=52 -DCUDA_ARCH_PTX=52 -DUSE_MKLDNN=1 -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release ${env.WORKSPACE}""" - bat ''' - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" - cd build_%BUILD_NAME% - set /a cores=%NUMBER_OF_PROCESSORS% * 2 - jom -j %cores% - ''' - bat '''rmdir /s/q pkg_%BUILD_NAME% - mkdir pkg_%BUILD_NAME%\\lib - mkdir pkg_%BUILD_NAME%\\python - mkdir pkg_%BUILD_NAME%\\include - mkdir pkg_%BUILD_NAME%\\build - copy build_%BUILD_NAME%\\libmxnet.lib pkg_%BUILD_NAME%\\lib - copy build_%BUILD_NAME%\\libmxnet.dll pkg_%BUILD_NAME%\\build - copy build_%BUILD_NAME%\\3rdparty\\mkldnn\\src\\mkldnn.dll pkg_%BUILD_NAME%\\build - copy build_%BUILD_NAME%\\libiomp5md.dll pkg_%BUILD_NAME%\\build - copy build_%BUILD_NAME%\\mklml.dll pkg_%BUILD_NAME%\\build - xcopy python pkg_%BUILD_NAME%\\python /E /I /Y - xcopy include pkg_%BUILD_NAME%\\include /E /I /Y - xcopy 3rdparty\\dmlc-core\\include pkg_%BUILD_NAME%\\include /E /I /Y - xcopy 3rdparty\\mshadow\\mshadow pkg_%BUILD_NAME%\\include\\mshadow /E /I /Y - xcopy 3rdparty\\nnvm\\include pkg_%BUILD_NAME%\\nnvm\\include /E /I /Y - del /Q *.7z - 7z.exe a %BUILD_NAME%.7z pkg_%BUILD_NAME%\\ - ''' - stash includes: 'vc14_gpu_mkldnn.7z', name: 'vc14_gpu_mkldnn' + init_git_win() + powershell 'python ci/build_windows.py -f WIN_GPU_MKLDNN' + stash includes: 'windows_package.7z', name: 'windows_package_gpu_mkldnn' } } } @@ -575,8 +508,8 @@ try { ws('workspace/ut-python2-cpu') { try { init_git() - unpack_lib('cpu') - python2_ut('ubuntu_cpu') + unpack_lib('cpu', mx_dist_lib_cython) + python2_ut_cython('ubuntu_cpu') publish_test_coverage() } finally { collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_cpu_unittest.xml') @@ -591,8 +524,8 @@ try { ws('workspace/ut-python3-cpu') { try { init_git() - unpack_lib('cpu', mx_cython_lib) - python3_cython_ut('ubuntu_cpu') + unpack_lib('cpu') + python3_ut('ubuntu_cpu') publish_test_coverage() } finally { collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_unittest.xml') @@ -620,8 +553,8 @@ try { ws('workspace/ut-python2-gpu') { try { init_git() - unpack_lib('gpu', mx_cython_lib) - python2_cython_gpu_ut('ubuntu_gpu') + unpack_lib('gpu', mx_lib) + python2_gpu_ut('ubuntu_gpu') publish_test_coverage() } finally { collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_gpu.xml') @@ -634,8 +567,8 @@ try { ws('workspace/ut-python3-gpu') { try { init_git() - unpack_lib('gpu', mx_lib) - python3_gpu_ut('ubuntu_gpu') + unpack_lib('cmake_gpu', mx_cmake_lib_cython) + python3_gpu_ut_cython('ubuntu_gpu') publish_test_coverage() } finally { collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_gpu.xml') @@ -884,16 +817,8 @@ try { ws('workspace/ut-python-cpu') { try { init_git_win() - unstash 'vc14_cpu' - bat '''rmdir /s/q pkg_vc14_cpu - 7z x -y vc14_cpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py2 - pip install mock - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc - C:\\mxnet\\test_cpu.bat""" + unstash 'windows_package_cpu' + powershell 'ci/windows/test_py2_cpu.ps1' } finally { collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python2_cpu.xml') } @@ -907,15 +832,8 @@ try { ws('workspace/ut-python-cpu') { try { init_git_win() - unstash 'vc14_cpu' - bat '''rmdir /s/q pkg_vc14_cpu - 7z x -y vc14_cpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py3 - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc - C:\\mxnet\\test_cpu.bat""" + unstash 'windows_package_cpu' + powershell 'ci/windows/test_py3_cpu.ps1' } finally { collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python3_cpu.xml') } @@ -929,19 +847,11 @@ try { ws('workspace/ut-python-gpu') { try { init_git_win() - unstash 'vc14_gpu' - bat '''rmdir /s/q pkg_vc14_gpu - 7z x -y vc14_gpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py2 - pip install mock - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc - C:\\mxnet\\test_gpu.bat""" + unstash 'windows_package_gpu' + powershell 'ci/windows/test_py2_gpu.ps1' } finally { - collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python2_gpu.xml') - collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python2_gpu.xml') + collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python2_gpu.xml') + collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python2_gpu.xml') } } } @@ -953,18 +863,11 @@ try { ws('workspace/ut-python-gpu') { try { init_git_win() - unstash 'vc14_gpu' - bat '''rmdir /s/q pkg_vc14_gpu - 7z x -y vc14_gpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py3 - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc - C:\\mxnet\\test_gpu.bat""" + unstash 'windows_package_gpu' + powershell 'ci/windows/test_py3_gpu.ps1' } finally { - collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu.xml') - collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu.xml') + collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu.xml') + collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu.xml') } } } @@ -976,18 +879,11 @@ try { ws('workspace/ut-python-gpu') { try { init_git_win() - unstash 'vc14_gpu_mkldnn' - bat '''rmdir /s/q pkg_vc14_gpu_mkldnn - 7z x -y vc14_gpu_mkldnn.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py3 - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu_mkldnn\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu_mkldnn\\python\\*.pyc - C:\\mxnet\\test_gpu.bat""" + unstash 'windows_package_gpu_mkldnn' + powershell 'ci/windows/test_py3_gpu.ps1' } finally { - collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu_mkldnn.xml') - collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu_mkldnn.xml') + collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu_mkldnn.xml') + collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu_mkldnn.xml') } } } diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 09ded070acec..840407444bce 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -343,7 +343,7 @@ build_ubuntu_cpu_openblas() { USE_BLAS=openblas \ USE_DIST_KVSTORE=1 \ -j$(nproc) - + export CC="gcc" export CXX="g++" make cython PYTHON=python2 @@ -366,7 +366,6 @@ build_ubuntu_cpu_cmake_debug() { /work/mxnet ninja -v - build_cython popd } @@ -544,6 +543,7 @@ build_ubuntu_gpu_cmake() { /work/mxnet ninja -v + build_cython } @@ -565,33 +565,37 @@ unittest_ubuntu_python2_cpu() { # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export MXNET_ENABLE_CYTHON=0 nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization } -unittest_ubuntu_python3_cpu() { +unittest_ubuntu_python2_cpu_cython() { set -ex export PYTHONPATH=./python/ # MXNET_MKLDNN_DEBUG is buggy and produces false positives # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 - nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest - nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization + export MXNET_ENABLE_CYTHON=1 + export MXNET_ENFORCE_CYTHON=1 + check_cython 2 + nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest + nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train + nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization } -unittest_ubuntu_python3_cython_cpu() { +unittest_ubuntu_python3_cpu() { set -ex export PYTHONPATH=./python/ # MXNET_MKLDNN_DEBUG is buggy and produces false positives # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 - export MXNET_ENFORCE_CYTHON=1 - check_cython 3 - nosetests-3.4 --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest - nosetests-3.4 --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization + export MXNET_ENABLE_CYTHON=0 + nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest + nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization } unittest_ubuntu_python3_cpu_mkldnn() { @@ -601,6 +605,7 @@ unittest_ubuntu_python3_cpu_mkldnn() { # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export MXNET_ENABLE_CYTHON=0 nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_mkl.xml --verbose tests/python/mkl } @@ -612,27 +617,17 @@ unittest_ubuntu_python2_gpu() { # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export MXNET_ENABLE_CYTHON=0 nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } -unittest_ubuntu_python2_cython_gpu() { - set -ex - export PYTHONPATH=./python/ - # MXNET_MKLDNN_DEBUG is buggy and produces false positives - # https://github.com/apache/incubator-mxnet/issues/10026 - #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 - export MXNET_ENFORCE_CYTHON=1 - check_cython 2 - nosetests-2.7 --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu -} - tutorialtest_ubuntu_python3_gpu() { set -ex cd /work/mxnet/docs export MXNET_DOCS_BUILD_MXNET=0 make html export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export MXNET_ENABLE_CYTHON=0 export PYTHONPATH=/work/mxnet/python/ export MXNET_TUTORIAL_TEST_KERNEL=python3 cd /work/mxnet/tests/tutorials @@ -645,6 +640,7 @@ tutorialtest_ubuntu_python2_gpu() { export MXNET_DOCS_BUILD_MXNET=0 make html export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export MXNET_ENABLE_CYTHON=0 export PYTHONPATH=/work/mxnet/python/ export MXNET_TUTORIAL_TEST_KERNEL=python2 cd /work/mxnet/tests/tutorials @@ -658,6 +654,20 @@ unittest_ubuntu_python3_gpu() { # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export MXNET_ENABLE_CYTHON=0 + nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu +} + +unittest_ubuntu_python3_gpu_cython() { + set -ex + export PYTHONPATH=./python/ + # MXNET_MKLDNN_DEBUG is buggy and produces false positives + # https://github.com/apache/incubator-mxnet/issues/10026 + #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present + export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export MXNET_ENABLE_CYTHON=1 + export MXNET_ENFORCE_CYTHON=1 + check_cython 3 nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } @@ -666,6 +676,7 @@ unittest_ubuntu_python3_gpu_nocudnn() { export PYTHONPATH=./python/ export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 export CUDNN_OFF_TEST_ONLY=true + export MXNET_ENABLE_CYTHON=0 nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } @@ -678,6 +689,7 @@ unittest_ubuntu_python2_quantization_gpu() { # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export MXNET_ENABLE_CYTHON=0 nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization_gpu.xml --verbose tests/python/quantization_gpu } @@ -690,6 +702,7 @@ unittest_ubuntu_python3_quantization_gpu() { # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export MXNET_ENABLE_CYTHON=0 nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization_gpu.xml --verbose tests/python/quantization_gpu } @@ -790,6 +803,7 @@ integrationtest_ubuntu_cpu_dist_kvstore() { ../../tools/launch.py -n 7 --launcher local python dist_sync_kvstore.py --no-multiprecision ../../tools/launch.py -n 7 --launcher local python dist_sync_kvstore.py --type=compressed_cpu ../../tools/launch.py -n 7 --launcher local python dist_sync_kvstore.py --type=compressed_cpu --no-multiprecision + ../../tools/launch.py -n 3 --launcher local python test_server_profiling.py } integrationtest_ubuntu_gpu_scala() { From bf798c94a2892c53fdfeb9b9fa5bd2ab014be48e Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sun, 5 Aug 2018 14:25:07 +0900 Subject: [PATCH 39/84] Fix invalid path for cython generated .so files in cmake build --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9d672dbb3a02..e400f12eab9b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,7 +27,7 @@ mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3r mx_dist_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' -mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' +mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, mxnet/python/mxnet/_cy2/*.so, mxnet/python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests' mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' From 4bca56c3a2994d50e56e437c245e761572fbefc3 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sun, 5 Aug 2018 19:25:50 +0900 Subject: [PATCH 40/84] Revert invalid fix --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e400f12eab9b..9d672dbb3a02 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,7 +27,7 @@ mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3r mx_dist_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' -mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, mxnet/python/mxnet/_cy2/*.so, mxnet/python/mxnet/_cy3/*.so' +mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests' mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' From e602d8822daf2dccb80e9da044415140ac28c1f3 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 7 Aug 2018 16:31:47 +0900 Subject: [PATCH 41/84] Revise docs --- docs/faq/env_var.md | 3 +++ docs/install/index.md | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/faq/env_var.md b/docs/faq/env_var.md index f3677e5ada3e..de902231c7a4 100644 --- a/docs/faq/env_var.md +++ b/docs/faq/env_var.md @@ -150,6 +150,9 @@ When USE_PROFILER is enabled in Makefile or CMake, the following environments ca - If set to 0, MXNet fallbacks to the ctypes if importing the cython modules fails. - If set to 1, MXNet raises an error if importing the cython modules fails. +If cython modules are used, `mx.nd._internal.NDArrayBase` must be `mxnet._cy3.ndarray.NDArrayBase` for python 3 or `mxnet._cy2.ndarray.NDArrayBase` for python 2. +If ctypes is used, it must be `mxnet._ctypes.ndarray.NDArrayBase`. + ## Other Environment Variables * MXNET_CUDNN_AUTOTUNE_DEFAULT diff --git a/docs/install/index.md b/docs/install/index.md index 3fc36f36bd7a..4be1bc7018b4 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -462,11 +462,14 @@ $ sudo apt-get install -y python-dev python-setuptools python-pip libgfortran3 **Step 2** Build cython modules (optional). ```bash -$ sudo apt-get install -y cython +$ pip install Cython $ make cython # You can set the python executable with `PYTHON` flag, e.g., make cython PYTHON=python3 ``` -Note that you can control the use of the cython modules at runtime via the environment variables `MXNET_ENABLE_CYTHON` and `MXNET_ENFORCE_CYTHON`. See [here](/faq/env_var.html) for details. +*MXNet* tries to use the cython modules unless the environment variable `MXNET_ENABLE_CYTHON` is set to `0`. +If loading the cython modules fails, the default behavior is falling back to ctypes without any warning. +To raise an exception at the failure, set the environment variable `MXNET_ENFORCE_CYTHON` to `1`. +See [here](/faq/env_var.html) for more details. **Step 3** Install the MXNet Python binding. From 7dd3fa6a31b38d5bfa9e1c3251238422c0be15b2 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 7 Aug 2018 16:32:48 +0900 Subject: [PATCH 42/84] Revise check_cython --- ci/docker/runtime_functions.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 840407444bce..f8a59402c776 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -36,13 +36,21 @@ clean_repo() { check_cython() { set -ex local python_ver=$1 - if [ "$(echo -e 'import mxnet as mx\nprint(mx.nd._internal.NDArrayBase.__module__)' | python${python_ver})" != "mxnet._cy${python_ver}.ndarray" ]; then + local is_cython_used=$(python${python_ver} < Date: Tue, 7 Aug 2018 19:14:27 +0900 Subject: [PATCH 43/84] Temporaily use make instead of cmake for debugging --- Jenkinsfile | 4 ++- ci/docker/runtime_functions.sh | 47 +++++++++++++++++++++++----------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9d672dbb3a02..14c25b96171b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,7 +27,9 @@ mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3r mx_dist_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' -mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' +//mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' +// TMP for debugging +mx_cmake_lib_cython = mx_dist_lib_cython // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests' mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index f8a59402c776..c62f4c973a00 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -537,21 +537,38 @@ build_ubuntu_gpu_cmake_mkldnn() { build_ubuntu_gpu_cmake() { set -ex - cd /work/build - cmake \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DUSE_CUDA=1 \ - -DUSE_CUDNN=1 \ - -DUSE_MKLML_MKL=0 \ - -DUSE_MKLDNN=0 \ - -DUSE_DIST_KVSTORE=1 \ - -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ - /work/mxnet - - ninja -v - build_cython + # cd /work/build + # cmake \ + # -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + # -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + # -DUSE_CUDA=1 \ + # -DUSE_CUDNN=1 \ + # -DUSE_MKLML_MKL=0 \ + # -DUSE_MKLDNN=0 \ + # -DUSE_DIST_KVSTORE=1 \ + # -DCMAKE_BUILD_TYPE=Release \ + # -G Ninja \ + # /work/mxnet + + # ninja -v + # build_cython + + # TMP for debugging + export CC="ccache gcc" + export CXX="ccache g++" + make \ + DEV=1 \ + USE_CUDA=1 \ + USE_CUDNN=1 \ + USE_MKLML_MKL=0 \ + USE_MKLDNN=0 \ + USE_DIST_KVSTORE=1 \ + -j$(nproc) + + export CC="gcc" + export CXX="g++" + make cython PYTHON=python2 + make cython PYTHON=python3 } From 8cb0e620103936778be70dd852faf44be4a73588 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 7 Aug 2018 20:28:00 +0900 Subject: [PATCH 44/84] Temporal changes for debugging --- ci/docker/runtime_functions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index c62f4c973a00..a0026f0c110d 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -554,12 +554,14 @@ build_ubuntu_gpu_cmake() { # build_cython # TMP for debugging + build_ccache_wrappers export CC="ccache gcc" export CXX="ccache g++" make \ DEV=1 \ USE_CUDA=1 \ USE_CUDNN=1 \ + USE_CUDA_PATH=/usr/local/cuda \ USE_MKLML_MKL=0 \ USE_MKLDNN=0 \ USE_DIST_KVSTORE=1 \ From 94f54270f7304b40216815d842f92341829d9514 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 7 Aug 2018 21:09:21 +0900 Subject: [PATCH 45/84] Temporal changes for debugging --- ci/docker/runtime_functions.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index a0026f0c110d..6d54366eb920 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -554,9 +554,6 @@ build_ubuntu_gpu_cmake() { # build_cython # TMP for debugging - build_ccache_wrappers - export CC="ccache gcc" - export CXX="ccache g++" make \ DEV=1 \ USE_CUDA=1 \ @@ -567,8 +564,6 @@ build_ubuntu_gpu_cmake() { USE_DIST_KVSTORE=1 \ -j$(nproc) - export CC="gcc" - export CXX="g++" make cython PYTHON=python2 make cython PYTHON=python3 } From f6cee0357d2c5104462ea5940dd3f3154746d2e8 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 7 Aug 2018 23:11:46 +0900 Subject: [PATCH 46/84] Temporaily use ctypes instead of cython modules for debugging --- Jenkinsfile | 4 +-- ci/docker/runtime_functions.sh | 51 +++++++++++++--------------------- 2 files changed, 20 insertions(+), 35 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 14c25b96171b..9d672dbb3a02 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,9 +27,7 @@ mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3r mx_dist_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' -//mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' -// TMP for debugging -mx_cmake_lib_cython = mx_dist_lib_cython +mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests' mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 6d54366eb920..d34353d65311 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -537,35 +537,21 @@ build_ubuntu_gpu_cmake_mkldnn() { build_ubuntu_gpu_cmake() { set -ex - # cd /work/build - # cmake \ - # -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - # -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - # -DUSE_CUDA=1 \ - # -DUSE_CUDNN=1 \ - # -DUSE_MKLML_MKL=0 \ - # -DUSE_MKLDNN=0 \ - # -DUSE_DIST_KVSTORE=1 \ - # -DCMAKE_BUILD_TYPE=Release \ - # -G Ninja \ - # /work/mxnet - - # ninja -v - # build_cython - - # TMP for debugging - make \ - DEV=1 \ - USE_CUDA=1 \ - USE_CUDNN=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_MKLML_MKL=0 \ - USE_MKLDNN=0 \ - USE_DIST_KVSTORE=1 \ - -j$(nproc) - - make cython PYTHON=python2 - make cython PYTHON=python3 + cd /work/build + cmake \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DUSE_CUDA=1 \ + -DUSE_CUDNN=1 \ + -DUSE_MKLML_MKL=0 \ + -DUSE_MKLDNN=0 \ + -DUSE_DIST_KVSTORE=1 \ + -DCMAKE_BUILD_TYPE=Release \ + -G Ninja \ + /work/mxnet + + ninja -v + build_cython } @@ -687,9 +673,10 @@ unittest_ubuntu_python3_gpu_cython() { # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 - export MXNET_ENABLE_CYTHON=1 - export MXNET_ENFORCE_CYTHON=1 - check_cython 3 + # export MXNET_ENABLE_CYTHON=1 + # export MXNET_ENFORCE_CYTHON=1 + # check_cython 3 + export MXNET_ENABLE_CYTHON=0 nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } From c505bf0020063ccea7a23c7c7ba45ee864b006b3 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Wed, 8 Aug 2018 01:02:54 +0900 Subject: [PATCH 47/84] Temporaily disable ccache for debugging --- ci/docker/runtime_functions.sh | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index d34353d65311..cdf46ecd9f87 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -379,10 +379,10 @@ build_ubuntu_cpu_cmake_debug() { build_ubuntu_cpu_clang39() { set -ex - export CXX=clang++-3.9 + export CXX=clang++-3.9 export CC=clang-3.9 - build_ccache_wrappers - make \ + build_ccache_wrappers + make \ USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ USE_OPENMP=0 \ @@ -539,8 +539,6 @@ build_ubuntu_gpu_cmake() { set -ex cd /work/build cmake \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DUSE_CUDA=1 \ -DUSE_CUDNN=1 \ -DUSE_MKLML_MKL=0 \ @@ -673,10 +671,9 @@ unittest_ubuntu_python3_gpu_cython() { # https://github.com/apache/incubator-mxnet/issues/10026 #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 - # export MXNET_ENABLE_CYTHON=1 - # export MXNET_ENFORCE_CYTHON=1 - # check_cython 3 - export MXNET_ENABLE_CYTHON=0 + export MXNET_ENABLE_CYTHON=1 + export MXNET_ENFORCE_CYTHON=1 + check_cython 3 nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } From 17f6c702e36d1994be5b78e42cb3c402b5d26b75 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Wed, 8 Aug 2018 11:45:07 +0900 Subject: [PATCH 48/84] Temporaily use make (DEV = 0) instead of cmake for debugging --- Jenkinsfile | 4 ++- ci/docker/runtime_functions.sh | 48 ++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9d672dbb3a02..14c25b96171b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,7 +27,9 @@ mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3r mx_dist_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' -mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' +//mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' +// TMP for debugging +mx_cmake_lib_cython = mx_dist_lib_cython // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests' mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index cdf46ecd9f87..6d54366eb920 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -379,10 +379,10 @@ build_ubuntu_cpu_cmake_debug() { build_ubuntu_cpu_clang39() { set -ex - export CXX=clang++-3.9 + export CXX=clang++-3.9 export CC=clang-3.9 - build_ccache_wrappers - make \ + build_ccache_wrappers + make \ USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ USE_OPENMP=0 \ @@ -537,19 +537,35 @@ build_ubuntu_gpu_cmake_mkldnn() { build_ubuntu_gpu_cmake() { set -ex - cd /work/build - cmake \ - -DUSE_CUDA=1 \ - -DUSE_CUDNN=1 \ - -DUSE_MKLML_MKL=0 \ - -DUSE_MKLDNN=0 \ - -DUSE_DIST_KVSTORE=1 \ - -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ - /work/mxnet - - ninja -v - build_cython + # cd /work/build + # cmake \ + # -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + # -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + # -DUSE_CUDA=1 \ + # -DUSE_CUDNN=1 \ + # -DUSE_MKLML_MKL=0 \ + # -DUSE_MKLDNN=0 \ + # -DUSE_DIST_KVSTORE=1 \ + # -DCMAKE_BUILD_TYPE=Release \ + # -G Ninja \ + # /work/mxnet + + # ninja -v + # build_cython + + # TMP for debugging + make \ + DEV=1 \ + USE_CUDA=1 \ + USE_CUDNN=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_MKLML_MKL=0 \ + USE_MKLDNN=0 \ + USE_DIST_KVSTORE=1 \ + -j$(nproc) + + make cython PYTHON=python2 + make cython PYTHON=python3 } From 8ccc5f23ee4d4cc7e40e6d00c471a15d59ae1bed Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Wed, 8 Aug 2018 14:36:52 +0900 Subject: [PATCH 49/84] Temporaily disable cudnn for debugging --- Jenkinsfile | 4 +--- ci/docker/runtime_functions.sh | 44 ++++++++++++---------------------- 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 14c25b96171b..9d672dbb3a02 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,9 +27,7 @@ mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3r mx_dist_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' -//mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' -// TMP for debugging -mx_cmake_lib_cython = mx_dist_lib_cython +mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests' mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 6d54366eb920..08a98fbec276 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -537,35 +537,21 @@ build_ubuntu_gpu_cmake_mkldnn() { build_ubuntu_gpu_cmake() { set -ex - # cd /work/build - # cmake \ - # -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - # -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - # -DUSE_CUDA=1 \ - # -DUSE_CUDNN=1 \ - # -DUSE_MKLML_MKL=0 \ - # -DUSE_MKLDNN=0 \ - # -DUSE_DIST_KVSTORE=1 \ - # -DCMAKE_BUILD_TYPE=Release \ - # -G Ninja \ - # /work/mxnet - - # ninja -v - # build_cython - - # TMP for debugging - make \ - DEV=1 \ - USE_CUDA=1 \ - USE_CUDNN=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_MKLML_MKL=0 \ - USE_MKLDNN=0 \ - USE_DIST_KVSTORE=1 \ - -j$(nproc) - - make cython PYTHON=python2 - make cython PYTHON=python3 + cd /work/build + cmake \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DUSE_CUDA=1 \ + -DUSE_CUDNN=0 \ + -DUSE_MKLML_MKL=0 \ + -DUSE_MKLDNN=0 \ + -DUSE_DIST_KVSTORE=1 \ + -DCMAKE_BUILD_TYPE=Release \ + -G Ninja \ + /work/mxnet + + ninja -v + build_cython } From 786b2fd04f972275f88b2a38741dfea6a21aafa8 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Wed, 8 Aug 2018 16:35:34 +0900 Subject: [PATCH 50/84] Restore temporal changes --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 08a98fbec276..f8a59402c776 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -542,7 +542,7 @@ build_ubuntu_gpu_cmake() { -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DUSE_CUDA=1 \ - -DUSE_CUDNN=0 \ + -DUSE_CUDNN=1 \ -DUSE_MKLML_MKL=0 \ -DUSE_MKLDNN=0 \ -DUSE_DIST_KVSTORE=1 \ From e02cc4f43c9b81798a6247ed9823c915dc698070 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 9 Aug 2018 12:10:58 +0900 Subject: [PATCH 51/84] Temporarily disable coverage report --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index f8a59402c776..3be9862b5253 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -676,7 +676,7 @@ unittest_ubuntu_python3_gpu_cython() { export MXNET_ENABLE_CYTHON=1 export MXNET_ENFORCE_CYTHON=1 check_cython 3 - nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu + nosetests-3.4 --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } unittest_ubuntu_python3_gpu_nocudnn() { From aa64c42837bdbf9ec4c3c2a49f8becd100cca1d5 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 9 Aug 2018 13:01:11 +0900 Subject: [PATCH 52/84] Adapt to Jenkinsfile_utils --- ci/Jenkinsfile_utils.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy index dfa2519bd0e5..1456372d8f5d 100644 --- a/ci/Jenkinsfile_utils.groovy +++ b/ci/Jenkinsfile_utils.groovy @@ -57,20 +57,20 @@ def init_git_win() { } // pack libraries for later use -def pack_lib(name, libs) { +def pack_lib(name, libs=mx_lib) { sh """ echo "Packing ${libs} into ${name}" -echo ${libs} | sed -e 's/,/ /g' | xargs md5sum +for i in \$(echo ${libs} | sed -e 's/,/ /g'); do md5sum \$i; done """ stash includes: libs, name: name } // unpack libraries saved before -def unpack_lib(name, libs) { +def unpack_lib(name, libs=mx_lib) { unstash name sh """ echo "Unpacked ${libs} from ${name}" -echo ${libs} | sed -e 's/,/ /g' | xargs md5sum +for i in \$(echo ${libs} | sed -e 's/,/ /g'); do md5sum \$i; done """ } From c90570c337c37f60a491eaef8dbe50da772f40bf Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 9 Aug 2018 14:06:56 +0900 Subject: [PATCH 53/84] Adapt to Jenkinsfile_utils (cont.) --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 52c1419d56da..e556482058b3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,7 +46,7 @@ def python2_ut(docker_container_name) { def python2_ut_cython(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - docker_run(docker_container_name, 'unittest_ubuntu_python2_cpu_cython', false) + utils.docker_run(docker_container_name, 'unittest_ubuntu_python2_cpu_cython', false) } } @@ -81,7 +81,7 @@ def python3_gpu_ut(docker_container_name) { def python3_gpu_ut_cython(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu_cython', true) + utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu_cython', true) } } From a2bb2e1939d0788ef1455c18ea3f8981ee427d61 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 9 Aug 2018 14:46:04 +0900 Subject: [PATCH 54/84] Restore unrelated changes --- ci/Jenkinsfile_utils.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy index 1456372d8f5d..47bde583203e 100644 --- a/ci/Jenkinsfile_utils.groovy +++ b/ci/Jenkinsfile_utils.groovy @@ -57,7 +57,7 @@ def init_git_win() { } // pack libraries for later use -def pack_lib(name, libs=mx_lib) { +def pack_lib(name, libs) { sh """ echo "Packing ${libs} into ${name}" for i in \$(echo ${libs} | sed -e 's/,/ /g'); do md5sum \$i; done @@ -66,7 +66,7 @@ for i in \$(echo ${libs} | sed -e 's/,/ /g'); do md5sum \$i; done } // unpack libraries saved before -def unpack_lib(name, libs=mx_lib) { +def unpack_lib(name, libs) { unstash name sh """ echo "Unpacked ${libs} from ${name}" From d89de2ff75c2d512a387b9ac7775694a46273d6c Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Fri, 10 Aug 2018 15:13:15 +0900 Subject: [PATCH 55/84] Restore temporal changes --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 3be9862b5253..f8a59402c776 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -676,7 +676,7 @@ unittest_ubuntu_python3_gpu_cython() { export MXNET_ENABLE_CYTHON=1 export MXNET_ENFORCE_CYTHON=1 check_cython 3 - nosetests-3.4 --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu + nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } unittest_ubuntu_python3_gpu_nocudnn() { From 927ca1a44f0e942d50de9cc65a7b900c024f7544 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Mon, 5 Nov 2018 15:58:40 +0900 Subject: [PATCH 56/84] Resolving conflict --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 84ef94f41196..88cee8c110b9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,6 +22,7 @@ // mxnet libraries mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' +mx_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // for scala build, need to pass extra libs when run with dist_kvstore mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' mx_dist_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' From d02b89bdffcfeb3cf837ffe32ced7a91de9f5a8f Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 8 Nov 2018 12:06:35 +0900 Subject: [PATCH 57/84] Test with the cmake build is removed --- Jenkinsfile | 11 +++++++++-- ci/docker/runtime_functions.sh | 3 +++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 88cee8c110b9..9492e04a929e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -34,7 +34,7 @@ mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-c mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so' -mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/lenet, build/cpp-package/example/alexnet, build/cpp-package/example/googlenet, build/cpp-package/example/lenet_with_mxdataiter, build/cpp-package/example/resnet, build/cpp-package/example/mlp, build/cpp-package/example/mlp_cpu, build/cpp-package/example/mlp_gpu, build/cpp-package/example/test_score, build/cpp-package/example/test_optimizer' +mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/lenet, build/cpp-package/example/alexnet, build/cpp-package/example/googlenet, build/cpp-package/example/lenet_with_mxdataiter, build/cpp-package/example/resnet, build/cpp-package/example/mlp, build/cpp-package/example/mlp_cpu, build/cpp-package/example/mlp_gpu, build/cpp-package/example/test_score, build/cpp-package/example/test_optimizer' mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/mlp_cpu' // timeout in minutes @@ -544,7 +544,14 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/ut-python3-gpu') { try { - utils.unpack_and_init('cmake_gpu', mx_cmake_lib_cython, true) + /* + * CMake build with llvm openmp causes a segmentation fault. + * We can restore this if the issue is addressed (#12160). + * + * utils.unpack_and_init('cmake_gpu', mx_cmake_lib_cython, true) + */ + + utils.unpack_and_init('gpu', mx_lib_cython, true) python3_gpu_ut_cython('ubuntu_gpu') utils.publish_test_coverage() } finally { diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 6fac2d11741b..72f567d5f5f0 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -630,6 +630,9 @@ build_ubuntu_gpu_cuda91_cudnn7() { USE_DIST_KVSTORE=1 \ CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ -j$(nproc) + + make cython PYTHON=python2 + make cython PYTHON=python3 } build_ubuntu_amalgamation() { From 17a702c4c6816ce501095b068b89dd96c28b2b1e Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 8 Nov 2018 12:33:53 +0900 Subject: [PATCH 58/84] Add MXNET_ENABLE_CYTHON=0 to tensorrt test --- ci/docker/runtime_functions.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 6893e1c30689..e4ba922d27e3 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -804,6 +804,7 @@ unittest_ubuntu_tensorrt_gpu() { export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 export LD_LIBRARY_PATH=/work/mxnet/lib:$LD_LIBRARY_PATH export CUDNN_VERSION=7.0.3 + export MXNET_ENABLE_CYTHON=0 python tests/python/tensorrt/lenet5_train.py nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_trt_gpu.xml --verbose --nocapture tests/python/tensorrt/ } From d5f6fe69339169d187ae5df117b157d41a0881dd Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 8 Nov 2018 14:44:02 +0900 Subject: [PATCH 59/84] Fix typo --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index e4ba922d27e3..316feb282366 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -723,7 +723,7 @@ unittest_ubuntu_python2_cpu_cython() { check_cython 2 nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train - nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/pytho + nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization } unittest_ubuntu_python2_cpu() { From fdf344ff8ea269277e0b44e7f4efd51d8988cf1d Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 13 Nov 2018 15:41:38 +0900 Subject: [PATCH 60/84] Trigger CI From e81dba98d43207d6de59412ecd66485ba4e4f6d2 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 22 Nov 2018 08:16:43 +0900 Subject: [PATCH 61/84] Trigger CI From fcf5986bfe975a383336fb83424c9103b7889f4d Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Fri, 23 Nov 2018 12:01:18 +0900 Subject: [PATCH 62/84] Adapt to Jenkinsfile refactoring --- Jenkinsfile | 36 +++++++-------------------------- ci/jenkins/Jenkins_steps.groovy | 27 +++++++++++++++++++------ 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3b9da4c18cf1..a7e3dbd9bcc0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -48,23 +48,20 @@ // mxnet libraries mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' -mx_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // Python wheels mx_pip = 'build/*.whl' // for scala build, need to pass extra libs when run with dist_kvstore mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' -mx_dist_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' -mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests' mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so' -mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/lenet, build/cpp-package/example/alexnet, build/cpp-package/example/googlenet, build/cpp-package/example/lenet_with_mxdataiter, build/cpp-package/example/resnet, build/cpp-package/example/mlp, build/cpp-package/example/mlp_cpu, build/cpp-package/example/mlp_gpu, build/cpp-package/example/test_score, build/cpp-package/example/test_optimizer' +mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/lenet, build/cpp-package/example/alexnet, build/cpp-package/example/googlenet, build/cpp-package/example/lenet_with_mxdataiter, build/cpp-package/example/resnet, build/cpp-package/example/mlp, build/cpp-package/example/mlp_cpu, build/cpp-package/example/mlp_gpu, build/cpp-package/example/test_score, build/cpp-package/example/test_optimizer' mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/mlp_cpu' // timeout in minutes @@ -79,12 +76,6 @@ def python2_ut(docker_container_name) { } } -def python2_ut_cython(docker_container_name) { - timeout(time: max_time, unit: 'MINUTES') { - utils.docker_run(docker_container_name, 'unittest_ubuntu_python2_cpu_cython', false) - } -} - // Python 3 def python3_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { @@ -121,12 +112,6 @@ def python3_gpu_ut(docker_container_name) { } } -def python3_gpu_ut_cython(docker_container_name) { - timeout(time: max_time, unit: 'MINUTES') { - utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu_cython', true) - } -} - // Python 3 NOCUDNN def python3_gpu_ut_nocudnn(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { @@ -226,7 +211,7 @@ core_logic: { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false) - utils.pack_lib('cpu', mx_dist_lib_cython, true) + utils.pack_lib('cpu', mx_dist_lib, true) } } } @@ -387,7 +372,7 @@ core_logic: { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false) - utils.pack_lib('cmake_gpu', mx_cmake_lib_cython, true) + utils.pack_lib('cmake_gpu', mx_cmake_lib, true) } } } @@ -512,8 +497,8 @@ core_logic: { node(NODE_LINUX_CPU) { ws('workspace/ut-python2-cpu') { try { - utils.unpack_and_init('cpu', mx_lib_cython, true) - python2_ut_cython('ubuntu_cpu') + utils.unpack_and_init('cpu', mx_lib, true) + python2_ut('ubuntu_cpu') utils.publish_test_coverage() } finally { utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_cpu_unittest.xml') @@ -575,15 +560,8 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/ut-python3-gpu') { try { - /* - * CMake build with llvm openmp causes a segmentation fault. - * We can restore this if the issue is addressed (#12160). - * - * utils.unpack_and_init('cmake_gpu', mx_cmake_lib_cython, true) - */ - - utils.unpack_and_init('gpu', mx_lib_cython, true) - python3_gpu_ut_cython('ubuntu_gpu') + utils.unpack_and_init('gpu', mx_lib, true) + python3_gpu_ut('ubuntu_gpu') utils.publish_test_coverage() } finally { utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_gpu.xml') diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index f48a26737308..a533fe7f3fdc 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -24,14 +24,17 @@ utils = load('ci/Jenkinsfile_utils.groovy') // mxnet libraries mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' +mx_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // Python wheels mx_pip = 'build/*.whl' // for scala build, need to pass extra libs when run with dist_kvstore mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' +mx_dist_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' +mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests' mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' @@ -48,6 +51,12 @@ def python2_ut(docker_container_name) { } } +def python2_ut_cython(docker_container_name) { + timeout(time: max_time, unit: 'MINUTES') { + utils.docker_run(docker_container_name, 'unittest_ubuntu_python2_cpu_cython', false) + } +} + // Python 3 def python3_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { @@ -91,6 +100,12 @@ def python3_gpu_ut_nocudnn(docker_container_name) { } } +def python3_gpu_ut_cython(docker_container_name) { + timeout(time: max_time, unit: 'MINUTES') { + utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu_cython', true) + } +} + //------------------------------------------------------------------------------------------ def compile_unix_cpu_openblas() { @@ -100,7 +115,7 @@ def compile_unix_cpu_openblas() { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false) - utils.pack_lib('cpu', mx_dist_lib, true) + utils.pack_lib('cpu', mx_dist_lib_cython, true) } } } @@ -198,7 +213,7 @@ def compile_unix_cmake_gpu() { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false) - utils.pack_lib('cmake_gpu', mx_cmake_lib, true) + utils.pack_lib('cmake_gpu', mx_cmake_lib_cython, true) } } } @@ -500,8 +515,8 @@ def test_unix_python2_cpu() { node(NODE_LINUX_CPU) { ws('workspace/ut-python2-cpu') { try { - utils.unpack_and_init('cpu', mx_lib, true) - python2_ut('ubuntu_cpu') + utils.unpack_and_init('cpu', mx_lib_cython, true) + python2_ut_cython('ubuntu_cpu') utils.publish_test_coverage() } finally { utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_cpu_unittest.xml') @@ -585,8 +600,8 @@ def test_unix_python3_gpu() { node(NODE_LINUX_GPU) { ws('workspace/ut-python3-gpu') { try { - utils.unpack_and_init('gpu', mx_lib, true) - python3_gpu_ut('ubuntu_gpu') + utils.unpack_and_init('gpu', mx_lib_cython, true) + python3_gpu_ut_cython('ubuntu_gpu') utils.publish_test_coverage() } finally { utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_gpu.xml') From 064f56fd1a888a27481772c3135c9f702476d139 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Fri, 23 Nov 2018 12:13:58 +0900 Subject: [PATCH 63/84] Adapt to Jenkinsfile refactoring (cont.) --- ci/docker/runtime_functions.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 596f4a40cb53..b07a0802eadc 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -722,9 +722,9 @@ unittest_ubuntu_python2_cpu_cython() { export MXNET_ENABLE_CYTHON=1 export MXNET_ENFORCE_CYTHON=1 check_cython 2 - nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest - nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train - nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization + nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest + nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train + nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization } unittest_ubuntu_python2_cpu() { From b30ba001b0dc9fb30e88724b019ac3195485e985 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Fri, 23 Nov 2018 14:19:59 +0900 Subject: [PATCH 64/84] Trigger CI From 9965e4cea503fc1c29e0f4539f7df44477a99a1f Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Fri, 23 Nov 2018 16:57:40 +0900 Subject: [PATCH 65/84] Trigger CI From 371a8c6078f57ffc1ef94935fce981bbf2d407c6 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Fri, 23 Nov 2018 20:17:10 +0900 Subject: [PATCH 66/84] Stash missing cython modules --- ci/jenkins/Jenkins_steps.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index a533fe7f3fdc..4c33b8b55d14 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -40,7 +40,7 @@ mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-c mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so' -mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*' +mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, build/cpp-package/example/*' mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*' // Python unittest for CPU From 9adca4da760d378ab23ff4dcbf881fb075a8811a Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Mon, 3 Dec 2018 13:31:15 +0900 Subject: [PATCH 67/84] Trigger CI From 10d817cea15ec0b3253f3b78a30052f764931026 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 11 Dec 2018 23:52:25 +0900 Subject: [PATCH 68/84] CMake build of cython modules without unit tests --- CMakeLists.txt | 10 +++++++++ ci/docker/runtime_functions.sh | 18 +--------------- ci/jenkins/Jenkins_steps.groovy | 6 ++++++ cmake/BuildCythonModules.cmake | 38 +++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 17 deletions(-) create mode 100644 cmake/BuildCythonModules.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b8bbd2e0272..ecba04deebe5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,7 @@ mxnet_option(USE_SIGNAL_HANDLER "Print stack traces on segfaults." OFF) mxnet_option(USE_TENSORRT "Enable infeference optimization with TensorRT." OFF) mxnet_option(USE_ASAN "Enable Clang/GCC ASAN sanitizers." OFF) mxnet_option(ENABLE_TESTCOVERAGE "Enable compilation with test coverage metric output" OFF) +mxnet_option(BUILD_CYTHON_MODULES "Build cython modules." OFF) message(STATUS "CMAKE_SYSTEM_NAME ${CMAKE_SYSTEM_NAME}") if(USE_CUDA AND NOT USE_OLDCMAKECUDA) @@ -783,3 +784,12 @@ endif() set(LINT_DIRS "include src plugin cpp-package tests") set(EXCLUDE_PATH "src/operator/contrib/ctc_include") add_custom_target(mxnet_lint COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} -DLINT_DIRS=${LINT_DIRS} -DPROJECT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -DPROJECT_NAME=mxnet -DEXCLUDE_PATH=${EXCLUDE_PATH} -P ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/dmlc-core/cmake/lint.cmake) + +if(BUILD_CYTHON_MODULES) + include(cmake/BuildCythonModules.cmake) + add_cython_modules(2) # Build cython module for python2 if python2 is found + add_cython_modules(3) # Build cython module for python3 if python3 is found + if((NOT ${PYTHON2_FOUND}) AND (NOT ${PYTHON3_FOUND})) + message(FATAL_ERROR "No python interpreter found to build cython modules") + endif() +endif() diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index b07a0802eadc..4b9192bbfa80 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -87,22 +87,6 @@ build_ccache_wrappers() { export CXX=`pwd`/cxx } -build_cython() { - set -ex - pushd . - - PYTHON_DIR=${1:-/work/mxnet/python} - BUILD_DIR=${2:-/work/build} - - export MXNET_LIBRARY_PATH=${BUILD_DIR}/libmxnet.so - - cd ${PYTHON_DIR} - python2 setup.py build_ext --inplace --with-cython - python3 setup.py build_ext --inplace --with-cython - - popd -} - build_wheel() { set -ex @@ -693,11 +677,11 @@ build_ubuntu_gpu_cmake() { -DCMAKE_BUILD_TYPE=Release \ -DCUDA_ARCH_NAME=Manual \ -DCUDA_ARCH_BIN=$CI_CMAKE_CUDA_ARCH_BIN \ + -DBUILD_CYTHON_MODULES \ -G Ninja \ /work/mxnet ninja -v - build_cython } build_ubuntu_blc() { diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 4c33b8b55d14..83d8441321e6 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -600,6 +600,12 @@ def test_unix_python3_gpu() { node(NODE_LINUX_GPU) { ws('workspace/ut-python3-gpu') { try { + /* + * CMake build with llvm openmp causes a segmentation fault. + * We can restore this if the issue is addressed (#12160). + * + * utils.unpack_and_init('cmake_gpu', mx_cmake_lib_cython, true) + */ utils.unpack_and_init('gpu', mx_lib_cython, true) python3_gpu_ut_cython('ubuntu_gpu') utils.publish_test_coverage() diff --git a/cmake/BuildCythonModules.cmake b/cmake/BuildCythonModules.cmake new file mode 100644 index 000000000000..d2c3a46f1a71 --- /dev/null +++ b/cmake/BuildCythonModules.cmake @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(add_cython_modules python_version) + unset(PYTHON_EXECUTABLE CACHE) + set(PYTHONINTERP_FOUND FALSE) + find_package(PythonInterp ${python_version} EXACT) + if(PYTHONINTERP_FOUND) + find_program(CYTHON_EXECUTABLE NAMES cython) + if(CYTHON_EXECUTABLE) + add_custom_command(COMMAND ${CMAKE_COMMAND} POST_BUILD + -E env MXNET_LIBRARY_PATH=${CMAKE_BINARY_DIR}/libmxnet.so + ${PYTHON_EXECUTABLE} setup.py build_ext --inplace --with-cython + TARGET mxnet + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}/python") + message("-- Cython modules for python${python_version} will be built") + set(PYTHON${python_version}_FOUND 1 PARENT_SCOPE) + else() + message(FATAL_ERROR "-- Cython not found") + endif() + else() + set(PYTHON${python_version}_FOUND 0 PARENT_SCOPE) + endif() +endfunction() From adce15e25afdf3a47f5dc26bda9db7371aa4882b Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Wed, 12 Dec 2018 00:38:28 +0900 Subject: [PATCH 69/84] Fix typo --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 4b9192bbfa80..577e82583f36 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -677,7 +677,7 @@ build_ubuntu_gpu_cmake() { -DCMAKE_BUILD_TYPE=Release \ -DCUDA_ARCH_NAME=Manual \ -DCUDA_ARCH_BIN=$CI_CMAKE_CUDA_ARCH_BIN \ - -DBUILD_CYTHON_MODULES \ + -DBUILD_CYTHON_MODULES=1 \ -G Ninja \ /work/mxnet From d7bdf6340716349edd482281c78509f034bf3a40 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Wed, 12 Dec 2018 22:12:27 +0900 Subject: [PATCH 70/84] Trigger CI From 550ae4a945445c1b91dc75500f90ff1ad544439c Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Fri, 18 Jan 2019 19:44:32 +0900 Subject: [PATCH 71/84] Fix a mistake introduced in merging process --- ci/docker/runtime_functions.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index f7eed3bdd8c1..eb28151bfe2e 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -386,6 +386,8 @@ build_ubuntu_cpu_openblas() { USE_MKLDNN=0 \ USE_DIST_KVSTORE=1 \ -j$(nproc) + make cython PYTHON=python2 + make cython PYTHON=python3 } build_ubuntu_cpu_mkl() { @@ -401,11 +403,6 @@ build_ubuntu_cpu_mkl() { USE_INTEL_PATH=/opt/intel \ USE_DIST_KVSTORE=1 \ -j$(nproc) - - export CC="gcc" - export CXX="g++" - make cython PYTHON=python2 - make cython PYTHON=python3 } build_ubuntu_cpu_cmake_debug() { From 2253b766f971e16a55d20628318f95d27c68ebdc Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Tue, 5 Mar 2019 08:58:57 -0800 Subject: [PATCH 72/84] trigger test --- python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/setup.py b/python/setup.py index fc72b26d7b02..c1857c39796f 100644 --- a/python/setup.py +++ b/python/setup.py @@ -97,6 +97,7 @@ def config_cython(): libraries=libraries, extra_link_args=extra_link_args, language="c++")) + # If `force=True` is not used and you cythonize the modules for python2 and python3 # successively, you need to delete `mxnet/cython/ndarray.cpp` after the first cythonize. return cythonize(ret, force=True) From 95f42bdf8bbb9e98d0052c2e5bf1231e0372fe00 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Sun, 17 Mar 2019 12:58:39 -0700 Subject: [PATCH 73/84] Update Jenkinsfile_utils.groovy From f42fd0a6b7eb2cd353cfbe80ba70af96e2d238df Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 11 Apr 2019 01:22:24 +0900 Subject: [PATCH 74/84] Trigger CI From e81093a3c26b4484dc63a30b5e6ae56f7086c6df Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 11 Apr 2019 01:24:02 +0900 Subject: [PATCH 75/84] Trigger CI From 9c836bc985ae5b0ce394a7d2e04600139c3b0028 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 13 Apr 2019 12:25:58 +0900 Subject: [PATCH 76/84] Trigger CI From d5de9c48fdeef1f61892f9d84f8af7cc48dd3d9c Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Mon, 6 May 2019 22:27:20 +0900 Subject: [PATCH 77/84] Trigger tests From d8c435c43805ccec7e76fddf7deec768722fea96 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 9 May 2019 14:05:53 +0900 Subject: [PATCH 78/84] Trigger tests From e30083421528111c9ff245fe5f444f75244d237e Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 14 May 2019 15:11:39 +0900 Subject: [PATCH 79/84] Trigger tests From 8dbc12ffe0b5f8bf8a1a1f316643f50daffd2bb0 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Tue, 14 May 2019 23:03:41 +0900 Subject: [PATCH 80/84] Trigger tests From 10bc51e554f9180b303ff26eddb308fd39a9a4eb Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 16 May 2019 17:39:08 +0900 Subject: [PATCH 81/84] Trigger tests From a6005c75e7a7f4d4337a42d014d641b32748493d Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Fri, 17 May 2019 23:21:23 +0900 Subject: [PATCH 82/84] Trigger tests From db80ee74b1f3361ac2feaa9e8756fba4041c13ac Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Sat, 18 May 2019 11:25:56 +0900 Subject: [PATCH 83/84] Trigger tests From 1443088b0735173caf61ffc781cc28ed941344c6 Mon Sep 17 00:00:00 2001 From: Deokjae Lee Date: Thu, 23 May 2019 17:05:55 +0900 Subject: [PATCH 84/84] Trigger tests