diff --git a/CMakeLists.txt b/CMakeLists.txt index 933c083ab220..2142a09d6d2e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,7 @@ mxnet_option(USE_TENSORRT "Enable infeference optimization with TensorRT mxnet_option(USE_ASAN "Enable Clang/GCC ASAN sanitizers." OFF) mxnet_option(ENABLE_TESTCOVERAGE "Enable compilation with test coverage metric output" OFF) mxnet_option(USE_INT64_TENSOR_SIZE "Use int64_t to represent the total number of elements in a tensor" OFF) +mxnet_option(BUILD_CYTHON_MODULES "Build cython modules." OFF) message(STATUS "CMAKE_CROSSCOMPILING ${CMAKE_CROSSCOMPILING}") message(STATUS "CMAKE_HOST_SYSTEM_PROCESSOR ${CMAKE_HOST_SYSTEM_PROCESSOR}") @@ -836,3 +837,12 @@ endif() set(LINT_DIRS "include src plugin cpp-package tests") set(EXCLUDE_PATH "src/operator/contrib/ctc_include") add_custom_target(mxnet_lint COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} -DLINT_DIRS=${LINT_DIRS} -DPROJECT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -DPROJECT_NAME=mxnet -DEXCLUDE_PATH=${EXCLUDE_PATH} -P ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/dmlc-core/cmake/lint.cmake) + +if(BUILD_CYTHON_MODULES) + include(cmake/BuildCythonModules.cmake) + add_cython_modules(2) # Build cython module for python2 if python2 is found + add_cython_modules(3) # Build cython module for python3 if python3 is found + if((NOT ${PYTHON2_FOUND}) AND (NOT ${PYTHON3_FOUND})) + message(FATAL_ERROR "No python interpreter found to build cython modules") + endif() +endif() diff --git a/Makefile b/Makefile index 3dbad1b8b7db..4a83d8c32d58 100644 --- a/Makefile +++ b/Makefile @@ -612,7 +612,7 @@ doxygen: # Cython build cython: - cd python; python setup.py build_ext --inplace --with-cython + cd python; $(PYTHON) setup.py build_ext --inplace --with-cython cython2: cd python; python2 setup.py build_ext --inplace --with-cython diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy index 38cc0b927c43..b8117f39377a 100644 --- a/ci/Jenkinsfile_utils.groovy +++ b/ci/Jenkinsfile_utils.groovy @@ -67,7 +67,7 @@ def pack_lib(name, libs, include_gcov_data = false) { sh returnStatus: true, script: """ set +e echo "Packing ${libs} into ${name}" -echo ${libs} | sed -e 's/,/ /g' | xargs md5sum +for i in \$(echo ${libs} | sed -e 's/,/ /g'); do md5sum \$i; done return 0 """ stash includes: libs, name: name @@ -86,7 +86,7 @@ def unpack_and_init(name, libs, include_gcov_data = false) { sh returnStatus: true, script: """ set +e echo "Unpacked ${libs} from ${name}" -echo ${libs} | sed -e 's/,/ /g' | xargs md5sum +for i in \$(echo ${libs} | sed -e 's/,/ /g'); do md5sum \$i; done return 0 """ if (include_gcov_data) { diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python.sh index 705d6abedbbd..2ca0cceec515 100755 --- a/ci/docker/install/ubuntu_python.sh +++ b/ci/docker/install/ubuntu_python.sh @@ -30,5 +30,5 @@ wget -nv https://bootstrap.pypa.io/get-pip.py python3 get-pip.py python2 get-pip.py -pip2 install nose cpplint==1.3.0 'numpy>1.16.0,<2.0.0' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 -pip3 install nose cpplint==1.3.0 pylint==2.3.1 'numpy>1.16.0,<2.0.0' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 +pip2 install nose cpplint==1.3.0 'numpy>1.16.0,<2.0.0' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 Cython==0.29.7 +pip3 install nose cpplint==1.3.0 pylint==2.3.1 'numpy>1.16.0,<2.0.0' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 Cython==0.29.7 diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 58e39efc2873..1ad67280617d 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -41,6 +41,26 @@ scala_prepare() { export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" } +check_cython() { + set -ex + local python_ver=$1 + local is_cython_used=$(python${python_ver} <nd).chandle = handle - (nd).cwritable = True - return nd + return _ndarray_cls(_ctypes.cast(handle, _ctypes.c_void_p), stype=stype) cdef class CachedOp: @@ -99,10 +96,21 @@ cdef class CachedOp: def __set__(self, value): self._set_handle(value) - def __init__(self, sym): - cdef unsigned long long ptr = sym.handle.value - CALL(MXCreateCachedOp( - (ptr), + def __init__(self, sym, flags=()): + cdef vector[string] s_flag_keys + cdef vector[string] s_flag_vals + if flags is not None: + for k, v in flags: + s_flag_keys.push_back(c_str(k)) + s_flag_vals.push_back(c_str(str(v))) + cdef vector[const char*] c_flag_keys = SVec2Ptr(s_flag_keys) + cdef vector[const char*] c_flag_vals = SVec2Ptr(s_flag_vals) + + CALL(MXCreateCachedOpEx( + (sym.handle.value), + len(flags), + CBeginPtr(c_flag_keys), + CBeginPtr(c_flag_vals), &self.chandle)) def __del__(self): @@ -115,6 +123,7 @@ cdef class CachedOp: cdef NDArrayHandle* p_output_vars cdef NDArrayHandle ret_handle cdef int num_output + cdef const int* p_output_stypes for i in args: ndvars.push_back((i).chandle) @@ -130,24 +139,24 @@ cdef class CachedOp: num_output = output_vars.size() if output_vars.size() == 0: - output_vars.resize(1) p_output_vars = NULL else: p_output_vars = &output_vars[0] - CALL(MXInvokeCachedOp( - (self).chandle, + CALL(MXInvokeCachedOpEx( + self.chandle, len(args), &ndvars[0] if ndvars.size() != 0 else NULL, &num_output, - &p_output_vars)) + &p_output_vars, + &p_output_stypes)) if original_output is not None: return original_output if num_output == 1: - return NewArray(p_output_vars[0]) + return NewArray(p_output_vars[0], p_output_stypes[0]) else: - return tuple(NewArray(p_output_vars[i]) for i in range(num_output)) + return [NewArray(p_output_vars[i], p_output_stypes[i]) for i in range(num_output)] def _imperative_invoke(handle, ndargs, keys, vals, out): @@ -161,6 +170,7 @@ def _imperative_invoke(handle, ndargs, keys, vals, out): cdef NDArrayHandle* p_output_vars cdef NDArrayHandle ret_handle cdef int num_output + cdef const int* p_output_stypes for i in ndargs: ndvars.push_back((i).chandle) @@ -180,7 +190,6 @@ def _imperative_invoke(handle, ndargs, keys, vals, out): num_output = output_vars.size() if output_vars.size() == 0: - output_vars.resize(1) p_output_vars = NULL else: p_output_vars = &output_vars[0] @@ -188,7 +197,7 @@ def _imperative_invoke(handle, ndargs, keys, vals, out): cdef vector[const char*] param_keys = SVec2Ptr(ckeys) cdef vector[const char*] param_vals = SVec2Ptr(cvals) - CALL(MXImperativeInvoke( + CALL(MXImperativeInvokeEx( chandle, ndvars.size(), &ndvars[0] if ndvars.size() != 0 else NULL, @@ -196,11 +205,12 @@ def _imperative_invoke(handle, ndargs, keys, vals, out): &p_output_vars, param_keys.size(), CBeginPtr(param_keys), - CBeginPtr(param_vals))) + CBeginPtr(param_vals), + &p_output_stypes)) if original_output is not None: return original_output if num_output == 1: - return NewArray(p_output_vars[0]) + return NewArray(p_output_vars[0], p_output_stypes[0]) else: - return tuple(NewArray(p_output_vars[i]) for i in range(num_output)) + return [NewArray(p_output_vars[i], p_output_stypes[i]) for i in range(num_output)] diff --git a/python/setup.py b/python/setup.py index 51c465b3eeba..e8545a5c7a90 100644 --- a/python/setup.py +++ b/python/setup.py @@ -20,8 +20,8 @@ from __future__ import absolute_import import os import sys +from setuptools import find_packages # This must precede distutils -from setuptools import find_packages # need to use distutils.core for correct placement of cython dll kwargs = {} if "--inplace" in sys.argv: @@ -70,7 +70,6 @@ def config_cython(): try: from Cython.Build import cythonize - # from setuptools.extension import Extension if sys.version_info >= (3, 0): subdir = "_cy3" else: @@ -81,20 +80,27 @@ def config_cython(): library_dirs = ['mxnet', '../build/Release', '../build'] libraries = ['libmxnet'] else: - library_dirs = None - libraries = None + library_dirs = [os.path.dirname(p) for p in LIB_PATH] + libraries = ['mxnet'] + # Default paths to libmxnet.so relative to the shared library file generated by cython. + # These precede LD_LIBRARY_PATH. + extra_link_args = ["-Wl,-rpath=$ORIGIN/..:$ORIGIN/../../../lib:$ORIGIN/../../../build"] for fn in os.listdir(path): if not fn.endswith(".pyx"): continue ret.append(Extension( - "mxnet/%s/.%s" % (subdir, fn[:-4]), + "mxnet.%s.%s" % (subdir, fn[:-4]), ["mxnet/cython/%s" % fn], include_dirs=["../include/", "../3rdparty/tvm/nnvm/include"], library_dirs=library_dirs, libraries=libraries, + extra_link_args=extra_link_args, language="c++")) - return cythonize(ret) + + # If `force=True` is not used and you cythonize the modules for python2 and python3 + # successively, you need to delete `mxnet/cython/ndarray.cpp` after the first cythonize. + return cythonize(ret, force=True) except ImportError: print("WARNING: Cython is not installed, will compile without cython module") return []