[MXNET-16] [DO NOT MERGE] Move submodules to 3rdparty

.gitmodules

@@ -1,17 +1,17 @@
 [submodule "mshadow"]
-	path = mshadow
+	path = 3rdparty/mshadow
 	url = https://github.com/dmlc/mshadow.git
 [submodule "dmlc-core"]
-	path = dmlc-core
+	path = 3rdparty/dmlc-core
 	url = https://github.com/dmlc/dmlc-core.git
 [submodule "ps-lite"]
-	path = ps-lite
+	path = 3rdparty/ps-lite
 	url = https://github.com/dmlc/ps-lite
 [submodule "nnvm"]
 	path = nnvm
 	url = https://github.com/dmlc/nnvm
 [submodule "dlpack"]
-	path = dlpack
+	path = 3rdparty/dlpack
 	url = https://github.com/dmlc/dlpack
 [submodule "3rdparty/openmp"]
 	path = 3rdparty/openmp

CMakeLists.txt

@@ -221,14 +221,14 @@ endif()
 
 if(USE_CUDA AND FIRST_CUDA)
   include(cmake/ChooseBlas.cmake)
-  include(mshadow/cmake/Utils.cmake)
+  include(3rdparty/mshadow/cmake/Utils.cmake)
   include(cmake/FirstClassLangCuda.cmake)
   include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
 else()
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/mshadow/cmake)
-    include(mshadow/cmake/mshadow.cmake)
-    include(mshadow/cmake/Utils.cmake)
-    include(mshadow/cmake/Cuda.cmake)
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/mshadow/cmake)
+    include(3rdparty/mshadow/cmake/mshadow.cmake)
+    include(3rdparty/mshadow/cmake/Utils.cmake)
+    include(3rdparty/mshadow/cmake/Cuda.cmake)
   else()
     include(mshadowUtils)
     include(Cuda)
@@ -243,16 +243,16 @@ foreach(var ${C_CXX_INCLUDE_DIRECTORIES})
 endforeach()
 
 include_directories("include")
-include_directories("mshadow")
+include_directories("3rdparty/mshadow")
 include_directories("3rdparty/cub")
 include_directories("nnvm/include")
 include_directories("nnvm/tvm/include")
-include_directories("dmlc-core/include")
-include_directories("dlpack/include")
+include_directories("3rdparty/dmlc-core/include")
+include_directories("3rdparty/dlpack/include")
 
 # commented out until PR goes through
-#if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/dlpack)
-#  add_subdirectory(dlpack)
+#if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/dlpack)
+#  add_subdirectory(3rdparty/dlpack)
 #endif()
 
 # Prevent stripping out symbols (operator registrations, for example)
@@ -390,12 +390,12 @@ if(USE_CUDNN AND USE_CUDA)
   endif()
 endif()
 
-if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/dmlc-core/cmake)
-  add_subdirectory("dmlc-core")
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/dmlc-core/cmake)
+  add_subdirectory("3rdparty/dmlc-core")
 endif()
 
-if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/mshadow/cmake)
-  add_subdirectory("mshadow")
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/mshadow/cmake)
+  add_subdirectory("3rdparty/mshadow")
 endif()
 FILE(GLOB_RECURSE SOURCE "src/*.cc" "src/*.h" "include/*.h")
 FILE(GLOB_RECURSE CUDA "src/*.cu" "src/*.cuh")
@@ -412,15 +412,15 @@ FILE(GLOB_RECURSE NNVMSOURCE
 list(APPEND SOURCE ${NNVMSOURCE})
 
 # add mshadow file
-FILE(GLOB_RECURSE MSHADOWSOURCE "mshadow/mshadow/*.h")
-FILE(GLOB_RECURSE MSHADOW_CUDASOURCE "mshadow/mshadow/*.cuh")
+FILE(GLOB_RECURSE MSHADOWSOURCE "3rdparty/mshadow/mshadow/*.h")
+FILE(GLOB_RECURSE MSHADOW_CUDASOURCE "3rdparty/mshadow/mshadow/*.cuh")
 list(APPEND SOURCE ${MSHADOWSOURCE})
 list(APPEND CUDA ${MSHADOW_CUDASOURCE})
 
 # add source group
 FILE(GLOB_RECURSE GROUP_SOURCE "src/*.cc" "nnvm/*.cc" "plugin/*.cc")
-FILE(GLOB_RECURSE GROUP_Include "src/*.h" "nnvm/*.h" "mshadow/mshadow/*.h" "plugin/*.h")
-FILE(GLOB_RECURSE GROUP_CUDA "src/*.cu" "src/*.cuh" "mshadow/mshadow/*.cuh" "plugin/*.cu"
+FILE(GLOB_RECURSE GROUP_Include "src/*.h" "nnvm/*.h" "3rdparty/mshadow/mshadow/*.h" "plugin/*.h")
+FILE(GLOB_RECURSE GROUP_CUDA "src/*.cu" "src/*.cuh" "3rdparty/mshadow/mshadow/*.cuh" "plugin/*.cu"
   "plugin/*.cuh" "3rdparty/cub/cub/*.cuh")
 assign_source_group("Source" ${GROUP_SOURCE})
 assign_source_group("Include" ${GROUP_Include})
@@ -593,8 +593,8 @@ if(USE_CUDA)
   endif()
 endif()
 if(USE_DIST_KVSTORE)
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/ps-lite/CMakeLists.txt)
-    add_subdirectory("ps-lite")
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/ps-lite/CMakeLists.txt)
+    add_subdirectory("3rdparty/ps-lite")
     list(APPEND pslite_LINKER_LIBS pslite protobuf)
     target_link_libraries(mxnet PUBLIC debug ${pslite_LINKER_LIBS_DEBUG})
     target_link_libraries(mxnet PUBLIC optimized ${pslite_LINKER_LIBS_RELEASE})
@@ -707,4 +707,4 @@ if(MSVC)
 endif()
 set(LINT_DIRS "include src plugin cpp-package tests")
 set(EXCLUDE_PATH "src/operator/contrib/ctc_include")
-add_custom_target(mxnet_lint COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} -DLINT_DIRS=${LINT_DIRS} -DPROJECT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -DPROJECT_NAME=mxnet -DEXCLUDE_PATH=${EXCLUDE_PATH} -P ${CMAKE_CURRENT_SOURCE_DIR}/dmlc-core/cmake/lint.cmake)
+add_custom_target(mxnet_lint COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} -DLINT_DIRS=${LINT_DIRS} -DPROJECT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -DPROJECT_NAME=mxnet -DEXCLUDE_PATH=${EXCLUDE_PATH} -P ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/dmlc-core/cmake/lint.cmake)

Jenkinsfile

@@ -21,11 +21,11 @@
 // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
 
 // mxnet libraries
-mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, dmlc-core/libdmlc.a, nnvm/lib/libnnvm.a'
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, nnvm/lib/libnnvm.a'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
-mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
-mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
-mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmklml_gnu.so, lib/libmkldnn.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, dmlc-core/libdmlc.a, nnvm/lib/libnnvm.a'
+mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
+mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
+mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmklml_gnu.so, lib/libmkldnn.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, nnvm/lib/libnnvm.a'
 // command to start a docker container
 docker_run = 'tests/ci_build/ci_build.sh'
 // timeout in minutes
@@ -278,7 +278,7 @@ try {
                 copy build_vc14_cpu\\Release\\libmxnet.dll pkg_vc14_cpu\\build
                 xcopy python pkg_vc14_cpu\\python /E /I /Y
                 xcopy include pkg_vc14_cpu\\include /E /I /Y
-                xcopy dmlc-core\\include pkg_vc14_cpu\\include /E /I /Y
+                xcopy 3rdparty\\dmlc-core\\include pkg_vc14_cpu\\include /E /I /Y
                 xcopy mshadow\\mshadow pkg_vc14_cpu\\include\\mshadow /E /I /Y
                 xcopy nnvm\\include pkg_vc14_cpu\\nnvm\\include /E /I /Y
                 del /Q *.7z
@@ -311,7 +311,7 @@ try {
               copy build_vc14_gpu\\libmxnet.dll pkg_vc14_gpu\\build
               xcopy python pkg_vc14_gpu\\python /E /I /Y
               xcopy include pkg_vc14_gpu\\include /E /I /Y
-              xcopy dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y
+              xcopy 3rdparty\\dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y
               xcopy mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y
               xcopy nnvm\\include pkg_vc14_gpu\\nnvm\\include /E /I /Y
               del /Q *.7z

LICENSE

@@ -219,15 +219,12 @@
     2. MXNet rcnn - For details, see, example/rcnn/LICENSE
     3. scala-package - For details, see, scala-package/LICENSE
     4. Warp-CTC - For details, see, src/operator/contrib/ctc_include/LICENSE
-    5. dlpack - For details, see, dlpack/LICENSE
-    6. dmlc-core - For details, see, dmlc-core/LICENSE
-    7. mshadow - For details, see, mshadow/LICENSE
-    8. nnvm/dmlc-core - For details, see, nnvm/dmlc-core/LICENSE
-    9. nnvm - For details, see, nnvm/LICENSE
-    10. nnvm-fusion - For details, see, nnvm/plugin/nnvm-fusion/LICENSE
-    11. ps-lite - For details, see, ps-lite/LICENSE
-    12. nnvm/tvm - For details, see, nnvm/tvm/LICENSE
-    13. googlemock scripts/generator - For details, see, 3rdparty/googletest/googlemock/scripts/generator/LICENSE
+    5. dmlc-core - For details, see, dmlc-core/LICENSE
+    6. nnvm/dmlc-core - For details, see, nnvm/dmlc-core/LICENSE
+    7. nnvm - For details, see, nnvm/LICENSE
+    8. nnvm-fusion - For details, see, nnvm/plugin/nnvm-fusion/LICENSE
+    9. nnvm/tvm - For details, see, nnvm/tvm/LICENSE
+    10. googlemock scripts/generator - For details, see, 3rdparty/googletest/googlemock/scripts/generator/LICENSE
 
 
     =======================================================================================

Makefile

@@ -16,6 +16,7 @@
 # under the License.
 
 ROOTDIR = $(CURDIR)
+TPARTYDIR = $(ROOTDIR)/3rdparty
 
 SCALA_VERSION_PROFILE := scala-2.11
 
@@ -36,7 +37,7 @@ endif
 endif
 
 ifndef DMLC_CORE
-	DMLC_CORE = $(ROOTDIR)/dmlc-core
+	DMLC_CORE = $(TPARTYDIR)/dmlc-core
 endif
 CORE_INC = $(wildcard $(DMLC_CORE)/include/*/*.h)
 
@@ -45,7 +46,7 @@ ifndef NNVM_PATH
 endif
 
 ifndef DLPACK_PATH
-	DLPACK_PATH = $(ROOTDIR)/dlpack
+	DLPACK_PATH = $(ROOTDIR)/3rdparty/dlpack
 endif
 
 ifndef AMALGAMATION_PATH
@@ -73,7 +74,7 @@ ifeq ($(USE_MKLDNN), 1)
 	export USE_MKLML = 1
 endif
 
-include mshadow/make/mshadow.mk
+include $(TPARTYDIR)/mshadow/make/mshadow.mk
 include $(DMLC_CORE)/make/dmlc.mk
 
 # all tge possible warning tread
@@ -91,7 +92,7 @@ ifeq ($(DEBUG), 1)
 else
 	CFLAGS += -O3 -DNDEBUG=1
 endif
-CFLAGS += -I$(ROOTDIR)/mshadow/ -I$(ROOTDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -I$(DLPACK_PATH)/include -I$(NNVM_PATH)/tvm/include -Iinclude $(MSHADOW_CFLAGS)
+CFLAGS += -I$(TPARTYDIR)/mshadow/ -I$(TPARTYDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -I$(DLPACK_PATH)/include -I$(NNVM_PATH)/tvm/include -Iinclude $(MSHADOW_CFLAGS)
 LDFLAGS = -pthread $(MSHADOW_LDFLAGS) $(DMLC_LDFLAGS)
 ifeq ($(DEBUG), 1)
 	NVCCFLAGS += -std=c++11 -Xcompiler -D_FORCE_INLINES -g -G -O0 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS)
@@ -293,7 +294,7 @@ $(info Running CUDA_ARCH: $(CUDA_ARCH))
 endif
 
 # ps-lite
-PS_PATH=$(ROOTDIR)/ps-lite
+PS_PATH=$(ROOTDIR)/3rdparty/ps-lite
 DEPS_PATH=$(shell pwd)/deps
 include $(PS_PATH)/make/ps.mk
 ifeq ($(USE_DIST_KVSTORE), 1)
@@ -472,7 +473,7 @@ test: $(TEST)
 lint: cpplint rcpplint jnilint pylint
 
 cpplint:
-	dmlc-core/scripts/lint.py mxnet cpp include src plugin cpp-package tests \
+	3rdparty/dmlc-core/scripts/lint.py mxnet cpp include src plugin cpp-package tests \
 	--exclude_path src/operator/contrib/ctc_include
 
 pylint:
@@ -504,7 +505,7 @@ cyclean:
 
 # R related shortcuts
 rcpplint:
-	dmlc-core/scripts/lint.py mxnet-rcpp ${LINT_LANG} R-package/src
+	3rdparty/dmlc-core/scripts/lint.py mxnet-rcpp ${LINT_LANG} R-package/src
 
 rpkg:
 	mkdir -p R-package/inst
@@ -513,7 +514,7 @@ rpkg:
 	cp -rf lib/libmxnet.so R-package/inst/libs
 	mkdir -p R-package/inst/include
 	cp -rf include/* R-package/inst/include
-	cp -rf dmlc-core/include/* R-package/inst/include/
+	cp -rf 3rdparty/dmlc-core/include/* R-package/inst/include/
 	cp -rf nnvm/include/* R-package/inst/include
 	Rscript -e "if(!require(devtools)){install.packages('devtools', repo = 'https://cloud.r-project.org/')}"
 	Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cloud.r-project.org/')); install_deps(pkg='R-package', dependencies = TRUE)"
@@ -562,7 +563,7 @@ scaladeploy:
 			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
 
 jnilint:
-	dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src
+	3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src
 
 ifneq ($(EXTRA_OPERATORS),)
 clean: cyclean $(EXTRA_PACKAGES_CLEAN)

amalgamation/Makefile

@@ -16,6 +16,7 @@
 # under the License.
 
 export MXNET_ROOT=`pwd`/..
+export TPARTYDIR=`pwd`/../3rdparty
 
 # Change this to path or specify in make command
 ifndef OPENBLAS_ROOT
@@ -72,15 +73,15 @@ nnvm.d:
 
 dmlc.d: dmlc-minimum0.cc
 	${CXX} ${CFLAGS} -M -MT dmlc-minimum0.o \
-	-I ${MXNET_ROOT}/dmlc-core/include \
+	-I ${TPARTYDIR}/dmlc-core/include \
 	-D__MIN__=$(MIN) $+ > dmlc.d
 
 
 mxnet_predict0.d: mxnet_predict0.cc nnvm.d dmlc.d
 	${CXX} ${CFLAGS} -M -MT mxnet_predict0.o \
-	-I ${MXNET_ROOT}/ -I ${MXNET_ROOT}/mshadow/ -I ${MXNET_ROOT}/dmlc-core/include -I ${MXNET_ROOT}/dmlc-core/src \
+	-I ${MXNET_ROOT}/ -I ${TPARTYDIR}/mshadow/ -I ${TPARTYDIR}/dmlc-core/include -I ${TPARTYDIR}/dmlc-core/src \
 	-I ${MXNET_ROOT}/nnvm/include \
-	-I ${MXNET_ROOT}/dlpack/include \
+	-I ${MXNET_ROOT}/3rdparty/dlpack/include \
 	-I ${MXNET_ROOT}/include \
 	-D__MIN__=$(MIN) mxnet_predict0.cc > mxnet_predict0.d
 	cat dmlc.d >> mxnet_predict0.d

amalgamation/dmlc-minimum0.cc

@@ -22,13 +22,13 @@
  * \brief Mininum DMLC library Amalgamation, used for easy plugin of dmlc lib.
  *  Normally this is not needed.
  */
-#include "../dmlc-core/src/io/line_split.cc"
-#include "../dmlc-core/src/io/recordio_split.cc"
-#include "../dmlc-core/src/io/indexed_recordio_split.cc"
-#include "../dmlc-core/src/io/input_split_base.cc"
-#include "../dmlc-core/src/io/local_filesys.cc"
-#include "../dmlc-core/src/data.cc"
-#include "../dmlc-core/src/io.cc"
-#include "../dmlc-core/src/recordio.cc"
+#include "../3rdparty/dmlc-core/src/io/line_split.cc"
+#include "../3rdparty/dmlc-core/src/io/recordio_split.cc"
+#include "../3rdparty/dmlc-core/src/io/indexed_recordio_split.cc"
+#include "../3rdparty/dmlc-core/src/io/input_split_base.cc"
+#include "../3rdparty/dmlc-core/src/io/local_filesys.cc"
+#include "../3rdparty/dmlc-core/src/data.cc"
+#include "../3rdparty/dmlc-core/src/io.cc"
+#include "../3rdparty/dmlc-core/src/recordio.cc"
 
 

amalgamation/prep_nnvm.sh

@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-DMLC_CORE=$(pwd)/../dmlc-core
+DMLC_CORE=$(pwd)/../3rdparty/dmlc-core
 cd ../nnvm/amalgamation
 make clean
 make DMLC_CORE_PATH=$DMLC_CORE nnvm.d

cpp-package/example/Makefile

@@ -18,7 +18,7 @@
 CPPEX_SRC = $(wildcard *.cpp)
 CPPEX_EXE = $(patsubst %.cpp, %, $(CPPEX_SRC))
 
-CFLAGS += -I../../include -I../../nnvm/include -I../../dmlc-core/include
+CFLAGS += -I../../include -I../../nnvm/include -I../../3rdparty/dmlc-core/include
 CPPEX_CFLAGS += -I../include
 CPPEX_EXTRA_LDFLAGS := -L. -lmxnet
 

docs/install/index.md

@@ -1213,7 +1213,7 @@ Edit the Makefile to install the MXNet with CUDA bindings to leverage the GPU on
     echo "USE_CUDNN=1" >> config.mk
 ```
 
-Edit the Mshadow Makefile to ensure MXNet builds with Pascal's hardware level low precision acceleration by editing mshadow/make/mshadow.mk and adding the following after line 122:
+Edit the Mshadow Makefile to ensure MXNet builds with Pascal's hardware level low precision acceleration by editing 3rdparty/mshadow/make/mshadow.mk and adding the following after line 122:
 ```bash
 MSHADOW_CFLAGS += -DMSHADOW_USE_PASCAL=1
 ```

example/image-classification/predict-cpp/Makefile

@@ -28,4 +28,4 @@ clean:
 	rm -f *.d *.o
 
 lint:
-	python ../../../dmlc-core/scripts/lint.py mxnet "cpp" ./
+	python ../../../3rdparty/dmlc-core/scripts/lint.py mxnet "cpp" ./

example/reinforcement-learning/a3c/launcher.py

@@ -27,8 +27,8 @@
 import argparse
 import signal
 
-sys.path.append(os.path.join(os.environ['HOME'], "mxnet/dmlc-core/tracker"))
-sys.path.append(os.path.join('/scratch', "mxnet/dmlc-core/tracker"))
+sys.path.append(os.path.join(os.environ['HOME'], "mxnet/3rdparty/dmlc-core/tracker"))
+sys.path.append(os.path.join('/scratch', "mxnet/3rdparty/dmlc-core/tracker"))
 from dmlc_tracker import tracker
 
 keepalive = """

tools/launch.py

@@ -26,7 +26,7 @@
 import logging
 
 curr_path = os.path.abspath(os.path.dirname(__file__))
-sys.path.append(os.path.join(curr_path, "../dmlc-core/tracker"))
+sys.path.append(os.path.join(curr_path, "..3rdparty/dmlc-core/tracker"))
 
 def dmlc_opts(opts):
     """convert from mxnet's opts to dmlc's opts

tools/license_header.py

@@ -63,12 +63,9 @@
 _WHITE_LIST = ['R-package/',
                'cub/',
                'docker/Dockerfiles',
-               'dlpack/',
                'dmlc-core/',
-               'mshadow/',
                'nnvm',
                '3rdparty',
-               'ps-lite',
                'src/operator/mkl/',
                'src/operator/special_functions-inl.h',
                'src/operator/nn/pool.h',