diff --git a/.travis.yml b/.travis.yml index 817d1ace7f87..d6b38f4ac8de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,12 @@ sudo: true language: cpp -cache: ccache +cache: + directories: + - $HOME/.ccache + - $HOME/.cache/pip + - $HOME/.mxnet + - $HOME/Library/Caches/Homebrew os: - osx @@ -17,7 +22,7 @@ before_install: - export PYTHONPATH=${PYTHONPATH}:${PWD}/python install: - - brew install ccache + - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache - export PATH="/usr/local/opt/ccache/libexec:$PATH" - source ci/travis/install.sh @@ -29,4 +34,7 @@ script: - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 - mv make/osx.mk config.mk - make -j 2 - - python -m nose --verbose tests/python/unittest/ + # We ignore several tests to avoid possible timeouts on large PRs. + # This lowers our test coverage, but is required for consistent Travis runs. + # These tests will be tested in a variety of environments in Jenkins based tests. + - python -m nose --with-timer --exclude-test=test_sparse_operator.test_elemwise_binary_ops --exclude-test=test_gluon_model_zoo.test_models --exclude-test=test_random.test_shuffle --exclude-test=test_operator.test_broadcast_binary_op --exclude-test=test_operator.test_pick --exclude-test=test_profiler.test_continuous_profile_and_instant_marker --exclude-test=test_metric_perf.test_metric_performance --exclude-test=test_operator.test_order --verbose tests/python/unittest/ diff --git a/src/operator/contrib/ctc_include/LICENSE b/3rdparty/ctc_include/LICENSE similarity index 100% rename from src/operator/contrib/ctc_include/LICENSE rename to 3rdparty/ctc_include/LICENSE diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/LICENSE b/3rdparty/ctc_include/contrib/moderngpu/LICENSE similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/LICENSE rename to 3rdparty/ctc_include/contrib/moderngpu/LICENSE diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctaloadbalance.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctaloadbalance.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctaloadbalance.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctaloadbalance.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctamerge.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctamerge.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctamerge.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctamerge.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctascan.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctascan.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctascan.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctascan.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasearch.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctasearch.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasearch.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctasearch.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasegreduce.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegreduce.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasegreduce.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegreduce.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasegscan.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegscan.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasegscan.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegscan.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasegsort.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegsort.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasegsort.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegsort.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasortedsearch.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctasortedsearch.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasortedsearch.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctasortedsearch.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/devicetypes.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/devicetypes.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/devicetypes.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/devicetypes.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/deviceutil.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/deviceutil.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/deviceutil.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/deviceutil.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/intrinsics.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/intrinsics.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/intrinsics.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/intrinsics.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/loadstore.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/loadstore.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/loadstore.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/loadstore.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/serialsets.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/serialsets.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/serialsets.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/serialsets.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/sortnetwork.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/sortnetwork.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/sortnetwork.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/sortnetwork.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/mgpudevice.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/mgpudevice.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/mgpudevice.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/mgpudevice.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/mgpuenums.h b/3rdparty/ctc_include/contrib/moderngpu/include/mgpuenums.h similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/mgpuenums.h rename to 3rdparty/ctc_include/contrib/moderngpu/include/mgpuenums.h diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/util/static.h b/3rdparty/ctc_include/contrib/moderngpu/include/util/static.h similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/util/static.h rename to 3rdparty/ctc_include/contrib/moderngpu/include/util/static.h diff --git a/src/operator/contrib/ctc_include/detail/cpu_ctc.h b/3rdparty/ctc_include/detail/cpu_ctc.h similarity index 100% rename from src/operator/contrib/ctc_include/detail/cpu_ctc.h rename to 3rdparty/ctc_include/detail/cpu_ctc.h diff --git a/src/operator/contrib/ctc_include/detail/ctc_helper.h b/3rdparty/ctc_include/detail/ctc_helper.h similarity index 100% rename from src/operator/contrib/ctc_include/detail/ctc_helper.h rename to 3rdparty/ctc_include/detail/ctc_helper.h diff --git a/src/operator/contrib/ctc_include/detail/gpu_ctc.h b/3rdparty/ctc_include/detail/gpu_ctc.h similarity index 100% rename from src/operator/contrib/ctc_include/detail/gpu_ctc.h rename to 3rdparty/ctc_include/detail/gpu_ctc.h diff --git a/src/operator/contrib/ctc_include/detail/gpu_ctc_kernels.h b/3rdparty/ctc_include/detail/gpu_ctc_kernels.h similarity index 100% rename from src/operator/contrib/ctc_include/detail/gpu_ctc_kernels.h rename to 3rdparty/ctc_include/detail/gpu_ctc_kernels.h diff --git a/src/operator/contrib/ctc_include/detail/hostdevice.h b/3rdparty/ctc_include/detail/hostdevice.h similarity index 100% rename from src/operator/contrib/ctc_include/detail/hostdevice.h rename to 3rdparty/ctc_include/detail/hostdevice.h diff --git a/3rdparty/mshadow b/3rdparty/mshadow index 8a9e337f3c47..696803bd7723 160000 --- a/3rdparty/mshadow +++ b/3rdparty/mshadow @@ -1 +1 @@ -Subproject commit 8a9e337f3c4794876bd04d5351d967333bcabee3 +Subproject commit 696803bd7723ade8230af878460d96c68a550fbc diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 55416355d8aa..90e548f42e3b 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -180,4 +180,6 @@ List of Contributors * [Per Goncalves da Silva](https://github.com/perdasilva) * [Zhijingcheng Yu](https://github.com/jasonyu1996) * [Cheng-Che Lee](https://github.com/stu1130) -* [Chaitanya Bapat](https://github.com/ChaiBapchya) \ No newline at end of file +* [Chaitanya Bapat](https://github.com/ChaiBapchya) +* [LuckyPigeon](https://github.com/LuckyPigeon) + diff --git a/Jenkinsfile b/Jenkinsfile index 81a25deca27b..af059c58e830 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -89,6 +89,30 @@ def python3_gpu_ut_nocudnn(docker_container_name) { } } +def deploy_docs() { + parallel 'Docs': { + node(NODE_LINUX_CPU) { + ws('workspace/docs') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git() + utils.docker_run('ubuntu_cpu', 'deploy_docs', false) + sh "ci/other/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}" + } + } + } + }, + 'Julia docs': { + node(NODE_LINUX_CPU) { + ws('workspace/julia-docs') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('cpu', mx_lib) + utils.docker_run('ubuntu_cpu', 'deploy_jl_docs', false) + } + } + } + } +} + node('mxnetlinux-cpu') { // Loading the utilities requires a node context unfortunately checkout scm @@ -746,6 +770,16 @@ core_logic: { } } }, + 'Julia 0.6: CPU': { + node(NODE_LINUX_CPU) { + ws('workspace/ut-julia06-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('cpu', mx_lib) + utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_julia06', false) + } + } + } + }, 'Python 2: CPU Win':{ node(NODE_WINDOWS_CPU) { @@ -911,15 +945,7 @@ core_logic: { } stage('Deploy') { - node(NODE_LINUX_CPU) { - ws('workspace/docs') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'deploy_docs', false) - sh "ci/other/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}" - } - } - } + deploy_docs() } } , diff --git a/Makefile b/Makefile index 1c8d70ecc695..a4b41b8d8371 100644 --- a/Makefile +++ b/Makefile @@ -66,8 +66,8 @@ $(warning "USE_MKL2017 is deprecated. We will switch to USE_MKLDNN.") endif ifeq ($(USE_MKLDNN), 1) - MKLDNNROOT = $(ROOTDIR)/3rdparty/mkldnn/install - MKLROOT = $(ROOTDIR)/3rdparty/mkldnn/install + MKLDNNROOT = $(ROOTDIR)/3rdparty/mkldnn/build/install + MKLROOT = $(ROOTDIR)/3rdparty/mkldnn/build/install export USE_MKLML = 1 endif diff --git a/amalgamation/python/mxnet_predict.py b/amalgamation/python/mxnet_predict.py index 091bfbb1cf14..a91d3849b0d2 100644 --- a/amalgamation/python/mxnet_predict.py +++ b/amalgamation/python/mxnet_predict.py @@ -163,7 +163,7 @@ def forward(self, **kwargs): for k, v in kwargs.items(): if not isinstance(v, np.ndarray): raise ValueError("Expect numpy ndarray as input") - v = np.ascontiguousarray(v, dtype=np.float32) + v = np.asarray(v, dtype=np.float32, order='C') _check_call(_LIB.MXPredSetInput( self.handle, c_str(k), v.ctypes.data_as(mx_float_p), diff --git a/benchmark/python/control_flow/rnn.py b/benchmark/python/control_flow/rnn.py index 8a44a9cab174..08498724b1b4 100644 --- a/benchmark/python/control_flow/rnn.py +++ b/benchmark/python/control_flow/rnn.py @@ -32,6 +32,7 @@ _parser.add_argument('--benchmark', choices=["foreach", "while_loop"], required=True) _parser.add_argument('--warmup_rounds', type=int, default=20) _parser.add_argument('--test_rounds', type=int, default=100) +_parser.add_argument('--gpu', type=bool, default=False) args = _parser.parse_args() @@ -66,8 +67,7 @@ def _func(*states): loop_vars=states, max_iterations=self.length, ) - assert len(out) == 1 - return out[0] + return out def _zeros(shape, ctx): @@ -124,7 +124,9 @@ def main(): cell_types = [gluon.rnn.RNNCell, gluon.rnn.GRUCell, gluon.rnn.LSTMCell] - ctxs = [mx.cpu(0)] + [mx.gpu(i) for i in _get_gpus()] + ctxs = [mx.cpu(0)] + if args.gpu: + ctxs = ctxs + [mx.gpu(i) for i in _get_gpus()] seq_lens = [100] batch_sizes = [1, 32] hidden_dims = [512] diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu index f45c8da4af87..7c7e2240ee61 100755 --- a/ci/docker/Dockerfile.build.ubuntu_cpu +++ b/ci/docker/Dockerfile.build.ubuntu_cpu @@ -45,6 +45,9 @@ RUN /work/ubuntu_r.sh COPY install/ubuntu_perl.sh /work/ RUN /work/ubuntu_perl.sh +COPY install/ubuntu_julia.sh /work/ +RUN /work/ubuntu_julia.sh + COPY install/ubuntu_clang.sh /work/ RUN /work/ubuntu_clang.sh diff --git a/ci/docker/install/docs_requirements b/ci/docker/install/docs_requirements index 7407223b3eed..4e3ce3e55e0b 100644 --- a/ci/docker/install/docs_requirements +++ b/ci/docker/install/docs_requirements @@ -6,7 +6,7 @@ h5py==2.8.0rc1 mock==2.0.0 nose==1.3.7 nose-timer==0.7.3 -numpy<1.15.0,>=1.8.2 +numpy<=1.15.2,>=1.8.2 pylint==1.8.3 pypandoc==1.4 recommonmark==0.4.0 diff --git a/ci/docker/install/ubuntu_julia.sh b/ci/docker/install/ubuntu_julia.sh new file mode 100755 index 000000000000..62013e36d8fd --- /dev/null +++ b/ci/docker/install/ubuntu_julia.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -ex + +export JLBINARY='julia.tar.gz' +export JULIADIR='/work/julia' +export JULIA="${JULIADIR}/bin/julia" + +mkdir -p $JULIADIR +# The julia version in Ubuntu repo is too old +# We download the tarball from the official link: +# https://julialang.org/downloads/ +wget -O $JLBINARY https://julialang-s3.julialang.org/bin/linux/x64/0.6/julia-0.6.2-linux-x86_64.tar.gz +tar xzvf $JLBINARY -C $JULIADIR --strip 1 +rm $JLBINARY + +$JULIA -e 'versioninfo()' diff --git a/ci/docker/install/ubuntu_nightly_tests.sh b/ci/docker/install/ubuntu_nightly_tests.sh index 68358908bdc9..406985ea3a4a 100755 --- a/ci/docker/install/ubuntu_nightly_tests.sh +++ b/ci/docker/install/ubuntu_nightly_tests.sh @@ -32,5 +32,5 @@ apt-get -y install time apt-get install -y subversion maven -y #>/dev/null # Packages needed for the Straight Dope Nightly tests. -pip2 install pandas scikit-image -pip3 install pandas scikit-image +pip2 install pandas scikit-image prompt_toolkit +pip3 install pandas scikit-image prompt_toolkit diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python.sh index 0fd91cbf706c..a60516386652 100755 --- a/ci/docker/install/ubuntu_python.sh +++ b/ci/docker/install/ubuntu_python.sh @@ -29,5 +29,5 @@ wget -nv https://bootstrap.pypa.io/get-pip.py python3 get-pip.py python2 get-pip.py -pip2 install nose cpplint==1.3.0 pylint==1.9.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 -pip3 install nose cpplint==1.3.0 pylint==2.1.1 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 +pip2 install nose cpplint==1.3.0 pylint==1.9.3 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 +pip3 install nose cpplint==1.3.0 pylint==2.1.1 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index d1fc2239a442..96b1646eff97 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -816,6 +816,35 @@ unittest_ubuntu_gpu_R() { make rpkgtest R_LIBS=/tmp/r-site-library R_GPU_ENABLE=1 } +unittest_ubuntu_cpu_julia06() { + set -ex + export PATH="/work/julia/bin:$PATH" + export MXNET_HOME='/work/mxnet' + export JULIA_PKGDIR='/work/julia-pkg' + export DEPDIR=`julia -e 'print(Pkg.dir())'` + + julia -e 'versioninfo()' + julia -e 'Pkg.init()' + + # install package + ln -sf ${MXNET_HOME}/julia ${DEPDIR}/MXNet + + # install dependencies + julia -e 'Pkg.resolve()' + + # FIXME + export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so' + + # use the prebuilt binary from $MXNET_HOME/lib + julia -e 'Pkg.build("MXNet")' + + # run the script `julia/test/runtests.jl` + julia -e 'Pkg.test("MXNet")' + + # See https://github.com/dmlc/MXNet.jl/pull/303#issuecomment-341171774 + julia -e 'using MXNet; mx._sig_checker()' +} + unittest_centos7_cpu() { set -ex cd /work/mxnet @@ -1090,6 +1119,31 @@ deploy_docs() { popd } +deploy_jl_docs() { + set -ex + export PATH="/work/julia/bin:$PATH" + export MXNET_HOME='/work/mxnet' + export JULIA_PKGDIR='/work/julia-pkg' + export DEPDIR=`julia -e 'print(Pkg.dir())'` + + julia -e 'versioninfo()' + julia -e 'Pkg.init()' + ln -sf ${MXNET_HOME}/julia ${DEPDIR}/MXNet + julia -e 'Pkg.resolve()' + + # FIXME + export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so' + + # use the prebuilt binary from $MXNET_HOME/lib + julia -e 'Pkg.build("MXNet")' + # build docs + julia -e 'Pkg.add("Documenter")' + julia -e 'cd(Pkg.dir("MXNet")); include(joinpath("docs", "make.jl"))' + + # TODO: make Jenkins worker push to MXNet.jl ph-pages branch if master build + # ... +} + # broken_link_checker broken_link_checker() { diff --git a/ci/travis/install.sh b/ci/travis/install.sh index 16db601211a5..ae959767133a 100644 --- a/ci/travis/install.sh +++ b/ci/travis/install.sh @@ -17,14 +17,10 @@ # specific language governing permissions and limitations # under the License. +# Disable brew auto-update to avoid long running updates while running tests in CI. +export HOMEBREW_NO_AUTO_UPDATE=1 + if [ ${TRAVIS_OS_NAME} == "osx" ]; then - brew update brew install opencv - brew install python3 - brew install fftw - brew install libpng - brew install ImageMagick - brew install swig - python -m pip install --user nose numpy cython scipy requests mock - python3 -m pip install --user nose numpy cython scipy requests mock + python -m pip install --user nose numpy cython scipy requests mock nose-timer nose-exclude fi diff --git a/contrib/clojure-package/README.md b/contrib/clojure-package/README.md index 66ba77b76135..8c71224a3a55 100644 --- a/contrib/clojure-package/README.md +++ b/contrib/clojure-package/README.md @@ -10,6 +10,8 @@ The motivation for creating a Clojure package is to be able to open the deep lea For high leverage, the Clojure package has been built on the existing Scala package using interop. This has allowed rapid development and close parity with the Scala functionality. This also leaves the door open to directly developing code against the jni-bindings with Clojure in the future in an incremental fashion, using the test suites as a refactoring guide. +For a **video introduction**, see [Clojure MXNet with Carin Meier - Clojure Virtual Meetup](https://www.crowdcast.io/e/clojure-mxnet-with-carin) (setup instructions from 20:49) + ## Current State and Plans The Clojure package is nearing the end of its first development milestone which is to achieve a close parity with the Scala package. diff --git a/cpp-package/README.md b/cpp-package/README.md index 2b6e0e39f0fd..c4fe63c9ec58 100644 --- a/cpp-package/README.md +++ b/cpp-package/README.md @@ -1,21 +1,46 @@ # MXNet C++ Package -To build the C++ package, please refer to [this guide](). +The MXNet C++ Package provides C++ API bindings to the users of MXNet. Currently, these bindings are not available as standalone package. +The users of these bindings are required to build this package as mentioned below. -A basic tutorial can be found at . +## Building C++ Package -The example directory contains examples for you to get started. +The cpp-package directory contains the implementation of C++ API. As mentioned above, users are required to build this directory or package before using it. +**The cpp-package is built while building the MXNet shared library, *libmxnet.so*.** + +###Steps to build the C++ package: +1. Building the MXNet C++ package requires building MXNet from source. +2. Clone the MXNet GitHub repository **recursively** to ensure the code in submodules is available for building MXNet. + ``` + git clone --recursive https://github.com/apache/incubator-mxnet mxnet + ``` + +3. Install the [prerequisites](), desired [BLAS libraries]() and optional [OpenCV, CUDA, and cuDNN]() for building MXNet from source. +4. There is a configuration file for make, [make/config.mk]() that contains all the compilation options. You can edit this file and set the appropriate options prior to running the **make** command. +5. Please refer to [platform specific build instructions]() and available [build configurations](https://mxnet.incubator.apache.org/install/build_from_source#build-configurations) for more details. +5. For enabling the build of C++ Package, set the **USE\_CPP\_PACKAGE = 1** in [make/config.mk](). Optionally, the compilation flag can also be specified on **make** command line as follows. + ``` + make -j USE_CPP_PACKAGE=1 + ``` + +## Usage -## Building C++ examples in examples folder +In order to consume the C++ API please follow the steps below. -From cpp-package/examples directory -- Build all examples in release mode: **make all** -- Build all examples in debug mode : **make debug** +1. Ensure that the MXNet shared library is built from source with the **USE\_CPP\_PACKAGE = 1**. +2. Include the [MxNetCpp.h]() in the program that is going to consume MXNet C++ API. + ``` + #include + ``` +3. While building the program, ensure that the correct paths to the directories containing header files and MXNet shared library. +4. The program links the MXNet shared library dynamically. Hence the library needs to be accessible to the program during runtime. This can be achieved by including the path to the shared library in the environment variable **LD\_LIBRARY\_PATH** for Linux, Mac. and Ubuntu OS and **PATH** for Windows OS. -By default, the examples are build to be run on GPU. -To build examples to run on CPU: -- Release: **make all MXNET_USE_CPU=1** -- Debug: **make debug MXNET_USE_CPU=1** +## Tutorial + +A basic tutorial can be found at . + +## Examples + +The example directory contains examples for you to get started. -The makefile will also download the necessary data files and store in data folder. (The download will take couple of minutes, but will be done only once on a fresh installation.) diff --git a/cpp-package/example/README.md b/cpp-package/example/README.md new file mode 100644 index 000000000000..76f6a0127c0b --- /dev/null +++ b/cpp-package/example/README.md @@ -0,0 +1,170 @@ +# MXNet C++ Package Examples + +## Building C++ examples + +The examples are built while building the MXNet library and cpp-package from source . However, they can be built manually as follows + +From cpp-package/examples directory + +- Build all examples in release mode: **make all** +- Build all examples in debug mode: **make debug** + +<<<<<<< HEAD +By default, the examples are built to be run on GPU. To build examples to run on CPU: +======= +By default, the examples are build to be run on GPU. To build examples to run on CPU: +>>>>>>> 39054b349e83ead13127cd1bd6b90e3141bc0451 + +- Release: **make all MXNET\_USE\_CPU=1** +- Debug: **make debug MXNET\_USE\_CPU=1** + +<<<<<<< HEAD +The examples that are built to be run on GPU may not work on the non-GPU machines. +The makefile will also download the necessary data files and store in a data folder. (The download will take couple of minutes, but will be done only once on a fresh installation.) +======= +The examples that are build to be run on GPU may not work on the non-GPU machines. +The makefile will also download the necessary data files and store in data folder. (The download will take couple of minutes, but will be done only once on a fresh installation.) +>>>>>>> 39054b349e83ead13127cd1bd6b90e3141bc0451 + + +## Examples + +<<<<<<< HEAD +This directory contains following examples. In order to run the examples, ensure that the path to the MXNet shared library is added to the OS specific environment variable viz. **LD\_LIBRARY\_PATH** for Linux, Mac and Ubuntu OS and **PATH** for Windows OS. + +### [alexnet.cpp]() + +The example implements the C++ version of AlexNet. The networks trains on MNIST data. The number of epochs can be specified as a command line argument. For example to train with 10 epochs use the following: + +======= +This directory contains following examples. In order to run the examples, ensure that the path to the MXNet shared library is added to the OS specific environment variable such as _LD\_LIBRARY\_PATH_ . + +### [alexnet.cpp]() + +The example implements C++ version of AlexNet. The networks trains the MNIST data. The number of epochs can be specified as command line arguement. For example: +>>>>>>> 39054b349e83ead13127cd1bd6b90e3141bc0451 + ``` + ./alexnet 10 + ``` + +<<<<<<< HEAD +### [googlenet.cpp]() + +The code implements a GoogLeNet/Inception network using the C++ API. The example uses MNIST data to train the network. By default, the example trains the model for 100 epochs. The number of epochs can also be specified in the command line. For example, to train the model for 10 epochs use the following: +======= +### [charRNN.cpp]() + +The code implements C++ version charRNN for mxnet\example\rnn\char-rnn.ipynb with MXNet.cpp API. The generated params file is compatiable with python version. The train() and predict() has been verified with original data samples. + +The example expects arguments as follows: + +``` + ./charRNN train [BuildIn\ [TImeMajor] {corpus file} { batch size} { max epoch} [{starting epoch}] + ./charRNN predict [BuildIn\ [TImeMajor] {param file} { batch size} { max epoch} [{starting epoch}] +``` + +### [googlenet.cpp]() + +The code implements GoogLeNet/Inception network using C++ API. The example uses MNIST data to train the network. The number of epochs can be specified in the command line as follows. If not specified, the model trains for 100 epochs. +>>>>>>> 39054b349e83ead13127cd1bd6b90e3141bc0451 + +``` +./googlenet 10 +``` + +### [mlp.cpp]() + +<<<<<<< HEAD +The code implements a multilayer perceptron from scratch. The example creates its own dummy data to train the model. The example does not require command line parameters. It trains the model for 20,000 epochs. +To run the example use the following command: +======= +The code implements multilayer perceptron from scratch. The example creates its own dummy data to train the model. The example does not require command line parameters. It trains the model for 20000 iterations. +>>>>>>> 39054b349e83ead13127cd1bd6b90e3141bc0451 + +``` +./mlp +``` + +### [mlp_cpu.cpp]() + +<<<<<<< HEAD +The code implements a multilayer perceptron to train the MNIST data. The code demonstrates the use of "SimpleBind" C++ API and MNISTIter. The example is designed to work on CPU. The example does not require command line parameters. +To run the example use the following command: +======= +The code implements multilayer perceptron to train the MNIST data. The code demonstrates the use of "SimpleBind" C++ API and MNISTIter. The example is designed to work on CPU. The example does not require command line parameters. + +>>>>>>> 39054b349e83ead13127cd1bd6b90e3141bc0451 +``` +./mlp_cpu +``` + +### [mlp_gpu.cpp]() +<<<<<<< HEAD +The code implements multilayer perceptron to train the MNIST data. The code demonstrates the use of the "SimpleBind" C++ API and MNISTIter. The example is designed to work on GPU. The example does not require command line arguments. To run the example execute following command: +======= +The code implements multilayer perceptron to train the MNIST data. The code demonstrates the use of "SimpleBind" C++ API and MNISTIter. The example is designed to work on GPU. The example does not require command line paratmeters. +>>>>>>> 39054b349e83ead13127cd1bd6b90e3141bc0451 + +``` +./mlp_gpu +``` + +### [mlp_csv.cpp]() +<<<<<<< HEAD +The code implements a multilayer perceptron to train the MNIST data. The code demonstrates the use of the "SimpleBind" C++ API and CSVIter. The CSVIter can iterate data that is in CSV format. The example can be run on CPU or GPU. The example usage is as follows: +======= +The code implements multilayer perceptron to train the MNIST data. The code demonstrates the use of "SimpleBind" C++ API and CSVIter. The CSVIter can iterate data that is in CSV format. The example can be run on CPU or GPU. The example usage is as follows: +>>>>>>> 39054b349e83ead13127cd1bd6b90e3141bc0451 + +``` +mlp_csv --train mnist_training_set.csv --test mnist_test_set.csv --epochs 10 --batch_size 100 --hidden_units "128,64,64 [--gpu]" +``` + +### [resnet.cpp]() + +<<<<<<< HEAD +The code implements a resnet model using the C++ API. The model is used to train MNIST data. The number of epochs for training the model can be specified on the command line. By default, model is trained for 100 epochs. For example, to train with 10 epochs use the following command: +======= +The code implements resnet model using C++ API. The model is used to train MNIST data. The number of epochs for training the model can be specified on the command line. By default, model is trained for 100 epochs. +>>>>>>> 39054b349e83ead13127cd1bd6b90e3141bc0451 + +``` +./resnet 10 +``` + +### [lenet.cpp]() + +<<<<<<< HEAD +The code implements a lenet model using the C++ API. It uses MNIST training data in CSV format to train the network. The example does not use built-in CSVIter to read the data from CSV file. The number of epochs can be specified on the command line. By default, the mode is trained for 100,000 epochs. For example, to train with 10 epochs use the following command: +======= +The code implements lenet model using C++ API. It uses MNIST training data in CSV format to train the network. The example does not use built-in CSVIter to read the data from CSV file. The number of epochs can be specified on the command line. By default, the mode is trained for 100000 epochs. +>>>>>>> 39054b349e83ead13127cd1bd6b90e3141bc0451 + +``` +./lenet 10 +``` +### [lenet\_with\_mxdataiter.cpp]() + +<<<<<<< HEAD +The code implements a lenet model using the C++ API. It uses MNIST training data to train the network. The example uses built-in MNISTIter to read the data. The number of epochs can be specified on the command line. By default, the mode is trained for 100 epochs. For example, to train with 10 epochs use the following command: +======= +The code implements lenet model using C++ API. It uses MNIST training data to train the network. The example uses built-in MNISTIter to read the data. The number of epochs can be specified on the command line. By default, the mode is trained for 100 epochs. +>>>>>>> 39054b349e83ead13127cd1bd6b90e3141bc0451 + +``` +./lenet\_with\_mxdataiter 10 +``` + +In addition, there is `run_lenet_with_mxdataiter.sh` that downloads the mnist data and run `lenet_with_mxdataiter` example. + +###[inception_bn.cpp]() + +<<<<<<< HEAD +The code implements an Inception network using the C++ API with batch normalization. The example uses MNIST data to train the network. The model trains for 100 epochs. The example can be run by executing the following command: +======= +The code implements Inception network using C++ API with batch normalization. The example uses MNIST data to train the network. The model trains for 100 epochs. +>>>>>>> 39054b349e83ead13127cd1bd6b90e3141bc0451 + +``` +./inception_bn +``` diff --git a/cpp-package/example/mlp_csv.cpp b/cpp-package/example/mlp_csv.cpp new file mode 100644 index 000000000000..8aec4b76d917 --- /dev/null +++ b/cpp-package/example/mlp_csv.cpp @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Example: mlp_csv + * Description: + * The following example demonstrates how to use CSVIter. This example creates + * mlp (multi-layer perceptron) model and trains the MNIST data which is in + * CSV format. + */ +#include +#include "utils.h" +#include "mxnet-cpp/MxNetCpp.h" + +using namespace mxnet::cpp; + +/* + * Implementing the mlp symbol with given hidden units configuration. + */ +Symbol mlp(const std::vector &hidden_units) { + auto data = Symbol::Variable("data"); + auto label = Symbol::Variable("label"); + + std::vector weights(hidden_units.size()); + std::vector biases(hidden_units.size()); + std::vector outputs(hidden_units.size()); + + for (size_t i = 0; i < hidden_units.size(); ++i) { + weights[i] = Symbol::Variable("w" + std::to_string(i)); + biases[i] = Symbol::Variable("b" + std::to_string(i)); + Symbol fc = FullyConnected( + i == 0? data : outputs[i-1], // data + weights[i], + biases[i], + hidden_units[i]); + outputs[i] = i == hidden_units.size()-1 ? fc : Activation(fc, ActivationActType::kRelu); + } + return SoftmaxOutput(outputs.back(), label); +} + +/* + * Convert the input string of number of hidden units into the vector of integers. + */ +std::vector getLayers(const std::string& hidden_units_string) { + std::vector hidden_units; + char *pNext; + int num_unit = strtol(hidden_units_string.c_str(), &pNext, 10); + hidden_units.push_back(num_unit); + while (*pNext) { + num_unit = strtol(pNext, &pNext, 10); + hidden_units.push_back(num_unit); + } + return hidden_units; +} + +void printUsage() { + std::cout << "Usage:" << std::endl; + std::cout << "mlp_csv --train mnist_training_set.csv --test mnist_test_set.csv --epochs 10 " + << "--batch_size 100 --hidden_units \"128 64 64\" [--gpu]" << std::endl; + std::cout << "The example uses mnist data in CSV format. The MNIST data in CSV format assumes " + << "the column 0 to be label and the rest 784 column to be data." << std::endl; + std::cout << "By default, the example uses 'cpu' context. If '--gpu' is specified, " + << "program uses 'gpu' context." <