[MXNET-1017] Updating the readme file for cpp-package and adding readme file for example directory.

.travis.yml

@@ -2,7 +2,12 @@ sudo: true
 
 language: cpp
 
-cache: ccache
+cache:
+  directories:
+    - $HOME/.ccache
+    - $HOME/.cache/pip
+    - $HOME/.mxnet
+    - $HOME/Library/Caches/Homebrew
 
 os:
   - osx
@@ -17,7 +22,7 @@ before_install:
   - export PYTHONPATH=${PYTHONPATH}:${PWD}/python
 
 install:
-  - brew install ccache
+  - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache
   - export PATH="/usr/local/opt/ccache/libexec:$PATH"
   - source ci/travis/install.sh
 
@@ -29,4 +34,7 @@ script:
   - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
   - mv make/osx.mk config.mk
   - make -j 2
-  - python -m nose --verbose tests/python/unittest/
+  # We ignore several tests to avoid possible timeouts on large PRs.
+  # This lowers our test coverage, but is required for consistent Travis runs.
+  # These tests will be tested in a variety of environments in Jenkins based tests.
+  - python -m nose --with-timer --exclude-test=test_sparse_operator.test_elemwise_binary_ops --exclude-test=test_gluon_model_zoo.test_models --exclude-test=test_random.test_shuffle --exclude-test=test_operator.test_broadcast_binary_op --exclude-test=test_operator.test_pick --exclude-test=test_profiler.test_continuous_profile_and_instant_marker --exclude-test=test_metric_perf.test_metric_performance --exclude-test=test_operator.test_order --verbose tests/python/unittest/

CONTRIBUTORS.md

@@ -180,4 +180,6 @@ List of Contributors
 * [Per Goncalves da Silva](https://github.com/perdasilva)
 * [Zhijingcheng Yu](https://github.com/jasonyu1996)
 * [Cheng-Che Lee](https://github.com/stu1130)
-* [Chaitanya Bapat](https://github.com/ChaiBapchya)
+* [Chaitanya Bapat](https://github.com/ChaiBapchya)
+* [LuckyPigeon](https://github.com/LuckyPigeon)
+

Jenkinsfile

@@ -89,6 +89,30 @@ def python3_gpu_ut_nocudnn(docker_container_name) {
   }
 }
 
+def deploy_docs() {
+  parallel 'Docs': {
+    node(NODE_LINUX_CPU) {
+      ws('workspace/docs') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          utils.init_git()
+          utils.docker_run('ubuntu_cpu', 'deploy_docs', false)
+          sh "ci/other/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}"
+        }
+      }
+    }
+  },
+  'Julia docs': {
+    node(NODE_LINUX_CPU) {
+      ws('workspace/julia-docs') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          utils.unpack_and_init('cpu', mx_lib)
+          utils.docker_run('ubuntu_cpu', 'deploy_jl_docs', false)
+        }
+      }
+    }
+  }
+}
+
 node('mxnetlinux-cpu') {
   // Loading the utilities requires a node context unfortunately
   checkout scm
@@ -746,6 +770,16 @@ core_logic: {
         }
       }
     },
+    'Julia 0.6: CPU': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/ut-julia06-cpu') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            utils.unpack_and_init('cpu', mx_lib)
+            utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_julia06', false)
+          }
+        }
+      }
+    },
 
     'Python 2: CPU Win':{
       node(NODE_WINDOWS_CPU) {
@@ -911,15 +945,7 @@ core_logic: {
   }
 
   stage('Deploy') {
-    node(NODE_LINUX_CPU) {
-      ws('workspace/docs') {
-        timeout(time: max_time, unit: 'MINUTES') {
-          utils.init_git()
-          utils.docker_run('ubuntu_cpu', 'deploy_docs', false)
-          sh "ci/other/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}"
-        }
-      }
-    }
+    deploy_docs()
   }
 }
 ,

Makefile

@@ -66,8 +66,8 @@ $(warning "USE_MKL2017 is deprecated. We will switch to USE_MKLDNN.")
 endif
 
 ifeq ($(USE_MKLDNN), 1)
-	MKLDNNROOT = $(ROOTDIR)/3rdparty/mkldnn/install
-	MKLROOT = $(ROOTDIR)/3rdparty/mkldnn/install
+	MKLDNNROOT = $(ROOTDIR)/3rdparty/mkldnn/build/install
+	MKLROOT = $(ROOTDIR)/3rdparty/mkldnn/build/install
 	export USE_MKLML = 1
 endif
 

amalgamation/python/mxnet_predict.py

@@ -163,7 +163,7 @@ def forward(self, **kwargs):
         for k, v in kwargs.items():
             if not isinstance(v, np.ndarray):
                 raise ValueError("Expect numpy ndarray as input")
-            v = np.ascontiguousarray(v, dtype=np.float32)
+            v = np.asarray(v, dtype=np.float32, order='C')
             _check_call(_LIB.MXPredSetInput(
                 self.handle, c_str(k),
                 v.ctypes.data_as(mx_float_p),

benchmark/python/control_flow/rnn.py

@@ -32,6 +32,7 @@
 _parser.add_argument('--benchmark', choices=["foreach", "while_loop"], required=True)
 _parser.add_argument('--warmup_rounds', type=int, default=20)
 _parser.add_argument('--test_rounds', type=int, default=100)
+_parser.add_argument('--gpu', type=bool, default=False)
 args = _parser.parse_args()
 
 
@@ -66,8 +67,7 @@ def _func(*states):
             loop_vars=states,
             max_iterations=self.length,
         )
-        assert len(out) == 1
-        return out[0]
+        return out
 
 
 def _zeros(shape, ctx):
@@ -124,7 +124,9 @@ def main():
     cell_types = [gluon.rnn.RNNCell,
                   gluon.rnn.GRUCell,
                   gluon.rnn.LSTMCell]
-    ctxs = [mx.cpu(0)] + [mx.gpu(i) for i in _get_gpus()]
+    ctxs = [mx.cpu(0)]
+    if args.gpu:
+        ctxs = ctxs + [mx.gpu(i) for i in _get_gpus()]
     seq_lens = [100]
     batch_sizes = [1, 32]
     hidden_dims = [512]

ci/docker/Dockerfile.build.ubuntu_cpu

@@ -45,6 +45,9 @@ RUN /work/ubuntu_r.sh
 COPY install/ubuntu_perl.sh /work/
 RUN /work/ubuntu_perl.sh
 
+COPY install/ubuntu_julia.sh /work/
+RUN /work/ubuntu_julia.sh
+
 COPY install/ubuntu_clang.sh /work/
 RUN /work/ubuntu_clang.sh
 

ci/docker/install/docs_requirements

@@ -6,7 +6,7 @@ h5py==2.8.0rc1
 mock==2.0.0
 nose==1.3.7
 nose-timer==0.7.3
-numpy<1.15.0,>=1.8.2
+numpy<=1.15.2,>=1.8.2
 pylint==1.8.3
 pypandoc==1.4
 recommonmark==0.4.0

ci/docker/install/ubuntu_julia.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+
+export JLBINARY='julia.tar.gz'
+export JULIADIR='/work/julia'
+export JULIA="${JULIADIR}/bin/julia"
+
+mkdir -p $JULIADIR
+# The julia version in Ubuntu repo is too old
+# We download the tarball from the official link:
+#   https://julialang.org/downloads/
+wget -O $JLBINARY https://julialang-s3.julialang.org/bin/linux/x64/0.6/julia-0.6.2-linux-x86_64.tar.gz
+tar xzvf $JLBINARY -C $JULIADIR --strip 1
+rm $JLBINARY
+
+$JULIA -e 'versioninfo()'

ci/docker/install/ubuntu_nightly_tests.sh

@@ -32,5 +32,5 @@ apt-get -y install time
 apt-get install -y subversion maven -y #>/dev/null
 
 # Packages needed for the Straight Dope Nightly tests.
-pip2 install pandas scikit-image
-pip3 install pandas scikit-image
+pip2 install pandas scikit-image prompt_toolkit
+pip3 install pandas scikit-image prompt_toolkit

ci/docker/install/ubuntu_python.sh

@@ -29,5 +29,5 @@ wget -nv https://bootstrap.pypa.io/get-pip.py
 python3 get-pip.py
 python2 get-pip.py
 
-pip2 install nose cpplint==1.3.0 pylint==1.9.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
-pip3 install nose cpplint==1.3.0 pylint==2.1.1 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+pip2 install nose cpplint==1.3.0 pylint==1.9.3 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+pip3 install nose cpplint==1.3.0 pylint==2.1.1 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3

ci/docker/runtime_functions.sh

@@ -816,6 +816,35 @@ unittest_ubuntu_gpu_R() {
     make rpkgtest R_LIBS=/tmp/r-site-library R_GPU_ENABLE=1
 }
 
+unittest_ubuntu_cpu_julia06() {
+    set -ex
+    export PATH="/work/julia/bin:$PATH"
+    export MXNET_HOME='/work/mxnet'
+    export JULIA_PKGDIR='/work/julia-pkg'
+    export DEPDIR=`julia -e 'print(Pkg.dir())'`
+
+    julia -e 'versioninfo()'
+    julia -e 'Pkg.init()'
+
+    # install package
+    ln -sf ${MXNET_HOME}/julia ${DEPDIR}/MXNet
+
+    # install dependencies
+    julia -e 'Pkg.resolve()'
+
+    # FIXME
+    export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'
+
+    # use the prebuilt binary from $MXNET_HOME/lib
+    julia -e 'Pkg.build("MXNet")'
+
+    # run the script `julia/test/runtests.jl`
+    julia -e 'Pkg.test("MXNet")'
+
+    # See https://github.com/dmlc/MXNet.jl/pull/303#issuecomment-341171774
+    julia -e 'using MXNet; mx._sig_checker()'
+}
+
 unittest_centos7_cpu() {
     set -ex
     cd /work/mxnet
@@ -1090,6 +1119,31 @@ deploy_docs() {
     popd
 }
 
+deploy_jl_docs() {
+    set -ex
+    export PATH="/work/julia/bin:$PATH"
+    export MXNET_HOME='/work/mxnet'
+    export JULIA_PKGDIR='/work/julia-pkg'
+    export DEPDIR=`julia -e 'print(Pkg.dir())'`
+
+    julia -e 'versioninfo()'
+    julia -e 'Pkg.init()'
+    ln -sf ${MXNET_HOME}/julia ${DEPDIR}/MXNet
+    julia -e 'Pkg.resolve()'
+
+    # FIXME
+    export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'
+
+    # use the prebuilt binary from $MXNET_HOME/lib
+    julia -e 'Pkg.build("MXNet")'
+    # build docs
+    julia -e 'Pkg.add("Documenter")'
+    julia -e 'cd(Pkg.dir("MXNet")); include(joinpath("docs", "make.jl"))'
+
+    # TODO: make Jenkins worker push to MXNet.jl ph-pages branch if master build
+    # ...
+}
+
 # broken_link_checker
 
 broken_link_checker() {

ci/travis/install.sh

@@ -17,14 +17,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# Disable brew auto-update to avoid long running updates while running tests in CI.
+export HOMEBREW_NO_AUTO_UPDATE=1
+
 if [ ${TRAVIS_OS_NAME} == "osx" ]; then
-    brew update
     brew install opencv
-    brew install python3
-    brew install fftw
-    brew install libpng
-    brew install ImageMagick
-    brew install swig
-    python -m pip install --user nose numpy cython scipy requests mock
-    python3 -m pip install --user nose numpy cython scipy requests mock
+    python -m pip install --user nose numpy cython scipy requests mock nose-timer nose-exclude
 fi

contrib/clojure-package/README.md

@@ -10,6 +10,8 @@ The motivation for creating a Clojure package is to be able to open the deep lea
 
 For high leverage, the Clojure package has been built on the existing Scala package using interop. This has allowed rapid development and close parity with the Scala functionality. This also leaves the door open to directly developing code against the jni-bindings with Clojure in the future in an incremental fashion, using the test suites as a refactoring guide.
 
+For a **video introduction**, see [Clojure MXNet with Carin Meier - Clojure Virtual Meetup](https://www.crowdcast.io/e/clojure-mxnet-with-carin) (setup instructions from 20:49)
+
 ## Current State and Plans
 
 The Clojure package is nearing the end of its first development milestone which is to achieve a close parity with the Scala package.

cpp-package/README.md

@@ -1,21 +1,46 @@
 # MXNet C++ Package
 
-To build the C++ package, please refer to [this guide](<https://mxnet.incubator.apache.org/install/build_from_source#build-the-c-package>).
+The MXNet C++ Package provides C++ API bindings to the users of MXNet.  Currently, these bindings are not available as standalone package.
+The users of these bindings are required to build this package as mentioned below.
 
-A basic tutorial can be found at <https://mxnet.incubator.apache.org/tutorials/c++/basics.html>.
+## Building C++ Package
 
-The example directory contains examples for you to get started.
+The cpp-package directory contains the implementation of C++ API. As mentioned above, users are required to build this directory or package before using it.
+**The cpp-package is built while building the MXNet shared library, *libmxnet.so*.**
+
+###Steps to build the C++ package:
+1.  Building the MXNet C++ package requires building MXNet from source.
+2.  Clone the MXNet GitHub repository **recursively** to ensure the code in submodules is available for building MXNet.
+	```
+	git clone --recursive https://github.com/apache/incubator-mxnet mxnet
+	```
+
+3.  Install the [prerequisites](<https://mxnet.incubator.apache.org/install/build_from_source#prerequisites>), desired [BLAS libraries](<https://mxnet.incubator.apache.org/install/build_from_source#blas-library>) and optional [OpenCV, CUDA, and cuDNN](<https://mxnet.incubator.apache.org/install/build_from_source#optional>) for building MXNet from source.
+4.  There is a configuration file for make, [make/config.mk](<https://github.com/apache/incubator-mxnet/blob/master/make/config.mk>) that contains all the compilation options. You can edit this file and set the appropriate options prior to running the **make** command.
+5.  Please refer to  [platform specific build instructions](<https://mxnet.incubator.apache.org/install/build_from_source#build-instructions-by-operating-system>) and available [build configurations](https://mxnet.incubator.apache.org/install/build_from_source#build-configurations) for more details. 
+5.  For enabling the build of C++ Package, set the **USE\_CPP\_PACKAGE = 1** in [make/config.mk](<https://github.com/apache/incubator-mxnet/blob/master/make/config.mk>). Optionally, the compilation flag can also be specified on **make** command line as follows.
+	```
+	make -j USE_CPP_PACKAGE=1 
+	```
+
+## Usage
 
-## Building C++ examples in examples folder
+In order to consume the C++ API please follow the steps below.
 
-From cpp-package/examples directory
--  Build all examples in release mode: **make all**
--  Build all examples in debug mode : **make debug**
+1. Ensure that the MXNet shared library is built from source with the **USE\_CPP\_PACKAGE = 1**.
+2. Include the [MxNetCpp.h](<https://github.com/apache/incubator-mxnet/blob/master/cpp-package/include/mxnet-cpp/MxNetCpp.h>) in the program that is going to consume MXNet C++ API.
+	```
+	#include <mxnet-cpp/MxNetCpp.h>
+	```
+3. While building the program, ensure that the correct paths to the directories containing header files and MXNet shared library.
+4. The program links the MXNet shared library dynamically. Hence the library needs to be accessible to the program during runtime. This can be achieved by including the path to the shared library in the environment variable  **LD\_LIBRARY\_PATH** for Linux, Mac. and Ubuntu OS and **PATH** for Windows OS.
 
-By default, the examples are build to be run on GPU.
-To build examples to run on CPU:
-- Release: **make all MXNET_USE_CPU=1**  
-- Debug: **make debug MXNET_USE_CPU=1**  
 
+## Tutorial
+
+A basic tutorial can be found at <https://mxnet.incubator.apache.org/tutorials/c++/basics.html>.
+
+## Examples
+
+The example directory contains examples for you to get started.
 
-The makefile will also download the necessary data files and store in data folder. (The download will take couple of minutes, but will be done only once on a fresh installation.)