[WIP] Serde Module for Import/Export of models between Onnx and Mxnet

CMakeLists.txt

@@ -234,6 +234,7 @@ include_directories("include")
 include_directories("mshadow")
 include_directories("3rdparty/cub")
 include_directories("nnvm/include")
+include_directories("nnvm/tvm/include")
 include_directories("dmlc-core/include")
 include_directories("dlpack/include")
 
@@ -696,4 +697,3 @@ endif()
 set(LINT_DIRS "include src plugin cpp-package tests")
 set(EXCLUDE_PATH "src/operator/contrib/ctc_include")
 add_custom_target(mxnet_lint COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} -DLINT_DIRS=${LINT_DIRS} -DPROJECT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -DPROJECT_NAME=mxnet -DEXCLUDE_PATH=${EXCLUDE_PATH} -P ${CMAKE_CURRENT_SOURCE_DIR}/dmlc-core/cmake/lint.cmake)
-

Jenkinsfile

@@ -38,12 +38,12 @@ def init_git() {
   deleteDir()
   retry(5) {
     try {
-      // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of 
+      // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of
       // retries as this will increase the amount of requests and worsen the throttling
       timeout(time: 15, unit: 'MINUTES') {
         checkout scm
-        sh 'git submodule update --init'
-        sh 'git clean -d -f'        
+        sh 'git submodule update --init --recursive'
+        sh 'git clean -d -f'
       }
     } catch (exc) {
       deleteDir()
@@ -61,8 +61,8 @@ def init_git_win() {
       // retries as this will increase the amount of requests and worsen the throttling
       timeout(time: 15, unit: 'MINUTES') {
         checkout scm
-        bat 'git submodule update --init'
-        bat 'git clean -d -f'        
+        bat 'git submodule update --init --recursive'
+        bat 'git clean -d -f'
       }
     } catch (exc) {
       deleteDir()
@@ -332,6 +332,7 @@ try {
           make('build_cuda', flag)
           pack_lib('gpu')
           stash includes: 'build/cpp-package/example/test_score', name: 'cpp_test_score'
+          stash includes: 'build/cpp-package/example/test_optimizer', name: 'cpp_test_optimizer'
         }
       }
     },
@@ -676,6 +677,7 @@ try {
           init_git()
           unpack_lib('gpu')
           unstash 'cpp_test_score'
+          unstash 'cpp_test_optimizer'
           timeout(time: max_time, unit: 'MINUTES') {
             sh "${docker_run} gpu --dockerbinary nvidia-docker cpp-package/tests/ci_test.sh"
           }

Makefile

@@ -91,7 +91,7 @@ ifeq ($(DEBUG), 1)
 else
 	CFLAGS += -O3 -DNDEBUG=1
 endif
-CFLAGS += -I$(ROOTDIR)/mshadow/ -I$(ROOTDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -I$(DLPACK_PATH)/include -Iinclude $(MSHADOW_CFLAGS)
+CFLAGS += -I$(ROOTDIR)/mshadow/ -I$(ROOTDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -I$(DLPACK_PATH)/include -I$(NNVM_PATH)/tvm/include -Iinclude $(MSHADOW_CFLAGS)
 LDFLAGS = -pthread $(MSHADOW_LDFLAGS) $(DMLC_LDFLAGS)
 ifeq ($(DEBUG), 1)
 	NVCCFLAGS += -std=c++11 -Xcompiler -D_FORCE_INLINES -g -G -O0 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS)
@@ -356,7 +356,7 @@ ifeq ($(USE_CUDA), 1)
 		LDFLAGS += -lcuda -lnvrtc
 		CFLAGS += -DMXNET_ENABLE_CUDA_RTC=1
 	endif
-	# Make sure to add stubs as fallback in order to be able to build 
+	# Make sure to add stubs as fallback in order to be able to build
 	# without full CUDA install (especially if run without nvidia-docker)
 	LDFLAGS += -L/usr/local/cuda/lib64/stubs
 	SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-gpu

cpp-package/example/test_optimizer.cpp

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "mxnet-cpp/MxNetCpp.h"
+
+using namespace std;
+using namespace mxnet::cpp;
+
+int main(int argc, char** argv) {
+  // Confirm >1 optimizers can be created w/o error
+  Optimizer* opt = OptimizerRegistry::Find("sgd");
+  opt = OptimizerRegistry::Find("adam");
+  int ret = (opt == 0) ? 1 : 0;
+
+  MXNotifyShutdown();
+  return ret;
+}

cpp-package/include/mxnet-cpp/optimizer.hpp

@@ -125,13 +125,16 @@ inline float Optimizer::GetWD_(int index) {
 }
 
 inline Optimizer* OptimizerRegistry::Find(const std::string& name) {
-  MXNETCPP_REGISTER_OPTIMIZER(sgd, SGDOptimizer);
-  MXNETCPP_REGISTER_OPTIMIZER(ccsgd, SGDOptimizer);  // For backward compatibility
-  MXNETCPP_REGISTER_OPTIMIZER(rmsprop, RMSPropOptimizer);
-  MXNETCPP_REGISTER_OPTIMIZER(adam, AdamOptimizer);
-  MXNETCPP_REGISTER_OPTIMIZER(adagrad, AdaGradOptimizer);
-  MXNETCPP_REGISTER_OPTIMIZER(adadelta, AdaDeltaOptimizer);
-  MXNETCPP_REGISTER_OPTIMIZER(signum, SignumOptimizer);
+  if (cmap().empty()) {
+    // Optimizers should only be registered once
+    MXNETCPP_REGISTER_OPTIMIZER(sgd, SGDOptimizer);
+    MXNETCPP_REGISTER_OPTIMIZER(ccsgd, SGDOptimizer);  // For backward compatibility
+    MXNETCPP_REGISTER_OPTIMIZER(rmsprop, RMSPropOptimizer);
+    MXNETCPP_REGISTER_OPTIMIZER(adam, AdamOptimizer);
+    MXNETCPP_REGISTER_OPTIMIZER(adagrad, AdaGradOptimizer);
+    MXNETCPP_REGISTER_OPTIMIZER(adadelta, AdaDeltaOptimizer);
+    MXNETCPP_REGISTER_OPTIMIZER(signum, SignumOptimizer);
+  }
   auto it = cmap().find(name);
   if (it == cmap().end())
     return nullptr;

cpp-package/tests/ci_test.sh

@@ -22,6 +22,9 @@ export LD_LIBRARY_PATH=$(readlink -f ../../lib):$LD_LIBRARY_PATH
 echo $LD_LIBRARY_PATH
 ls -l ../../lib/
 
+cp ../../build/cpp-package/example/test_optimizer .
+./test_optimizer
+
 cp ../../build/cpp-package/example/test_score .
 ./get_mnist.sh
 ./test_score 0.93

docs/_static/mxnet-theme/index.html

@@ -21,9 +21,9 @@
   <div class="container">
     <div class="row">
       <div class="col-lg-4 col-sm-12">
-        <h3>Apache MXNet 1.0 Released</h3>
-        <p>We're excited to announce the release of MXNet 1.0! Check out the release notes for latest updates.</p>
-        <a href="https://github.com/apache/incubator-mxnet/releases/tag/1.0.0">Learn More</a>
+        <h3>Apache MXNet 1.1.0 Released</h3>
+        <p>We're excited to announce the release of MXNet 1.1.0! Check out the release notes for latest updates.</p>
+        <a href="https://github.com/apache/incubator-mxnet/releases/tag/1.1.0">Learn More</a>
       </div>
       <div class="col-lg-4 col-sm-12">
         <h3>MXNet Model Server</h3>

docs/build_version_doc/AddVersion.py

@@ -57,6 +57,9 @@
         for name in files:
             if not name.endswith('.html'):
                 continue
+            if 'install' in path:
+                print("Skipping this path: {}".format(path))
+                continue
             with open(os.path.join(path, name), 'r') as html_file:
                 content = bs(html_file, 'html.parser')
             navbar = content.find(id="main-nav")
@@ -74,7 +77,7 @@
                 outstr = str(content).replace('&lt;', '<').replace('&gt;', '>')
             # Fix link
             if args.current_version == tag_list[0]:
-                print("Fixing" + os.path.join(path, name))
+                print("Fixing " + os.path.join(path, name))
                 outstr = outstr.replace('https://mxnet.io', 'https://mxnet.incubator.apache.org')
                 outstr = outstr.replace('http://mxnet.io', 'https://mxnet.incubator.apache.org')
             else:

docs/build_version_doc/Dockerfile

@@ -0,0 +1,44 @@
+FROM ubuntu:16.04
+LABEL maintainer="markhama@amazon.com"
+
+# Install dependencies
+RUN apt-get update && apt-get install -y \
+    apt-transport-https \
+    build-essential \
+    ca-certificates \
+    curl \
+    doxygen \
+    git \
+    libatlas-base-dev \
+    liblapack-dev \
+    libopenblas-dev \
+    libopencv-dev \
+    pandoc \
+    python-numpy \
+    python-pip \
+    software-properties-common \
+    unzip \
+    wget
+
+# Setup Scala
+RUN echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823
+RUN apt-get update && apt-get install -y \
+    sbt \
+    scala 
+
+RUN pip install --upgrade pip && pip install \
+    beautifulsoup4 \
+    breathe \
+    CommonMark==0.5.4 \
+    h5py \
+    mock==1.0.1 \
+    pypandoc \
+    recommonmark==0.4.0 \
+    sphinx==1.5.6 
+
+
+COPY *.sh /
+COPY *.py /
+RUN /build_all_version.sh "1.1.0 1.0.0 0.12.1 0.12.0 0.11.0 master"
+RUN /update_all_version.sh "1.1.0 1.0.0 0.12.1 0.12.0 0.11.0 master" 1.1.0 http://mxnet.incubator.apache.org/

docs/build_version_doc/build_all_version.sh

@@ -19,67 +19,65 @@
 
 # This script is for locally building website for all versions
 # Built files are stored in $built
-# Version numbers are stored in $tag_list.
-# Version numbers are ordered from latest to old and final one is master.
+
+# Takes one argument:
+# * tag list - space delimited list of Github tags; Example: "1.1.0 1.0.0 master"
+# Example Usage:
+# ./build_all_version.sh "1.1.0 1.0.0 master"
+
 set -e
 set -x
 
-tag_list="1.1.0 1.0.0 0.12.1 0.12.0 0.11.0 master"
+if [ -z "$1" ]
+  then
+    echo "Please provide a list of version tags you wish to run."
+    exit 1
+  else
+    tag_list="$1"
+    echo "Using these tags: $1"
+fi
 
 mxnet_url="https://github.com/apache/incubator-mxnet.git"
 mxnet_folder="apache_mxnet"
 built="VersionedWeb"
-mkdir $built
-mkdir "$built/versions"
 
-git clone $mxnet_url $mxnet_folder --recursive
-cd "$mxnet_folder/docs"
-tag_file="tag_list.txt"
+if [ ! -d "$mxnet_folder" ]; then
+  mkdir $mxnet_folder
+  git clone $mxnet_url $mxnet_folder --recursive
+fi
 
-# Write all version numbers into $tag_file
-for tag in $tag_list; do
-    if [ $tag != 'master' ]
-    then
-        echo "$tag" >> "$tag_file"
-    fi
-done
+if [ ! -d "$built" ]; then
+  mkdir $built
+  mkdir "$built/versions"
+fi
 
 # Build all versions and use latest version(First version number in $tag_list) as landing page.
-version_num=0
 for tag in $tag_list; do
+    cd "$mxnet_folder"
+    git fetch
     if [ $tag == 'master' ]
-    then
-        git checkout master
-    else
-        git checkout "tags/$tag"
+        then
+            git checkout master
+            git pull
+        else
+            git checkout "tags/$tag"
+    fi
+    if [ $tag == '0.11.0' ]
+      then
+          git checkout master -- docs/mxdoc.py
     fi
-
     git submodule update || exit 1
-    cd ..
     make clean
     cd docs
     make clean
-    make html USE_OPENMP=0 || exit 1
-    python build_version_doc/AddVersion.py --file_path "_build/html/" --current_version "$tag" || exit 1
-
-    if [ $tag != 'master' ]
-    then 
-        python build_version_doc/AddPackageLink.py --file_path "_build/html/get_started/install.html" \
-                                                   --current_version "$tag" || exit 1
-    fi
-
-    if [ $version_num == 0 ]
-    then
-        cp -a _build/html/. "../../$built"
-    else
-        file_loc="../../$built/versions/$tag"
-        mkdir "$file_loc"
-        cp -a _build/html/. "$file_loc"
+    make html USE_OPENMP=1 || exit 1
+    cd ../../
+    file_loc="$built/versions/$tag"
+    if [ -d "$file_loc" ] ; then
+        rm -rf "$file_loc"
     fi
-
-    ((++version_num))
+    mkdir "$file_loc"
+    cp -a "$mxnet_folder/docs/_build/html/." "$file_loc"
 done
-
-mv "$tag_file" "../../$built/tag.txt"
-cd ../..
-rm -rf "$mxnet_folder"
+
+echo "Now you may want to run update_all_version.sh to create the production layout with the versions dropdown and other per-version corrections."