[MXAPPS-581] Nightly Straight Dope tests.

ci/docker/runtime_functions.sh

@@ -895,6 +895,44 @@ nightly_test_javascript() {
     make -C /work/mxnet/amalgamation libmxnet_predict.js MIN=1 EMCC=/work/deps/emscripten/emcc
 }
 
+# Nightly 'MXNet: The Straight Dope' Single-GPU Tests
+nightly_straight_dope_python2_single_gpu_tests() {
+    set -ex
+    cd /work/mxnet/tests/nightly/straight_dope
+    export PYTHONPATH=/work/mxnet/python/
+    export MXNET_TEST_KERNEL=python2
+    nosetests-2.7 --with-xunit --xunit-file nosetests_straight_dope_python2_single_gpu.xml \
+      test_notebooks_single_gpu.py --nologcapture
+}
+
+nightly_straight_dope_python3_single_gpu_tests() {
+    set -ex
+    cd /work/mxnet/tests/nightly/straight_dope
+    export PYTHONPATH=/work/mxnet/python/
+    export MXNET_TEST_KERNEL=python3
+    nosetests-3.4 --with-xunit --xunit-file nosetests_straight_dope_python3_single_gpu.xml \
+      test_notebooks_single_gpu.py --nologcapture
+}
+
+# Nightly 'MXNet: The Straight Dope' Multi-GPU Tests
+nightly_straight_dope_python2_multi_gpu_tests() {
+    set -ex
+    cd /work/mxnet/tests/nightly/straight_dope
+    export PYTHONPATH=/work/mxnet/python/
+    export MXNET_TEST_KERNEL=python2
+    nosetests-2.7 --with-xunit --xunit-file nosetests_straight_dope_python2_multi_gpu.xml \
+      test_notebooks_multi_gpu.py --nologcapture
+}
+
+nightly_straight_dope_python3_multi_gpu_tests() {
+    set -ex
+    cd /work/mxnet/tests/nightly/straight_dope
+    export PYTHONPATH=/work/mxnet/python/
+    export MXNET_TEST_KERNEL=python3
+    nosetests-3.4 --with-xunit --xunit-file nosetests_straight_dope_python3_multi_gpu.xml \
+      test_notebooks_multi_gpu.py --nologcapture
+}
+
 # Deploy
 
 deploy_docs() {

tests/nightly/JenkinsfileForBinaries

@@ -98,6 +98,42 @@ try {
           docker_run('ubuntu_nightly_gpu', 'nightly_test_KVStore_singleNode', true) 
         }
       }
+    },
+    'StraightDope: Python2 Single-GPU': {
+      node('mxnetlinux-gpu-p3') {
+        ws('workspace/straight_dope-single_gpu') {
+          init_git()
+          unpack_lib('gpu', mx_lib)
+          docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python2_single_gpu_tests', true)
+        }
+      }
+    },
+    'StraightDope: Python2 Multi-GPU': {
+      node('mxnetlinux-gpu') {
+        ws('workspace/straight_dope-multi_gpu') {
+          init_git()
+          unpack_lib('gpu', mx_lib)
+          docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python2_multi_gpu_tests', true)
+        }
+      }
+    },
+    'StraightDope: Python3 Single-GPU': {
+      node('mxnetlinux-gpu-p3') {
+        ws('workspace/straight_dope-single_gpu') {
+          init_git()
+          unpack_lib('gpu', mx_lib)
+          docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python3_single_gpu_tests', true)
+        }
+      }
+    },
+    'StraightDope: Python3 Multi-GPU': {
+      node('mxnetlinux-gpu') {
+        ws('workspace/straight_dope-multi_gpu') {
+          init_git()
+          unpack_lib('gpu', mx_lib)
+          docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python3_multi_gpu_tests', true)
+        }
+      }
     }
   }
 } catch (caughtError) {

tests/nightly/straight_dope/README.md

@@ -0,0 +1,7 @@
+# Nightly Tests for MXNet: The Straight Dope
+
+These are some longer running tests that are scheduled to run every night. 
+
+### Description
+These tests verify that the straight dope tutorials run without error. They are
+run on both single and multi-gpu configurations.

tests/nightly/straight_dope/straight_dope_test_utils.py

@@ -0,0 +1,130 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#pylint: disable=no-member, too-many-locals, too-many-branches, no-self-use, broad-except, lost-exception, too-many-nested-blocks, too-few-public-methods, invalid-name
+"""
+    This file tests and ensures that all straight dope notebooks run
+    without warning or exception.
+
+    env variable MXNET_TEST_KERNEL controls which kernel to use when running
+    the notebook. e.g: `export MXNET_TEST_KERNEL=python2`
+"""
+import io
+import os
+import re
+import shutil
+import subprocess
+import sys
+from time import sleep
+
+#TODO(vishaalk): Find a cleaner way to import this notebook.
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'utils'))
+from notebook_test import run_notebook
+
+EPOCHS_REGEX = r'epochs\s+=\s+[0-9]+'  # Regular expression that matches 'epochs = #'
+GIT_PATH = '/usr/bin/git'
+GIT_REPO = 'https://github.com/zackchase/mxnet-the-straight-dope'
+KERNEL = os.getenv('MXNET_TEST_KERNEL', None)
+NOTEBOOKS_DIR = os.path.join(os.path.dirname(__file__), 'tmp_notebook')
+
+def _test_notebook(notebook, override_epochs=True):
+    """Run Jupyter notebook to catch any execution error.
+
+    Args:
+        notebook : string
+            notebook name in folder/notebook format
+        epochs : boolean
+            whether or not to override the number of epochs to 1
+    Returns:
+        True if the notebook runs without warning or error.
+    """
+    if override_epochs:
+        _override_epochs(notebook)
+    return run_notebook(notebook, NOTEBOOKS_DIR, kernel=KERNEL, temp_dir=NOTEBOOKS_DIR)
+
+
+def _override_epochs(notebook):
+    """Overrides the number of epochs in the notebook to 1 epoch. Note this operation is idempotent.
+
+    Args:
+        notebook : string
+            notebook name in folder/notebook format
+
+    """
+    notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) + ".ipynb"
+
+    # Read the notebook and set epochs to num_epochs
+    with io.open(notebook_path, 'r', encoding='utf-8') as f:
+        notebook = f.read()
+
+    # Set number of epochs to 1
+    modified_notebook = re.sub(EPOCHS_REGEX, 'epochs = 1', notebook)
+
+    # Replace the original notebook with the modified one.
+    with io.open(notebook_path, 'w', encoding='utf-8') as f:
+        f.write(modified_notebook)
+
+
+def _download_straight_dope_notebooks():
+    """Downloads the Straight Dope Notebooks.
+
+    Returns:
+        True if it succeeds in downloading the notebooks without error.
+    """
+    print('Cleaning and setting up notebooks directory "{}"'.format(NOTEBOOKS_DIR))
+    shutil.rmtree(NOTEBOOKS_DIR, ignore_errors=True)
+
+    cmd = [GIT_PATH,
+           'clone',
+           GIT_REPO,
+           NOTEBOOKS_DIR]
+
+    proc, msg = _run_command(cmd)
+
+    if proc.returncode != 0:
+        err_msg = 'Error downloading Straight Dope notebooks.\n'
+        err_msg += msg
+        print(err_msg)
+        return False
+    return True
+
+def _run_command(cmd, timeout_secs=300):
+    """ Runs a command with a specified timeout.
+
+    Args:
+        cmd : list of string
+            The command with arguments to run.
+        timeout_secs: integer
+            The timeout in seconds
+
+    Returns:
+        Returns the process and the output as a pair.
+    """
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT)
+
+    for i in range(timeout_secs):
+        sleep(1)
+        if proc.poll() is not None:
+            (out, _) = proc.communicate()
+            return proc, out.decode('utf-8')
+
+    proc.kill()
+    return proc, "Timeout of %s secs exceeded." % timeout_secs
+

tests/nightly/straight_dope/test_notebooks_multi_gpu.py

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#pylint: disable=no-member, too-many-locals, too-many-branches, no-self-use, broad-except, lost-exception, too-many-nested-blocks, too-few-public-methods, invalid-name, missing-docstring
+"""
+    This file tests that the notebooks requiring multi GPUs run without
+    warning or exception.
+"""
+import unittest
+from straight_dope_test_utils import _test_notebook
+from straight_dope_test_utils import _download_straight_dope_notebooks
+
+class StraightDopeMultiGpuTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        assert _download_straight_dope_notebooks()
+
+    # Chapter 7
+
+    # TODO(vishaalk): module 'mxnet.gluon' has no attribute 'autograd'
+    #def test_multiple_gpus_scratch(self):
+    #    assert _test_notebook('chapter07_distributed-learning/multiple-gpus-scratch')
+
+    def test_multiple_gpus_gluon(self):
+        assert _test_notebook('chapter07_distributed-learning/multiple-gpus-gluon')
+
+    # TODO(vishaalk): Do a dry run, and then enable.
+    #def test_training_with_multiple_machines(self):
+    #    assert _test_notebook('chapter07_distributed-learning/training-with-multiple-machines')
+
+    # Chapter 8
+
+    # TODO(vishaalk): Module skimage needs to be added to docker image.
+    # def test_fine_tuning(self):
+    #    assert _test_notebook('chapter08_computer-vision/fine-tuning')