[MXAPPS-581] Fixes for broken Straight Dope tests.

tests/nightly/straight_dope/straight_dope_test_utils.py

@@ -24,6 +24,7 @@
     the notebook. e.g: `export MXNET_TEST_KERNEL=python2`
 """
 import io
+import logging
 import os
 import re
 import shutil
@@ -40,20 +41,26 @@
 GIT_REPO = 'https://github.com/zackchase/mxnet-the-straight-dope'
 KERNEL = os.getenv('MXNET_TEST_KERNEL', None)
 NOTEBOOKS_DIR = os.path.join(os.path.dirname(__file__), 'tmp_notebook')
+RELATIVE_DATA_PATH_REGEX = r'\.\.\/data\/'  # Regular expression to match the relative data path.
 
 def _test_notebook(notebook, override_epochs=True):
     """Run Jupyter notebook to catch any execution error.
 
     Args:
         notebook : string
             notebook name in folder/notebook format
-        epochs : boolean
+        override_epochs : boolean
             whether or not to override the number of epochs to 1
+
     Returns:
         True if the notebook runs without warning or error.
     """
+    # Some notebooks will fail to run without error if we do not override the data path.
+    _override_data_path(notebook)
+
     if override_epochs:
         _override_epochs(notebook)
+
     return run_notebook(notebook, NOTEBOOKS_DIR, kernel=KERNEL, temp_dir=NOTEBOOKS_DIR)
 
 
@@ -63,29 +70,49 @@ def _override_epochs(notebook):
     Args:
         notebook : string
             notebook name in folder/notebook format
-
     """
     notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) + ".ipynb"
 
-    # Read the notebook and set epochs to num_epochs
+    # Read the notebook and set epochs to num_epochs.
     with io.open(notebook_path, 'r', encoding='utf-8') as f:
         notebook = f.read()
 
-    # Set number of epochs to 1
+    # Set number of epochs to 1.
     modified_notebook = re.sub(EPOCHS_REGEX, 'epochs = 1', notebook)
 
     # Replace the original notebook with the modified one.
     with io.open(notebook_path, 'w', encoding='utf-8') as f:
         f.write(modified_notebook)
 
 
+def _override_data_path(notebook):
+    """Overrides the relative path for the data directory to point to the right place. This is
+    required as we run the notebooks in a different directory hierarchy more suitable for testing.
+
+    Args:
+        notebook : string
+            notebook name in folder/notebook format
+    """
+    notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) + ".ipynb"
+
+    # Read the notebook and set epochs to num_epochs.
+    with io.open(notebook_path, 'r', encoding='utf-8') as f:
+        notebook = f.read()
+
+    # Update the location for the data directory.
+    modified_notebook = re.sub(RELATIVE_DATA_PATH_REGEX, NOTEBOOKS_DIR + '/data/', notebook)
+
+    # Replace the original notebook with the modified one.
+    with io.open(notebook_path, 'w', encoding='utf-8') as f:
+        f.write(modified_notebook)
+
 def _download_straight_dope_notebooks():
     """Downloads the Straight Dope Notebooks.
 
     Returns:
         True if it succeeds in downloading the notebooks without error.
     """
-    print('Cleaning and setting up notebooks directory "{}"'.format(NOTEBOOKS_DIR))
+    logging.info('Cleaning and setting up notebooks directory "{}"'.format(NOTEBOOKS_DIR))
     shutil.rmtree(NOTEBOOKS_DIR, ignore_errors=True)
 
     cmd = [GIT_PATH,
@@ -98,7 +125,7 @@ def _download_straight_dope_notebooks():
     if proc.returncode != 0:
         err_msg = 'Error downloading Straight Dope notebooks.\n'
         err_msg += msg
-        print(err_msg)
+        logging.error(err_msg)
         return False
     return True
 

tests/nightly/straight_dope/test_notebooks_single_gpu.py

@@ -32,9 +32,19 @@
     'chapter01_crashcourse/introduction',
     'chapter01_crashcourse/chapter-one-problem-set',
     'chapter02_supervised-learning/environment',
+    'chapter03_deep-neural-networks/kaggle-gluon-kfold',
     'chapter07_distributed-learning/multiple-gpus-scratch',
     'chapter07_distributed-learning/multiple-gpus-gluon',
-    'chapter07_distributed-learning/training-with-multiple-machines'
+    'chapter07_distributed-learning/training-with-multiple-machines',
+    'chapter12_time-series/intro-forecasting-gluon',
+    'chapter12_time-series/intro-forecasting-2-gluon',
+    'chapter13_unsupervised-learning/vae-gluon',
+    'chapter18_variational-methods-and-uncertainty/bayes-by-backprop-rnn',
+    'chapter17_deep-reinforcement-learning/DQN',
+    'chapter17_deep-reinforcement-learning/DDQN',
+    'chapter19_graph-neural-networks/Graph-Neural-Networks',
+    'chapter16_tensor_methods/tensor_basics',
+    'cheatsheets/kaggle-gluon-kfold'
 ]
 
 
@@ -91,10 +101,8 @@ def test_linear_regression_scratch(self):
     def test_linear_regression_gluon(self):
         assert _test_notebook('chapter02_supervised-learning/linear-regression-gluon')
 
-    # TODO(vishaalk): There is a relative file path needs to be fixed so that the
-    # python code can be run from another directory.
-    #def test_logistic_regression_gluon(self):
-    #    assert _test_notebook('chapter02_supervised-learning/logistic-regression-gluon')
+    def test_logistic_regression_gluon(self):
+        assert _test_notebook('chapter02_supervised-learning/logistic-regression-gluon')
 
     def test_softmax_regression_scratch(self):
         assert _test_notebook('chapter02_supervised-learning/softmax-regression-scratch')
@@ -132,9 +140,6 @@ def test_plumbing(self):
     def test_custom_layer(self):
         assert _test_notebook('chapter03_deep-neural-networks/custom-layer')
 
-    #def test_kaggle_gluon_kfold(self):
-    #    assert _test_notebook('chapter03_deep-neural-networks/kaggle-gluon-kfold')
-
     # TODO(vishaalk): Load params and Save params are deprecated warning.
     #def test_serialization(self):
     #    assert _test_notebook('chapter03_deep-neural-networks/serialization')
@@ -162,20 +167,14 @@ def test_cnn_batch_norm_gluon(self):
 
     # Chapter 5
 
-    # TODO(vishaalk): There is a relative file path needs to be fixed so that the
-    # python code can be run from another directory.
-    #def test_simple_rnn(self):
-    #    assert _test_notebook('chapter05_recurrent-neural-networks/simple-rnn')
+    def test_simple_rnn(self):
+        assert _test_notebook('chapter05_recurrent-neural-networks/simple-rnn')
 
-    # TODO(vishaalk): There is a relative file path needs to be fixed so that the
-    # python code can be run from another directory.
-    #def test_lstm_scratch(self):
-    #    assert _test_notebook('chapter05_recurrent-neural-networks/lstm-scratch')
+    def test_lstm_scratch(self):
+        assert _test_notebook('chapter05_recurrent-neural-networks/lstm-scratch')
 
-    # TODO(vishaalk): There is a relative file path needs to be fixed so that the
-    # python code can be run from another directory.
-    #def test_gru_scratch(self):
-    #    assert _test_notebook('chapter05_recurrent-neural-networks/gru-scratch')
+    def test_gru_scratch(self):
+        assert _test_notebook('chapter05_recurrent-neural-networks/gru-scratch')
 
     #def test_rnns_gluon(self):
     #    assert _test_notebook('chapter05_recurrent-neural-networks/rnns-gluon')
@@ -263,19 +262,6 @@ def test_lds_scratch(self):
     #def test_issm_scratch(self):
     #    assert _test_notebook('chapter12_time-series/issm-scratch')
 
-    # TODO(vishaalk): Error: sequential1_batchnorm0_running_mean' has not been initialized
-    # def test_intro_forecasting_gluon(self):
-    #    assert _test_notebook('chapter12_time-series/intro-forecasting-gluon')
-
-    #def test_intro_forecasting_2_gluon(self):
-    #    assert _test_notebook('chapter12_time-series/intro-forecasting-2-gluon')
-
-    # Chapter 13
-
-    # TODO(vishaalk): Load params and Save params are deprecated warning.
-    #def test_vae_gluon(self):
-    #    assert _test_notebook('chapter13_unsupervised-learning/vae-gluon')
-
     # Chapter 14
 
     def test_igan_intro(self):
@@ -287,46 +273,14 @@ def test_dcgan(self):
     def test_generative_adversarial_networks(self):
         assert _test_notebook('chapter14_generative-adversarial-networks/conditional')
 
-    # Chapter 16
-
-    # TODO(vishaalk): Checked failed oshape.Size() != dshape.Size()
-    #def test_tensor_basics(self):
-    #    assert _test_notebook('chapter16_tensor_methods/tensor_basics')
-
     # TODO(vishaalk): Notebook does not appear to be valid JSON.
     #def test_pixel2pixel(self):
     #    assert _test_notebook('chapter14_generative-adversarial-networks/pixel2pixel')
 
-    # Chapter 17
-
-    # TODO(vishaalk): Requires OpenAI Gym. Also uses deprecated load_params.
-    #def test_dqn(self):
-#    assert _test_notebook('chapter17_deep-reinforcement-learning/DQN')
-
-#def test_ddqn(self):
-#    assert _test_notebook('chapter17_deep-reinforcement-learning/DDQN')
-
-# Chapter 18
-
-#def test_bayes_by_backprop(self):
-#    assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop')
-
-#def test_bayes_by_backprop_gluon(self):
-#    assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-gluon')
-
-# TODO(vishaalk): AttributeError: 'list' object has no attribute 'keys'
-#def test_bayes_by_backprop_rnn(self):
-#    assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-rnn')
-
-# Chapter 19
-
-# TODO(vishaalk): Requires deepchem
-#def test_graph_neural_networks(self):
-#    assert _test_notebook('chapter19_graph-neural-networks/Graph-Neural-Networks')
+    # Chapter 18
 
-# Cheatsheets
+    #def test_bayes_by_backprop(self):
+    #    assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop')
 
-# TODO(vishaalk): There is a relative file path needs to be fixed so that the
-# python code can be run from another directory.
-#def test_kaggle_gluon_kfold(self):
-#    assert _test_notebook('cheatsheets/kaggle-gluon-kfold')
+    #def test_bayes_by_backprop_gluon(self):
+    #    assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-gluon')

tests/utils/notebook_test/__init__.py

@@ -21,6 +21,7 @@
     warning or exception.
 """
 import io
+import logging
 import os
 import shutil
 import time
@@ -57,12 +58,12 @@ def run_notebook(notebook, notebook_dir, kernel=None, no_cache=False, temp_dir='
     -------
        Returns true if the workbook runs with no warning or exception.
     """
-
+    logging.info("Running notebook '{}'".format(notebook))
     notebook_path = os.path.join(*([notebook_dir] + notebook.split('/')))
     working_dir = os.path.join(*([temp_dir] + notebook.split('/')))
 
     if no_cache == '1':
-        print("Cleaning and setting up temp directory '{}'".format(working_dir))
+        logging.info("Cleaning and setting up temp directory '{}'".format(working_dir))
         shutil.rmtree(temp_dir, ignore_errors=True)
 
     errors = []
@@ -92,6 +93,6 @@ def run_notebook(notebook, notebook_dir, kernel=None, no_cache=False, temp_dir='
                 if "Warning:" in line:
                     errors.append("Warning:\n" + line)
         if len(errors) > 0:
-            print('\n'.join(errors))
+            logging.error('\n'.join(errors))
             return False
         return True