From e7d726b5440bd5b803faaf52fe147f3fbfe2f958 Mon Sep 17 00:00:00 2001 From: Sandeep Krishnamurthy Date: Tue, 29 Aug 2017 14:42:47 -0700 Subject: [PATCH 1/3] Parallelize Python 2 and 3 unit test cases. --- Jenkinsfile | 64 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fe0151a879d6..b7f431dbb4cf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,11 +76,18 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum } // Python unittest for CPU -def python_ut(docker_type) { +// Python 2 +def python2_ut(docker_type) { timeout(time: max_time, unit: 'MINUTES') { sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete" sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests --with-timer --verbose tests/python/unittest" sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests --with-timer --verbose tests/python/train" + } +} + +// Python 3 +def python3_ut(docker_type) { + timeout(time: max_time, unit: 'MINUTES') { sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete" sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/unittest" } @@ -88,10 +95,17 @@ def python_ut(docker_type) { // GPU test has two parts. 1) run unittest on GPU, 2) compare the results on // both CPU and GPU -def python_gpu_ut(docker_type) { +// Python 2 +def python2_gpu_ut(docker_type) { timeout(time: max_time, unit: 'MINUTES') { sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete" sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests --with-timer --verbose tests/python/gpu" + } +} + +// Python 3 +def python3_gpu_ut(docker_type) { + timeout(time: max_time, unit: 'MINUTES') { sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete" sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/gpu" } @@ -250,31 +264,59 @@ try { } stage('Unit Test') { - parallel 'Python2/3: CPU': { + parallel 'Python2: CPU': { node('mxnetlinux') { - ws('workspace/ut-python-cpu') { + ws('workspace/ut-python2-cpu') { init_git() unpack_lib('cpu') - python_ut('cpu') + python2_ut('cpu') } } }, - 'Python2/3: GPU': { + 'Python3: CPU': { node('mxnetlinux') { - ws('workspace/ut-python-gpu') { + ws('workspace/ut-python3-cpu') { + init_git() + unpack_lib('cpu') + python3_ut('cpu') + } + } + }, + 'Python2: GPU': { + node('mxnetlinux') { + ws('workspace/ut-python2-gpu') { + init_git() + unpack_lib('gpu', mx_lib) + python2_gpu_ut('gpu') + } + } + }, + 'Python3: GPU': { + node('mxnetlinux') { + ws('workspace/ut-python3-gpu') { init_git() unpack_lib('gpu', mx_lib) - python_gpu_ut('gpu') + python3_gpu_ut('gpu') + } + } + }, + 'Python2: MKLML': { + node('mxnetlinux') { + ws('workspace/ut-python2-mklml') { + init_git() + unpack_lib('mklml') + python2_ut('mklml_gpu') + python2_gpu_ut('mklml_gpu') } } }, - 'Python2/3: MKLML': { + 'Python3: MKLML': { node('mxnetlinux') { ws('workspace/ut-python-mklml') { init_git() unpack_lib('mklml') - python_ut('mklml_gpu') - python_gpu_ut('mklml_gpu') + python3_ut('mklml_gpu') + python3_gpu_ut('mklml_gpu') } } }, From df38c6738d668c17816cb92509ca5a185984fc23 Mon Sep 17 00:00:00 2001 From: Sandeep Krishnamurthy Date: Tue, 29 Aug 2017 15:00:22 -0700 Subject: [PATCH 2/3] Parallelize python 2 and 3 unit tests cases in jenkins --- Jenkinsfile | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b7f431dbb4cf..fc08068f9961 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -300,22 +300,38 @@ try { } } }, - 'Python2: MKLML': { + 'Python2: MKLML-CPU': { node('mxnetlinux') { ws('workspace/ut-python2-mklml') { init_git() unpack_lib('mklml') python2_ut('mklml_gpu') + } + } + }, + 'Python2: MKLML-GPU': { + node('mxnetlinux') { + ws('workspace/ut-python2-mklml') { + init_git() + unpack_lib('mklml') python2_gpu_ut('mklml_gpu') } } }, - 'Python3: MKLML': { + 'Python3: MKLML-CPU': { node('mxnetlinux') { ws('workspace/ut-python-mklml') { init_git() unpack_lib('mklml') python3_ut('mklml_gpu') + } + } + }, + 'Python3: MKLML-GPU': { + node('mxnetlinux') { + ws('workspace/ut-python-mklml') { + init_git() + unpack_lib('mklml') python3_gpu_ut('mklml_gpu') } } From 2fc39e4aa906e49ef320dac0a1d31e85ed06da08 Mon Sep 17 00:00:00 2001 From: Sandeep Krishnamurthy Date: Tue, 29 Aug 2017 15:03:27 -0700 Subject: [PATCH 3/3] Parallelize python 2 and 3 unit tests cases in jenkins --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fc08068f9961..bf514bc527d0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -302,7 +302,7 @@ try { }, 'Python2: MKLML-CPU': { node('mxnetlinux') { - ws('workspace/ut-python2-mklml') { + ws('workspace/ut-python2-mklml-cpu') { init_git() unpack_lib('mklml') python2_ut('mklml_gpu') @@ -311,7 +311,7 @@ try { }, 'Python2: MKLML-GPU': { node('mxnetlinux') { - ws('workspace/ut-python2-mklml') { + ws('workspace/ut-python2-mklml-gpu') { init_git() unpack_lib('mklml') python2_gpu_ut('mklml_gpu') @@ -320,7 +320,7 @@ try { }, 'Python3: MKLML-CPU': { node('mxnetlinux') { - ws('workspace/ut-python-mklml') { + ws('workspace/ut-python3-mklml-cpu') { init_git() unpack_lib('mklml') python3_ut('mklml_gpu') @@ -329,7 +329,7 @@ try { }, 'Python3: MKLML-GPU': { node('mxnetlinux') { - ws('workspace/ut-python-mklml') { + ws('workspace/ut-python3-mklml-gpu') { init_git() unpack_lib('mklml') python3_gpu_ut('mklml_gpu')