Skip to content
This repository was archived by the owner on Nov 17, 2023. It is now read-only.

[DO NOT MERGE][CI] Upgrade windows CI #20676

Merged
barry-jin merged 1 commit into from
Oct 19, 2021
Merged

[DO NOT MERGE][CI] Upgrade windows CI #20676

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 21 additions & 4 deletions ci/build_windows.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,8 @@ class BuildFlavour(Enum):

def windows_build(args):
logging.info("Using vcvars environment:\n{}".format(args.vcvars))
if args.vcvars_ver:
logging.info("Using vcvars version:\n{}".format(args.vcvars_ver))

path = args.output

Expand All @@ -160,13 +162,22 @@ def windows_build(args):
env = os.environ.copy()
if 'GPU' in args.flavour:
env["CXXFLAGS"] = '/FS /MD /O2 /Ob2'
cmd = "\"{}\" && cmake -GNinja {} {}".format(args.vcvars,
CMAKE_FLAGS[args.flavour],
mxnet_root)
if not args.vcvars_ver:
cmd = "\"{}\" && cmake -GNinja {} {}".format(args.vcvars,
CMAKE_FLAGS[args.flavour],
mxnet_root)
else:
cmd = "\"{}\" -vcvars_ver={} && cmake -GNinja {} {}".format(args.vcvars,
args.vcvars_ver,
CMAKE_FLAGS[args.flavour],
mxnet_root)
logging.info("Generating project with CMake:\n{}".format(cmd))
check_call(cmd, shell=True, env=env)

cmd = "\"{}\" && ninja".format(args.vcvars)
if not args.vcvars_ver:
cmd = "\"{}\" && ninja".format(args.vcvars)
else:
cmd = "\"{}\" -vcvars_ver={} && ninja".format(args.vcvars, args.vcvars_ver)
logging.info("Building:\n{}".format(cmd))

t0 = int(time.time())
Expand Down Expand Up @@ -250,6 +261,12 @@ def main():
default=KNOWN_VCVARS['VS 2019'],
type=str)

parser.add_argument("--vcvars_ver",
help="Optionally specifies the Visual Studio compiler toolset to use.\
By default, the environment is set to use the current Visual Studio compiler toolset.",
default=None,
type=str)

parser.add_argument("--arch",
help="architecture",
default='x64',
Expand Down
3 changes: 1 addition & 2 deletions ci/docker/install/requirements
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ onnxruntime==1.7.0
protobuf==3.14.0
scipy==1.4.1
tabulate==0.7.5
Cython==0.29.7
Cython==0.29.24

# Development dependencies
cpplint==1.3.0
Expand All @@ -51,7 +51,6 @@ decorator==4.4.0
# Used in examples
boto3==1.9.229
h5py==2.10.0
Pillow<6

# Array API Standardization requirements
hypothesis==6.14.0
12 changes: 6 additions & 6 deletions ci/jenkins/Jenkins_steps.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -522,7 +522,7 @@ def compile_windows_cpu(lib_name) {
ws('workspace/build-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git_win()
powershell 'py -3 ci/build_windows.py -f WIN_CPU'
powershell 'py -3 ci/build_windows.py -f WIN_CPU --vcvars_ver 14.28'
stash includes: 'windows_package.7z', name: lib_name
}
}
Expand All @@ -536,7 +536,7 @@ def compile_windows_cpu_onednn(lib_name) {
ws('workspace/build-cpu-onednn') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git_win()
powershell 'py -3 ci/build_windows.py -f WIN_CPU_ONEDNN'
powershell 'py -3 ci/build_windows.py -f WIN_CPU_ONEDNN --vcvars_ver 14.28'
stash includes: 'windows_package.7z', name: lib_name
}
}
Expand All @@ -550,7 +550,7 @@ def compile_windows_cpu_onednn_mkl(lib_name) {
ws('workspace/build-cpu-onednn-mkl') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git_win()
powershell 'py -3 ci/build_windows.py -f WIN_CPU_ONEDNN_MKL'
powershell 'py -3 ci/build_windows.py -f WIN_CPU_ONEDNN_MKL --vcvars_ver 14.28'
stash includes: 'windows_package.7z', name: lib_name
}
}
Expand All @@ -564,7 +564,7 @@ def compile_windows_cpu_mkl(lib_name) {
ws('workspace/build-cpu-mkl') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git_win()
powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKL'
powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKL --vcvars_ver 14.28'
stash includes: 'windows_package.7z', name: lib_name
}
}
Expand All @@ -578,7 +578,7 @@ def compile_windows_gpu(lib_name) {
ws('workspace/build-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git_win()
powershell 'py -3 ci/build_windows.py -f WIN_GPU'
powershell 'py -3 ci/build_windows.py -f WIN_GPU --vcvars_ver 14.28'
stash includes: 'windows_package.7z', name: lib_name
}
}
Expand All @@ -592,7 +592,7 @@ def compile_windows_gpu_onednn(lib_name) {
ws('workspace/build-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git_win()
powershell 'py -3 ci/build_windows.py -f WIN_GPU_ONEDNN'
powershell 'py -3 ci/build_windows.py -f WIN_GPU_ONEDNN --vcvars_ver 14.28'
stash includes: 'windows_package.7z', name: lib_name
}
}
Expand Down
14 changes: 7 additions & 7 deletions ci/windows/test_py3_cpu.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,20 @@ $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
$env:MXNET_SUBGRAPH_VERBOSE=0
$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home')

C:\Python37\Scripts\pip install -r ci\docker\install\requirements
C:\Python37\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_unittest.xml tests\python\unittest
C:\Python38\Scripts\pip install -r ci\docker\install\requirements
C:\Python38\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_unittest.xml tests\python\unittest
if ($LastExitCode -ne 0) { Throw ("Error running parallel unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
C:\Python37\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\python\unittest
C:\Python38\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\python\unittest
if ($LastExitCode -ne 0) { Throw ("Error running serial unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
C:\Python37\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_train.xml tests\python\train
C:\Python38\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_train.xml tests\python\train
if ($LastExitCode -ne 0) { Throw ("Error running parallel train tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
C:\Python37\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_train.xml --cov-append tests\python\train
C:\Python38\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_train.xml --cov-append tests\python\train
if ($LastExitCode -ne 0) { Throw ("Error running serial train tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
# Adding this extra test since it's not possible to set env var on the fly in Windows.
C:\Python37\python.exe -m pytest -v --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\python\unittest\test_operator.py::test_norm
C:\Python38\python.exe -m pytest -v --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\python\unittest\test_operator.py::test_norm
if ($LastExitCode -ne 0) { Throw ("Error running unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) }

# Need to explicitly set the environment variable for MXNET_MEMORY_OPT.
$env:MXNET_MEMORY_OPT=1
C:\Python37\python.exe -m pytest -v --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\python\unittest\test_memory_opt.py
C:\Python38\python.exe -m pytest -v --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\python\unittest\test_memory_opt.py
if ($LastExitCode -ne 0) { Throw ("Error running unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
20 changes: 10 additions & 10 deletions ci/windows/test_py3_gpu.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,29 +24,29 @@ $env:MXNET_SUBGRAPH_VERBOSE=0
$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home')
$env:MXNET_GPU_MEM_POOL_TYPE="Unpooled"

C:\Python37\Scripts\pip install -r ci\docker\install\requirements
C:\Python37\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_unittest.xml tests\python\unittest
C:\Python38\Scripts\pip install -r ci\docker\install\requirements
C:\Python38\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_unittest.xml tests\python\unittest
if ($LastExitCode -ne 0) { Throw ("Error running parallel unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
C:\Python37\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\python\unittest
C:\Python38\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\python\unittest
if ($LastExitCode -ne 0) { Throw ("Error running serial unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) }

C:\Python37\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_operator.xml tests\python\gpu\test_operator_gpu.py
C:\Python38\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_operator.xml tests\python\gpu\test_operator_gpu.py
if ($LastExitCode -ne 0) { Throw ("Error running parallel tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
C:\Python37\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_operator.xml --cov-append tests\python\gpu\test_operator_gpu.py
C:\Python38\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_operator.xml --cov-append tests\python\gpu\test_operator_gpu.py
if ($LastExitCode -ne 0) { Throw ("Error running serial tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }

C:\Python37\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_train.xml tests\python\train
C:\Python38\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_train.xml tests\python\train
if ($LastExitCode -ne 0) { Throw ("Error running parallel tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
C:\Python37\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_train.xml --cov-append tests\python\train
C:\Python38\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_train.xml --cov-append tests\python\train
if ($LastExitCode -ne 0) { Throw ("Error running serial tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }

# Adding this extra test since it's not possible to set env var on the fly in Windows.
C:\Python37\python.exe -m pytest -v --durations=50 --cov-report xml:tests_operator.xml --cov-append tests\python\gpu\test_operator_gpu.py::test_norm
C:\Python38\python.exe -m pytest -v --durations=50 --cov-report xml:tests_operator.xml --cov-append tests\python\gpu\test_operator_gpu.py::test_norm
if ($LastExitCode -ne 0) { Throw ("Error running tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
C:\Python37\python.exe -m pytest -v --durations=50 --cov-report xml:tests_tvm_op.xml tests\python\gpu\test_tvm_op_gpu.py
C:\Python38\python.exe -m pytest -v --durations=50 --cov-report xml:tests_tvm_op.xml tests\python\gpu\test_tvm_op_gpu.py
if ($LastExitCode -ne 0) { Throw ("Error running TVM op tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }

# Need to explicitly set the environment variable for MXNET_MEMORY_OPT.
$env:MXNET_MEMORY_OPT=1
C:\Python37\python.exe -m pytest -v --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\python\unittest\test_memory_opt.py
C:\Python38\python.exe -m pytest -v --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\python\unittest\test_memory_opt.py
if ($LastExitCode -ne 0) { Throw ("Error running memory optimization tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }