diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy index e7aeae935d06..8ecc7e193b97 100644 --- a/ci/Jenkinsfile_utils.groovy +++ b/ci/Jenkinsfile_utils.groovy @@ -255,6 +255,7 @@ def assign_node_labels(args) { // knowing about the limitations. NODE_LINUX_CPU = args.linux_cpu NODE_LINUX_GPU = args.linux_gpu + NODE_LINUX_GPU_G4 = args.linux_gpu_g4 NODE_LINUX_GPU_P3 = args.linux_gpu_p3 NODE_WINDOWS_CPU = args.windows_cpu NODE_WINDOWS_GPU = args.windows_gpu diff --git a/ci/build.py b/ci/build.py index cbc41218f042..8c2a6e9ac20b 100755 --- a/ci/build.py +++ b/ci/build.py @@ -66,23 +66,18 @@ def get_dockerfile(platform: str, path=get_dockerfiles_path()) -> str: return os.path.join(path, "Dockerfile.{0}".format(platform)) -def get_docker_binary(use_nvidia_docker: bool) -> str: - return "nvidia-docker" if use_nvidia_docker else "docker" - - -def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, no_cache: bool, +def build_docker(platform: str, registry: str, num_retries: int, no_cache: bool, cache_intermediate: bool) -> str: """ Build a container for the given platform :param platform: Platform - :param docker_binary: docker binary to use (docker/nvidia-docker) :param registry: Dockerhub registry name :param num_retries: Number of retries to build the docker image :param no_cache: pass no-cache to docker to rebuild the images :return: Id of the top level image """ tag = get_docker_tag(platform=platform, registry=registry) - logging.info("Building docker container tagged '%s' with %s", tag, docker_binary) + logging.info("Building docker container tagged '%s'", tag) # # We add a user with the same group as the executing non-root user so files created in the # container match permissions of the local user. Same for the group. @@ -99,7 +94,7 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries: # # This doesn't work with multi head docker files. # - cmd = [docker_binary, "build", + cmd = ["docker", "build", "-f", get_dockerfile(platform), "--build-arg", "USER_ID={}".format(os.getuid()), "--build-arg", "GROUP_ID={}".format(os.getgid())] @@ -119,19 +114,19 @@ def run_cmd(): run_cmd() # Get image id by reading the tag. It's guaranteed (except race condition) that the tag exists. Otherwise, the # check_call would have failed - image_id = _get_local_image_id(docker_binary=docker_binary, docker_tag=tag) + image_id = _get_local_image_id(docker_tag=tag) if not image_id: raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag)) return image_id -def _get_local_image_id(docker_binary, docker_tag): +def _get_local_image_id(docker_tag): """ Get the image id of the local docker layer with the passed tag :param docker_tag: docker tag :return: Image id as string or None if tag does not exist """ - cmd = [docker_binary, "images", "-q", docker_tag] + cmd = ["docker", "images", "-q", docker_tag] image_id_b = check_output(cmd) image_id = image_id_b.decode('utf-8').strip() if not image_id: @@ -196,8 +191,9 @@ def container_run(docker_client: SafeDockerClient, # Equivalent command docker_cmd_list = [ - get_docker_binary(nvidia_runtime), + "docker", 'run', + "--gpus all" if nvidia_runtime else "", "--cap-add", "SYS_PTRACE", # Required by ASAN '--rm', @@ -352,7 +348,6 @@ def main() -> int: args = parser.parse_args() command = list(chain(*args.command)) - docker_binary = get_docker_binary(args.nvidiadocker) docker_client = SafeDockerClient() environment = dict([(e.split('=')[:2] if '=' in e else (e, os.environ[e])) @@ -366,7 +361,7 @@ def main() -> int: if args.docker_registry: load_docker_cache(tag=tag, docker_registry=args.docker_registry) if not args.run_only: - build_docker(platform=platform, docker_binary=docker_binary, registry=args.docker_registry, + build_docker(platform=platform, registry=args.docker_registry, num_retries=args.docker_build_retries, no_cache=args.no_cache, cache_intermediate=args.cache_intermediate) else: @@ -410,7 +405,7 @@ def main() -> int: for platform in platforms: tag = get_docker_tag(platform=platform, registry=args.docker_registry) load_docker_cache(tag=tag, docker_registry=args.docker_registry) - build_docker(platform, docker_binary=docker_binary, registry=args.docker_registry, + build_docker(platform, registry=args.docker_registry, num_retries=args.docker_build_retries, no_cache=args.no_cache) if args.build_only: continue diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu101 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu101 index 717a5aa47f0e..a17261bfa820 100644 --- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu101 +++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu101 @@ -79,4 +79,3 @@ RUN /work/ubuntu_adduser.sh COPY runtime_functions.sh /work/ WORKDIR /work/mxnet -ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/compat diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 4b544e4b63ef..4523e1f017f5 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -767,7 +767,7 @@ build_ubuntu_gpu_mkldnn() { USE_CUDA=1 \ USE_CUDA_PATH=/usr/local/cuda \ USE_CUDNN=1 \ - USE_TVM_OP=1 \ + USE_TVM_OP=0 \ CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ USE_SIGNAL_HANDLER=1 \ -j$(nproc) @@ -784,7 +784,7 @@ build_ubuntu_gpu_mkldnn_nocudnn() { USE_CUDA=1 \ USE_CUDA_PATH=/usr/local/cuda \ USE_CUDNN=0 \ - USE_TVM_OP=1 \ + USE_TVM_OP=0 \ CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ USE_SIGNAL_HANDLER=1 \ -j$(nproc) @@ -799,7 +799,7 @@ build_ubuntu_gpu_cuda101_cudnn7() { USE_CUDA=1 \ USE_CUDA_PATH=/usr/local/cuda \ USE_CUDNN=1 \ - USE_TVM_OP=1 \ + USE_TVM_OP=0 \ USE_CPP_PACKAGE=1 \ USE_DIST_KVSTORE=1 \ CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ @@ -827,26 +827,6 @@ build_ubuntu_gpu_cuda101_cudnn7_mkldnn_cpp_test() { make cython PYTHON=python3 } -build_ubuntu_gpu_cuda101_cudnn7_no_tvm_op() { - set -ex - build_ccache_wrappers - make \ - DEV=1 \ - USE_BLAS=openblas \ - USE_MKLDNN=0 \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=1 \ - USE_TVM_OP=0 \ - USE_CPP_PACKAGE=1 \ - USE_DIST_KVSTORE=1 \ - CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ - USE_SIGNAL_HANDLER=1 \ - -j$(nproc) - - make cython PYTHON=python3 -} - build_ubuntu_amalgamation() { set -ex # Amalgamation can not be run with -j nproc @@ -874,7 +854,7 @@ build_ubuntu_gpu_cmake_mkldnn() { -DUSE_SIGNAL_HANDLER=ON \ -DUSE_CUDA=1 \ -DUSE_CUDNN=1 \ - -DUSE_TVM_OP=1 \ + -DUSE_TVM_OP=0 \ -DPython3_EXECUTABLE=/usr/bin/python3 \ -DUSE_MKLML_MKL=1 \ -DCMAKE_BUILD_TYPE=Release \ @@ -893,7 +873,7 @@ build_ubuntu_gpu_cmake() { -DUSE_SIGNAL_HANDLER=ON \ -DUSE_CUDA=ON \ -DUSE_CUDNN=ON \ - -DUSE_TVM_OP=ON \ + -DUSE_TVM_OP=OFF \ -DPython3_EXECUTABLE=/usr/bin/python3 \ -DUSE_MKL_IF_AVAILABLE=OFF \ -DUSE_MKLML_MKL=OFF \ @@ -916,7 +896,7 @@ build_ubuntu_gpu_cmake_no_rtc() { -DUSE_SIGNAL_HANDLER=ON \ -DUSE_CUDA=ON \ -DUSE_CUDNN=ON \ - -DUSE_TVM_OP=ON \ + -DUSE_TVM_OP=OFF \ -DPython3_EXECUTABLE=/usr/bin/python3 \ -DUSE_MKL_IF_AVAILABLE=OFF \ -DUSE_MKLML_MKL=OFF \ @@ -932,29 +912,6 @@ build_ubuntu_gpu_cmake_no_rtc() { ninja } -build_ubuntu_gpu_cmake_no_tvm_op() { - set -ex - cd /work/build - build_ccache_wrappers - cmake \ - -DUSE_SIGNAL_HANDLER=ON \ - -DUSE_CUDA=ON \ - -DUSE_CUDNN=ON \ - -DUSE_TVM_OP=OFF \ - -DPython3_EXECUTABLE=/usr/bin/python3 \ - -DUSE_MKL_IF_AVAILABLE=OFF \ - -DUSE_MKLML_MKL=OFF \ - -DUSE_MKLDNN=OFF \ - -DUSE_DIST_KVSTORE=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DMXNET_CUDA_ARCH="$CI_CMAKE_CUDA_ARCH" \ - -DBUILD_CYTHON_MODULES=1 \ - -G Ninja \ - /work/mxnet - - ninja -} - build_ubuntu_cpu_large_tensor() { set -ex cd /work/build @@ -980,7 +937,7 @@ build_ubuntu_gpu_large_tensor() { -DUSE_SIGNAL_HANDLER=ON \ -DUSE_CUDA=ON \ -DUSE_CUDNN=ON \ - -DUSE_TVM_OP=ON \ + -DUSE_TVM_OP=OFF \ -DPython3_EXECUTABLE=/usr/bin/python3 \ -DUSE_MKL_IF_AVAILABLE=OFF \ -DUSE_MKLML_MKL=OFF \ diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 98c774b284ec..c4fd96e65ac0 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -143,7 +143,7 @@ def compile_unix_int64_cpu() { def compile_unix_int64_gpu() { return ['GPU: USE_INT64_TENSOR_SIZE': { - node(NODE_LINUX_GPU) { + node(NODE_LINUX_GPU_G4) { ws('workspace/build-gpu-int64') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() @@ -253,20 +253,6 @@ def compile_unix_full_gpu_mkldnn_cpp_test() { }] } -def compile_unix_full_gpu_no_tvm_op() { - return ['GPU: CUDA10.1+cuDNN7 TVM_OP OFF': { - node(NODE_LINUX_CPU) { - ws('workspace/build-gpu-no-tvm-op') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda101_cudnn7_no_tvm_op', false) - utils.pack_lib('gpu_no_tvm_op', mx_lib_cpp_examples_no_tvm_op) - } - } - } - }] -} - def compile_unix_cmake_mkldnn_gpu() { return ['GPU: CMake MKLDNN': { node(NODE_LINUX_CPU) { @@ -295,19 +281,6 @@ def compile_unix_cmake_gpu() { }] } -def compile_unix_cmake_gpu_no_tvm_op() { - return ['GPU: CMake TVM_OP OFF': { - node(NODE_LINUX_CPU) { - ws('workspace/build-cmake-gpu-no-tvm-op') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_gpu_cu101', 'build_ubuntu_gpu_cmake_no_tvm_op', false) - } - } - } - }] -} - def compile_unix_cmake_gpu_no_rtc() { return ['GPU: CMake CUDA RTC OFF': { node(NODE_LINUX_CPU) { @@ -750,7 +723,7 @@ def test_unix_python3_mkl_cpu() { def test_unix_python3_gpu() { return ['Python3: GPU': { - node(NODE_LINUX_GPU) { + node(NODE_LINUX_GPU_G4) { ws('workspace/ut-python3-gpu') { try { utils.unpack_and_init('gpu', mx_lib_cython) @@ -764,22 +737,6 @@ def test_unix_python3_gpu() { }] } -def test_unix_python3_gpu_no_tvm_op() { - return ['Python3: GPU TVM_OP OFF': { - node(NODE_LINUX_GPU) { - ws('workspace/ut-python3-gpu-no-tvm-op') { - try { - utils.unpack_and_init('gpu_no_tvm_op', mx_lib_cpp_examples_no_tvm_op) - python3_gpu_ut_cython('ubuntu_gpu_cu101') - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_gpu.xml') - } - } - } - }] -} - def test_unix_python3_quantize_gpu() { return ['Python3: Quantize GPU': { node(NODE_LINUX_GPU_P3) { @@ -866,7 +823,7 @@ def test_unix_python3_mkldnn_mkl_cpu() { def test_unix_python3_mkldnn_gpu() { return ['Python3: MKLDNN-GPU': { - node(NODE_LINUX_GPU) { + node(NODE_LINUX_GPU_G4) { ws('workspace/ut-python3-mkldnn-gpu') { try { utils.unpack_and_init('mkldnn_gpu', mx_mkldnn_lib) @@ -882,7 +839,7 @@ def test_unix_python3_mkldnn_gpu() { def test_unix_python3_mkldnn_nocudnn_gpu() { return ['Python3: MKLDNN-GPU-NOCUDNN': { - node(NODE_LINUX_GPU) { + node(NODE_LINUX_GPU_G4) { ws('workspace/ut-python3-mkldnn-gpu-nocudnn') { try { utils.unpack_and_init('mkldnn_gpu_nocudnn', mx_mkldnn_lib) @@ -916,7 +873,7 @@ def test_unix_python3_tensorrt_gpu() { def test_unix_python3_integration_gpu() { return ['Python Integration GPU': { - node(NODE_LINUX_GPU) { + node(NODE_LINUX_GPU_G4) { ws('workspace/it-python-gpu') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('gpu', mx_lib) @@ -944,8 +901,8 @@ def test_unix_caffe_gpu() { } def test_unix_cpp_package_gpu() { - return ['cpp-package GPU': { - node(NODE_LINUX_GPU) { + return ['cpp-package GPU Makefile': { + node(NODE_LINUX_GPU_G4) { ws('workspace/it-cpp-package') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('gpu', mx_lib_cpp_examples) @@ -958,8 +915,8 @@ def test_unix_cpp_package_gpu() { } def test_unix_capi_cpp_package() { - return ['capi-cpp-package GPU': { - node(NODE_LINUX_GPU) { + return ['capi-cpp-package GPU Makefile': { + node(NODE_LINUX_GPU_G4) { ws('workspace/it-capi-cpp-package') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('gpu_mkldnn_cpp_test', mx_lib_cpp_capi) @@ -1000,8 +957,8 @@ def test_unix_scala_mkldnn_cpu(){ } def test_unix_scala_gpu() { - return ['Scala: GPU': { - node(NODE_LINUX_GPU) { + return ['Scala: GPU Makefile': { + node(NODE_LINUX_GPU_G4) { ws('workspace/ut-scala-gpu') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('gpu', mx_lib) @@ -1084,7 +1041,7 @@ def test_unix_perl_cpu() { def test_unix_cpp_gpu() { return ['Cpp: GPU': { - node(NODE_LINUX_GPU) { + node(NODE_LINUX_GPU_G4) { ws('workspace/ut-cpp-gpu') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('cmake_gpu', mx_cmake_lib) @@ -1125,8 +1082,8 @@ def test_unix_cpp_cpu() { } def test_unix_perl_gpu() { - return ['Perl: GPU': { - node(NODE_LINUX_GPU) { + return ['Perl: GPU Makefile': { + node(NODE_LINUX_GPU_G4) { ws('workspace/ut-perl-gpu') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('gpu', mx_lib) @@ -1140,7 +1097,7 @@ def test_unix_perl_gpu() { def test_unix_r_gpu() { return ['R: GPU': { - node(NODE_LINUX_GPU) { + node(NODE_LINUX_GPU_G4) { ws('workspace/ut-r-gpu') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('gpu', mx_lib) @@ -1208,7 +1165,7 @@ def test_unix_distributed_kvstore_cpu() { def test_unix_distributed_kvstore_gpu() { return ['dist-kvstore tests GPU': { - node(NODE_LINUX_GPU) { + node(NODE_LINUX_GPU_G4) { ws('workspace/it-dist-kvstore') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('gpu', mx_lib) diff --git a/ci/jenkins/Jenkinsfile_unix_gpu b/ci/jenkins/Jenkinsfile_unix_gpu index f8c28d5d1994..5e26a9f41380 100644 --- a/ci/jenkins/Jenkinsfile_unix_gpu +++ b/ci/jenkins/Jenkinsfile_unix_gpu @@ -29,7 +29,7 @@ node('utility') { utils = load('ci/Jenkinsfile_utils.groovy') custom_steps = load('ci/jenkins/Jenkins_steps.groovy') } -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3') +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', linux_gpu_g4: 'mxnetlinux-gpu-g4') utils.main_wrapper( core_logic: { @@ -41,8 +41,6 @@ core_logic: { custom_steps.compile_unix_cmake_gpu(), custom_steps.compile_unix_tensorrt_gpu(), custom_steps.compile_unix_int64_gpu(), - custom_steps.compile_unix_full_gpu_no_tvm_op(), - custom_steps.compile_unix_cmake_gpu_no_tvm_op(), custom_steps.compile_unix_cmake_gpu_no_rtc(), custom_steps.compile_unix_full_gpu_mkldnn_cpp_test() ]) @@ -63,7 +61,6 @@ core_logic: { custom_steps.test_unix_distributed_kvstore_gpu(), custom_steps.test_static_python_gpu(), custom_steps.test_static_python_gpu_cmake(), - custom_steps.test_unix_python3_gpu_no_tvm_op(), custom_steps.test_unix_capi_cpp_package(), // Disabled due to: https://github.com/apache/incubator-mxnet/issues/11407 diff --git a/contrib/clojure-package/examples/bert/get_bert_data.sh b/contrib/clojure-package/examples/bert/get_bert_data.sh index 10ed8e9a1f8e..e640857d1193 100755 --- a/contrib/clojure-package/examples/bert/get_bert_data.sh +++ b/contrib/clojure-package/examples/bert/get_bert_data.sh @@ -28,5 +28,5 @@ if [ ! -d "$data_path" ]; then curl https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci/BertQA/static_bert_qa-symbol.json -o $data_path/static_bert_qa-symbol.json curl https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci/BertQA/static_bert_base_net-symbol.json -o $data_path/static_bert_base_net-symbol.json curl https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci/BertQA/static_bert_base_net-0000.params -o $data_path/static_bert_base_net-0000.params - curl https://raw.githubusercontent.com/dmlc/gluon-nlp/master/docs/examples/sentence_embedding/dev.tsv -o $data_path/dev.tsv + curl https://raw.githubusercontent.com/dmlc/gluon-nlp/d75185ec7eb1eb082ee92992be8677666aaf7ec7/docs/examples/sentence_embedding/dev.tsv -o $data_path/dev.tsv fi