diff --git a/Jenkinsfile b/Jenkinsfile index 135f64dc1d94..e95cd3e8f33f 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-10-19T13:44:32.119961 +// Generated at 2022-11-10T11:23:40.325082 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -661,6 +661,13 @@ def ci_setup(image) { ) } +def standalone_crt_build(image) { + sh ( + script: "${docker_run} ${image} ./tests/scripts/task_standalone_crt.sh", + label: 'Build standalone_crt', + ) +} + def python_unittest(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", @@ -816,6 +823,7 @@ stage('Build') { label: 'Upload artifacts to S3', ) + standalone_crt_build(ci_cpu) ci_setup(ci_cpu) // sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh" // TODO(@jroesch): need to resolve CI issue will turn back on in follow up patch @@ -874,6 +882,7 @@ stage('Build') { ) make(ci_wasm, 'build', '-j2') cpp_unittest(ci_wasm) + standalone_crt_build(ci_wasm) ci_setup(ci_wasm) sh ( script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh", @@ -1122,6 +1131,7 @@ def shard_run_unittest_GPU_1_of_3() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) cpp_unittest(ci_gpu) sh ( @@ -1187,6 +1197,7 @@ def shard_run_unittest_GPU_2_of_3() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh", @@ -1255,6 +1266,7 @@ def shard_run_unittest_GPU_3_of_3() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", @@ -1322,6 +1334,7 @@ def shard_run_integration_CPU_1_of_4() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_cpu) ci_setup(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", @@ -1384,6 +1397,7 @@ def shard_run_integration_CPU_2_of_4() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_cpu) ci_setup(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", @@ -1446,6 +1460,7 @@ def shard_run_integration_CPU_3_of_4() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_cpu) ci_setup(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", @@ -1508,6 +1523,7 @@ def shard_run_integration_CPU_4_of_4() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_cpu) ci_setup(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", @@ -1569,6 +1585,7 @@ def shard_run_python_i386_1_of_3() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_i386) ci_setup(ci_i386) cpp_unittest(ci_i386) python_unittest(ci_i386) @@ -1631,6 +1648,7 @@ def shard_run_python_i386_2_of_3() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_i386) ci_setup(ci_i386) python_unittest(ci_i386) sh ( @@ -1693,6 +1711,7 @@ def shard_run_python_i386_3_of_3() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_i386) ci_setup(ci_i386) python_unittest(ci_i386) sh ( @@ -2237,6 +2256,7 @@ def shard_run_integration_aarch64_1_of_4() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_arm) ci_setup(ci_arm) python_unittest(ci_arm) sh ( @@ -2298,6 +2318,7 @@ def shard_run_integration_aarch64_2_of_4() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_arm) ci_setup(ci_arm) python_unittest(ci_arm) sh ( @@ -2359,6 +2380,7 @@ def shard_run_integration_aarch64_3_of_4() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_arm) ci_setup(ci_arm) python_unittest(ci_arm) sh ( @@ -2420,6 +2442,7 @@ def shard_run_integration_aarch64_4_of_4() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_arm) ci_setup(ci_arm) python_unittest(ci_arm) sh ( @@ -2482,6 +2505,7 @@ def shard_run_topi_GPU_1_of_3() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", @@ -2542,6 +2566,7 @@ def shard_run_topi_GPU_2_of_3() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", @@ -2602,6 +2627,7 @@ def shard_run_topi_GPU_3_of_3() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", @@ -2663,6 +2689,7 @@ def shard_run_frontend_GPU_1_of_6() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", @@ -2723,6 +2750,7 @@ def shard_run_frontend_GPU_2_of_6() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", @@ -2783,6 +2811,7 @@ def shard_run_frontend_GPU_3_of_6() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", @@ -2843,6 +2872,7 @@ def shard_run_frontend_GPU_4_of_6() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", @@ -2903,6 +2933,7 @@ def shard_run_frontend_GPU_5_of_6() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", @@ -2963,6 +2994,7 @@ def shard_run_frontend_GPU_6_of_6() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", @@ -3024,6 +3056,7 @@ def shard_run_topi_aarch64_1_of_2() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_arm) ci_setup(ci_arm) cpp_unittest(ci_arm) sh ( @@ -3089,6 +3122,7 @@ def shard_run_topi_aarch64_2_of_2() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_arm) ci_setup(ci_arm) sh ( script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_arm_compute_library.sh", @@ -3154,6 +3188,7 @@ def shard_run_frontend_aarch64_1_of_2() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_arm) ci_setup(ci_arm) sh ( script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_frontend_cpu.sh", @@ -3214,6 +3249,7 @@ def shard_run_frontend_aarch64_2_of_2() { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_arm) ci_setup(ci_arm) sh ( script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_frontend_cpu.sh", @@ -3275,6 +3311,7 @@ def shard_run_test_Cortex_M_1_of_12() { ) add_microtvm_permissions() + standalone_crt_build(ci_cortexm) ci_setup(ci_cortexm) cpp_unittest(ci_cortexm) sh ( @@ -3340,6 +3377,7 @@ def shard_run_test_Cortex_M_2_of_12() { ) add_microtvm_permissions() + standalone_crt_build(ci_cortexm) ci_setup(ci_cortexm) sh ( script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh", @@ -3400,6 +3438,7 @@ def shard_run_test_Cortex_M_3_of_12() { ) add_microtvm_permissions() + standalone_crt_build(ci_cortexm) ci_setup(ci_cortexm) sh ( script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh", @@ -3460,6 +3499,7 @@ def shard_run_test_Cortex_M_4_of_12() { ) add_microtvm_permissions() + standalone_crt_build(ci_cortexm) ci_setup(ci_cortexm) sh ( script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh", @@ -3520,6 +3560,7 @@ def shard_run_test_Cortex_M_5_of_12() { ) add_microtvm_permissions() + standalone_crt_build(ci_cortexm) ci_setup(ci_cortexm) sh ( script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh", @@ -3580,6 +3621,7 @@ def shard_run_test_Cortex_M_6_of_12() { ) add_microtvm_permissions() + standalone_crt_build(ci_cortexm) ci_setup(ci_cortexm) sh ( script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh", @@ -3640,6 +3682,7 @@ def shard_run_test_Cortex_M_7_of_12() { ) add_microtvm_permissions() + standalone_crt_build(ci_cortexm) ci_setup(ci_cortexm) sh ( script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh", @@ -3700,6 +3743,7 @@ def shard_run_test_Cortex_M_8_of_12() { ) add_microtvm_permissions() + standalone_crt_build(ci_cortexm) ci_setup(ci_cortexm) sh ( script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh", @@ -3760,6 +3804,7 @@ def shard_run_test_Cortex_M_9_of_12() { ) add_microtvm_permissions() + standalone_crt_build(ci_cortexm) ci_setup(ci_cortexm) sh ( script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh", @@ -3820,6 +3865,7 @@ def shard_run_test_Cortex_M_10_of_12() { ) add_microtvm_permissions() + standalone_crt_build(ci_cortexm) ci_setup(ci_cortexm) sh ( script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh", @@ -3880,6 +3926,7 @@ def shard_run_test_Cortex_M_11_of_12() { ) add_microtvm_permissions() + standalone_crt_build(ci_cortexm) ci_setup(ci_cortexm) sh ( script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh", @@ -3940,6 +3987,7 @@ def shard_run_test_Cortex_M_12_of_12() { ) add_microtvm_permissions() + standalone_crt_build(ci_cortexm) ci_setup(ci_cortexm) sh ( script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh", @@ -4001,6 +4049,7 @@ def shard_run_test_RISC_V_1_of_1() { ) add_microtvm_permissions() + standalone_crt_build(ci_riscv) ci_setup(ci_riscv) cpp_unittest(ci_cortexm) sh ( @@ -4265,6 +4314,7 @@ stage('Test') { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_cpu) ci_setup(ci_cpu) cpp_unittest(ci_cpu) python_unittest(ci_cpu) @@ -4324,6 +4374,7 @@ stage('Test') { label: 'Download artifacts from S3', ) + standalone_crt_build(ci_cpu) ci_setup(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_frontend_cpu.sh", @@ -4378,6 +4429,7 @@ stage('Test') { add_microtvm_permissions() timeout(time: 180, unit: 'MINUTES') { + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh", diff --git a/ci/jenkins/Build.groovy.j2 b/ci/jenkins/Build.groovy.j2 index 49cffacdc16e..9be6279953fb 100644 --- a/ci/jenkins/Build.groovy.j2 +++ b/ci/jenkins/Build.groovy.j2 @@ -5,6 +5,13 @@ def ci_setup(image) { ) } +def standalone_crt_build(image) { + sh ( + script: "${docker_run} ${image} ./tests/scripts/task_standalone_crt.sh", + label: 'Build standalone_crt', + ) +} + def python_unittest(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", @@ -111,6 +118,7 @@ stage('Build') { ) make(ci_cpu, 'build', '-j2') {{ m.upload_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} + standalone_crt_build(ci_cpu) ci_setup(ci_cpu) // sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh" // TODO(@jroesch): need to resolve CI issue will turn back on in follow up patch @@ -145,6 +153,7 @@ stage('Build') { ) make(ci_wasm, 'build', '-j2') cpp_unittest(ci_wasm) + standalone_crt_build(ci_wasm) ci_setup(ci_wasm) sh ( script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh", diff --git a/ci/jenkins/Test.groovy.j2 b/ci/jenkins/Test.groovy.j2 index 52ed742d4cc0..16e9a69a7bb8 100644 --- a/ci/jenkins/Test.groovy.j2 +++ b/ci/jenkins/Test.groovy.j2 @@ -18,10 +18,12 @@ cpp_unittest(ci_gpu) {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) cpp_unittest(ci_gpu) {% else %} {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) {% endif %} {% if shard_index == 2 or num_shards < 2 %} @@ -49,6 +51,7 @@ test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} + standalone_crt_build(ci_cpu) ci_setup(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", @@ -65,6 +68,7 @@ test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='i386', filenames=tvm_multilib) }} + standalone_crt_build(ci_i386) ci_setup(ci_i386) {% if shard_index == 1 %} cpp_unittest(ci_i386) @@ -108,6 +112,7 @@ test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }} + standalone_crt_build(ci_arm) ci_setup(ci_arm) python_unittest(ci_arm) sh ( @@ -125,6 +130,7 @@ test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", @@ -141,6 +147,7 @@ test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", @@ -157,6 +164,7 @@ test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }} + standalone_crt_build(ci_arm) ci_setup(ci_arm) {% if shard_index == 1 %} cpp_unittest(ci_arm) @@ -180,6 +188,7 @@ test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }} + standalone_crt_build(ci_arm) ci_setup(ci_arm) sh ( script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_frontend_cpu.sh", @@ -197,6 +206,7 @@ ) %} {{ m.download_artifacts(tag='cortexm', filenames=tvm_lib, folders=microtvm_template_projects) }} add_microtvm_permissions() + standalone_crt_build(ci_cortexm) ci_setup(ci_cortexm) {% if shard_index == 1%} cpp_unittest(ci_cortexm) @@ -221,6 +231,7 @@ ) %} {{ m.download_artifacts(tag='riscv', filenames=tvm_lib, folders=microtvm_template_projects) }} add_microtvm_permissions() + standalone_crt_build(ci_riscv) ci_setup(ci_riscv) {% if shard_index == 1%} cpp_unittest(ci_cortexm) @@ -267,6 +278,7 @@ stage('Test') { docker_image="ci_cpu", ) %} {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} + standalone_crt_build(ci_cpu) ci_setup(ci_cpu) cpp_unittest(ci_cpu) python_unittest(ci_cpu) @@ -284,6 +296,7 @@ stage('Test') { docker_image="ci_cpu", ) %} {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib) }} + standalone_crt_build(ci_cpu) ci_setup(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_frontend_cpu.sh", @@ -299,6 +312,7 @@ stage('Test') { {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }} add_microtvm_permissions() timeout(time: 180, unit: 'MINUTES') { + standalone_crt_build(ci_gpu) ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh", diff --git a/tests/scripts/task_ci_setup.sh b/tests/scripts/task_ci_setup.sh index 5af0d0c9801e..81b82d1b51d8 100755 --- a/tests/scripts/task_ci_setup.sh +++ b/tests/scripts/task_ci_setup.sh @@ -18,23 +18,8 @@ set -euxo pipefail -# Script to setup additional python env. -# -# Use the following command to install the -# package to /workspace/.local, these additional -# packages will have precedence over the system packages. -# -# command: python3 -m pip install --user == -# echo "Additional setup in ${CI_IMAGE_NAME}" -# Rebuild standalone_crt in build/ tree. This file is not currently archived by pack_lib() in -# Jenkinsfile. We expect config.cmake to be present from pack_lib(). -# TODO(areusch): Make pack_lib() pack all the data dependencies of TVM. -python3 tests/scripts/task_build.py \ - --sccache-bucket tvm-sccache-prod \ - --cmake-target standalone_crt - # Ensure no stale pytest-results remain from a previous test run. pushd build rm -rf pytest-results diff --git a/tests/scripts/task_standalone_crt.sh b/tests/scripts/task_standalone_crt.sh new file mode 100755 index 000000000000..04afb5a42e52 --- /dev/null +++ b/tests/scripts/task_standalone_crt.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euxo pipefail + +echo "Build standalone_crt in ${CI_IMAGE_NAME}" + +# Rebuild standalone_crt in build/ tree. This file is not currently archived by pack_lib() in +# Jenkinsfile. We expect config.cmake to be present from pack_lib(). +# TODO(areusch): Make pack_lib() pack all the data dependencies of TVM. +python3 tests/scripts/task_build.py \ + --sccache-bucket tvm-sccache-prod \ + --cmake-target standalone_crt