diff --git a/Jenkinsfile b/Jenkinsfile index ed1cf4b09e6e..5835100fde3e 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-09-01T11:52:42.195970 +// Generated at 2022-09-14T11:22:31.582192 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -145,26 +145,7 @@ def init_git() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done -} - + . ci/scripts/retry.sh retry 3 timeout 5m git submodule update --init -f --jobs 0 """, label: 'Update git submodules', @@ -196,27 +177,8 @@ def docker_init(image) { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done -} - - retry 3 docker pull ${image} + . ci/scripts/retry.sh + retry 5 docker pull ${image} """, label: 'Pull docker image', ) @@ -453,8 +415,9 @@ def ecr_push(full_name) { sh( script: """ set -x + . ci/scripts/retry.sh docker tag ${full_name} \$AWS_ECR_REPO/${full_name} - docker push \$AWS_ECR_REPO/${full_name} + retry 5 docker push \$AWS_ECR_REPO/${full_name} """, label: 'Upload image to ECR' ) @@ -495,7 +458,8 @@ def ecr_pull(full_name) { sh( script: """ set -eux - docker pull ${full_name} + . ci/scripts/retry.sh + retry 5 docker pull ${full_name} """, label: 'Pull image from ECR' ) @@ -649,8 +613,8 @@ def lint() { 'Lint 1 of 2': { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") { - docker_init(ci_lint) init_git() + docker_init(ci_lint) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'TVM_NUM_SHARDS=2', @@ -669,8 +633,8 @@ def lint() { 'Lint 2 of 2': { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") { - docker_init(ci_lint) init_git() + docker_init(ci_lint) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'TVM_NUM_SHARDS=2', @@ -771,33 +735,14 @@ stage('Build') { if (!skip_ci) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" make("${ci_gpu} --no-gpu", 'build', '-j2') sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh md5sum build/libtvm.so retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/gpu/build/libtvm.so md5sum build/libvta_fsim.so @@ -818,26 +763,7 @@ stage('Build') { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh md5sum build/libtvm.so retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/gpu2/build/libtvm.so md5sum build/libvta_fsim.so @@ -858,8 +784,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") { - docker_init(ci_cpu) init_git() + docker_init(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build", label: 'Create CPU cmake config', @@ -868,26 +794,7 @@ stage('Build') { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh md5sum build/libvta_tsim.so retry 3 aws s3 cp --no-progress build/libvta_tsim.so s3://${s3_prefix}/cpu/build/libvta_tsim.so md5sum build/libtvm.so @@ -918,8 +825,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu-minimal") { - docker_init(ci_minimal) init_git() + docker_init(ci_minimal) sh ( script: "${docker_run} ${ci_minimal} ./tests/scripts/task_config_build_minimal.sh build", label: 'Create CPU minimal cmake config', @@ -928,26 +835,7 @@ stage('Build') { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh md5sum build/libtvm.so retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/cpu-minimal/build/libtvm.so md5sum build/libtvm_runtime.so @@ -968,8 +856,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") { - docker_init(ci_wasm) init_git() + docker_init(ci_wasm) sh ( script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build", label: 'Create WASM cmake config', @@ -993,8 +881,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-i386") { - docker_init(ci_i386) init_git() + docker_init(ci_i386) sh ( script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build", label: 'Create i386 cmake config', @@ -1003,26 +891,7 @@ stage('Build') { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh md5sum build/libvta_tsim.so retry 3 aws s3 cp --no-progress build/libvta_tsim.so s3://${s3_prefix}/i386/build/libvta_tsim.so md5sum build/libtvm.so @@ -1047,8 +916,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-arm") { - docker_init(ci_arm) init_git() + docker_init(ci_arm) sh ( script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build", label: 'Create ARM cmake config', @@ -1057,26 +926,7 @@ stage('Build') { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh md5sum build/libtvm.so retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/arm/build/libtvm.so md5sum build/libvta_fsim.so @@ -1099,8 +949,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cortexm") { - docker_init(ci_cortexm) init_git() + docker_init(ci_cortexm) sh ( script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_config_build_cortexm.sh build", label: 'Create Cortex-M cmake config', @@ -1109,26 +959,7 @@ stage('Build') { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh md5sum build/libtvm.so retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/cortexm/build/libtvm.so md5sum build/libtvm_runtime.so @@ -1150,8 +981,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-hexagon") { - docker_init(ci_hexagon) init_git() + docker_init(ci_hexagon) sh ( script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build", label: 'Create Hexagon cmake config', @@ -1164,26 +995,7 @@ stage('Build') { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh md5sum build/libtvm.so retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/hexagon/build/libtvm.so md5sum build/libtvm_runtime.so @@ -1205,8 +1017,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-riscv") { - docker_init(ci_riscv) init_git() + docker_init(ci_riscv) sh ( script: "${docker_run} ${ci_riscv} ./tests/scripts/task_config_build_riscv.sh build", label: 'Create RISC-V cmake config', @@ -1215,26 +1027,7 @@ stage('Build') { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh md5sum build/libtvm.so retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/riscv/build/libtvm.so md5sum build/libtvm_runtime.so @@ -1266,8 +1059,8 @@ def shard_run_unittest_GPU_1_of_3() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { try { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', @@ -1278,26 +1071,7 @@ def shard_run_unittest_GPU_1_of_3() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libvta_fsim.so build/libvta_fsim.so @@ -1315,26 +1089,7 @@ def shard_run_unittest_GPU_1_of_3() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -1382,8 +1137,8 @@ def shard_run_unittest_GPU_2_of_3() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { try { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', @@ -1394,26 +1149,7 @@ def shard_run_unittest_GPU_2_of_3() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -1464,8 +1200,8 @@ def shard_run_unittest_GPU_3_of_3() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { try { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', @@ -1476,26 +1212,7 @@ def shard_run_unittest_GPU_3_of_3() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -1543,8 +1260,8 @@ def shard_run_integration_CPU_1_of_4() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { - docker_init(ci_cpu) init_git() + docker_init(ci_cpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cpu', @@ -1555,26 +1272,7 @@ def shard_run_integration_CPU_1_of_4() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so @@ -1619,8 +1317,8 @@ def shard_run_integration_CPU_2_of_4() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { - docker_init(ci_cpu) init_git() + docker_init(ci_cpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cpu', @@ -1631,26 +1329,7 @@ def shard_run_integration_CPU_2_of_4() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so @@ -1695,8 +1374,8 @@ def shard_run_integration_CPU_3_of_4() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { - docker_init(ci_cpu) init_git() + docker_init(ci_cpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cpu', @@ -1707,26 +1386,7 @@ def shard_run_integration_CPU_3_of_4() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so @@ -1771,8 +1431,8 @@ def shard_run_integration_CPU_4_of_4() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { - docker_init(ci_cpu) init_git() + docker_init(ci_cpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cpu', @@ -1783,26 +1443,7 @@ def shard_run_integration_CPU_4_of_4() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so @@ -1848,8 +1489,8 @@ def shard_run_python_i386_1_of_3() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { try { - docker_init(ci_i386) init_git() + docker_init(ci_i386) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=i386', @@ -1860,26 +1501,7 @@ def shard_run_python_i386_1_of_3() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so @@ -1924,8 +1546,8 @@ def shard_run_python_i386_2_of_3() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { try { - docker_init(ci_i386) init_git() + docker_init(ci_i386) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=i386', @@ -1936,26 +1558,7 @@ def shard_run_python_i386_2_of_3() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so @@ -2000,8 +1603,8 @@ def shard_run_python_i386_3_of_3() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { try { - docker_init(ci_i386) init_git() + docker_init(ci_i386) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=i386', @@ -2012,26 +1615,7 @@ def shard_run_python_i386_3_of_3() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so @@ -2076,8 +1660,8 @@ def shard_run_test_Hexagon_1_of_8() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() + docker_init(ci_hexagon) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', @@ -2088,26 +1672,7 @@ def shard_run_test_Hexagon_1_of_8() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -2151,8 +1716,8 @@ def shard_run_test_Hexagon_2_of_8() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() + docker_init(ci_hexagon) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', @@ -2163,26 +1728,7 @@ def shard_run_test_Hexagon_2_of_8() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -2225,8 +1771,8 @@ def shard_run_test_Hexagon_3_of_8() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() + docker_init(ci_hexagon) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', @@ -2237,26 +1783,7 @@ def shard_run_test_Hexagon_3_of_8() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -2299,8 +1826,8 @@ def shard_run_test_Hexagon_4_of_8() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() + docker_init(ci_hexagon) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', @@ -2311,26 +1838,7 @@ def shard_run_test_Hexagon_4_of_8() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -2373,8 +1881,8 @@ def shard_run_test_Hexagon_5_of_8() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() + docker_init(ci_hexagon) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', @@ -2385,26 +1893,7 @@ def shard_run_test_Hexagon_5_of_8() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -2447,8 +1936,8 @@ def shard_run_test_Hexagon_6_of_8() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() + docker_init(ci_hexagon) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', @@ -2459,26 +1948,7 @@ def shard_run_test_Hexagon_6_of_8() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -2521,8 +1991,8 @@ def shard_run_test_Hexagon_7_of_8() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() + docker_init(ci_hexagon) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', @@ -2533,26 +2003,7 @@ def shard_run_test_Hexagon_7_of_8() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -2595,8 +2046,8 @@ def shard_run_test_Hexagon_8_of_8() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() + docker_init(ci_hexagon) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', @@ -2607,26 +2058,7 @@ def shard_run_test_Hexagon_8_of_8() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -2670,8 +2102,8 @@ def shard_run_integration_aarch64_1_of_4() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { - docker_init(ci_arm) init_git() + docker_init(ci_arm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', @@ -2682,26 +2114,7 @@ def shard_run_integration_aarch64_1_of_4() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -2745,8 +2158,8 @@ def shard_run_integration_aarch64_2_of_4() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { - docker_init(ci_arm) init_git() + docker_init(ci_arm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', @@ -2757,26 +2170,7 @@ def shard_run_integration_aarch64_2_of_4() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -2820,8 +2214,8 @@ def shard_run_integration_aarch64_3_of_4() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { - docker_init(ci_arm) init_git() + docker_init(ci_arm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', @@ -2832,26 +2226,7 @@ def shard_run_integration_aarch64_3_of_4() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -2895,8 +2270,8 @@ def shard_run_integration_aarch64_4_of_4() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { - docker_init(ci_arm) init_git() + docker_init(ci_arm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', @@ -2907,26 +2282,7 @@ def shard_run_integration_aarch64_4_of_4() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -2971,8 +2327,8 @@ def shard_run_topi_GPU_1_of_3() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { try { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', @@ -2983,26 +2339,7 @@ def shard_run_topi_GPU_1_of_3() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -3045,8 +2382,8 @@ def shard_run_topi_GPU_2_of_3() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { try { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', @@ -3057,26 +2394,7 @@ def shard_run_topi_GPU_2_of_3() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -3119,8 +2437,8 @@ def shard_run_topi_GPU_3_of_3() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { try { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', @@ -3131,26 +2449,7 @@ def shard_run_topi_GPU_3_of_3() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -3194,8 +2493,8 @@ def shard_run_frontend_GPU_1_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', @@ -3206,26 +2505,7 @@ def shard_run_frontend_GPU_1_of_6() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -3268,8 +2548,8 @@ def shard_run_frontend_GPU_2_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', @@ -3280,26 +2560,7 @@ def shard_run_frontend_GPU_2_of_6() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -3342,8 +2603,8 @@ def shard_run_frontend_GPU_3_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', @@ -3354,26 +2615,7 @@ def shard_run_frontend_GPU_3_of_6() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -3416,8 +2658,8 @@ def shard_run_frontend_GPU_4_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', @@ -3428,26 +2670,7 @@ def shard_run_frontend_GPU_4_of_6() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -3490,8 +2713,8 @@ def shard_run_frontend_GPU_5_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', @@ -3502,26 +2725,7 @@ def shard_run_frontend_GPU_5_of_6() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -3564,8 +2768,8 @@ def shard_run_frontend_GPU_6_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', @@ -3576,26 +2780,7 @@ def shard_run_frontend_GPU_6_of_6() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -3639,8 +2824,8 @@ def shard_run_topi_aarch64_1_of_2() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { - docker_init(ci_arm) init_git() + docker_init(ci_arm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', @@ -3651,26 +2836,7 @@ def shard_run_topi_aarch64_1_of_2() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -3718,8 +2884,8 @@ def shard_run_topi_aarch64_2_of_2() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { - docker_init(ci_arm) init_git() + docker_init(ci_arm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', @@ -3730,26 +2896,7 @@ def shard_run_topi_aarch64_2_of_2() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -3797,8 +2944,8 @@ def shard_run_frontend_aarch64_1_of_2() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") { try { - docker_init(ci_arm) init_git() + docker_init(ci_arm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', @@ -3809,26 +2956,7 @@ def shard_run_frontend_aarch64_1_of_2() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -3871,8 +2999,8 @@ def shard_run_frontend_aarch64_2_of_2() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") { try { - docker_init(ci_arm) init_git() + docker_init(ci_arm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', @@ -3883,26 +3011,7 @@ def shard_run_frontend_aarch64_2_of_2() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -3946,8 +3055,8 @@ def shard_run_test_Cortex_M_1_of_12() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") { try { - docker_init(ci_cortexm) init_git() + docker_init(ci_cortexm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', @@ -3958,26 +3067,7 @@ def shard_run_test_Cortex_M_1_of_12() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4025,8 +3115,8 @@ def shard_run_test_Cortex_M_2_of_12() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") { try { - docker_init(ci_cortexm) init_git() + docker_init(ci_cortexm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', @@ -4037,26 +3127,7 @@ def shard_run_test_Cortex_M_2_of_12() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4099,8 +3170,8 @@ def shard_run_test_Cortex_M_3_of_12() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") { try { - docker_init(ci_cortexm) init_git() + docker_init(ci_cortexm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', @@ -4111,26 +3182,7 @@ def shard_run_test_Cortex_M_3_of_12() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4173,8 +3225,8 @@ def shard_run_test_Cortex_M_4_of_12() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") { try { - docker_init(ci_cortexm) init_git() + docker_init(ci_cortexm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', @@ -4185,26 +3237,7 @@ def shard_run_test_Cortex_M_4_of_12() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4247,8 +3280,8 @@ def shard_run_test_Cortex_M_5_of_12() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") { try { - docker_init(ci_cortexm) init_git() + docker_init(ci_cortexm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', @@ -4259,26 +3292,7 @@ def shard_run_test_Cortex_M_5_of_12() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4321,8 +3335,8 @@ def shard_run_test_Cortex_M_6_of_12() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") { try { - docker_init(ci_cortexm) init_git() + docker_init(ci_cortexm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', @@ -4333,26 +3347,7 @@ def shard_run_test_Cortex_M_6_of_12() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4395,8 +3390,8 @@ def shard_run_test_Cortex_M_7_of_12() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") { try { - docker_init(ci_cortexm) init_git() + docker_init(ci_cortexm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', @@ -4407,26 +3402,7 @@ def shard_run_test_Cortex_M_7_of_12() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4469,8 +3445,8 @@ def shard_run_test_Cortex_M_8_of_12() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") { try { - docker_init(ci_cortexm) init_git() + docker_init(ci_cortexm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', @@ -4481,26 +3457,7 @@ def shard_run_test_Cortex_M_8_of_12() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4543,8 +3500,8 @@ def shard_run_test_Cortex_M_9_of_12() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") { try { - docker_init(ci_cortexm) init_git() + docker_init(ci_cortexm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', @@ -4555,26 +3512,7 @@ def shard_run_test_Cortex_M_9_of_12() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4617,8 +3555,8 @@ def shard_run_test_Cortex_M_10_of_12() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") { try { - docker_init(ci_cortexm) init_git() + docker_init(ci_cortexm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', @@ -4629,26 +3567,7 @@ def shard_run_test_Cortex_M_10_of_12() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4691,8 +3610,8 @@ def shard_run_test_Cortex_M_11_of_12() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") { try { - docker_init(ci_cortexm) init_git() + docker_init(ci_cortexm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', @@ -4703,26 +3622,7 @@ def shard_run_test_Cortex_M_11_of_12() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4765,8 +3665,8 @@ def shard_run_test_Cortex_M_12_of_12() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") { try { - docker_init(ci_cortexm) init_git() + docker_init(ci_cortexm) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', @@ -4777,26 +3677,7 @@ def shard_run_test_Cortex_M_12_of_12() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4840,8 +3721,8 @@ def shard_run_test_RISC_V_1_of_1() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-riscv") { try { - docker_init(ci_riscv) init_git() + docker_init(ci_riscv) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=riscv', @@ -4852,26 +3733,7 @@ def shard_run_test_RISC_V_1_of_1() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/riscv/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/riscv/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4917,32 +3779,13 @@ def run_unittest_minimal() { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu-minimal") { timeout(time: max_time, unit: 'MINUTES') { try { - docker_init(ci_minimal) init_git() + docker_init(ci_minimal) withEnv(['PLATFORM=minimal'], { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu-minimal/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu-minimal/build/libtvm_runtime.so build/libtvm_runtime.so @@ -5134,34 +3977,15 @@ stage('Test') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") { timeout(time: max_time, unit: 'MINUTES') { try { - docker_init(ci_cpu) init_git() + docker_init(ci_cpu) withEnv(['PLATFORM=cpu', 'TEST_STEP_NAME=unittest: CPU', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so @@ -5209,34 +4033,15 @@ stage('Test') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-cpu") { timeout(time: max_time, unit: 'MINUTES') { try { - docker_init(ci_cpu) init_git() + docker_init(ci_cpu) withEnv(['PLATFORM=cpu', 'TEST_STEP_NAME=frontend: CPU', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so @@ -5277,31 +4082,12 @@ stage('Test') { if (!skip_ci) { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/docs-python-gpu") { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -5326,26 +4112,7 @@ stage('Test') { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh md5sum docs.tgz retry 3 aws s3 cp --no-progress docs.tgz s3://${s3_prefix}/docs/docs.tgz """, @@ -5395,10 +4162,11 @@ def update_docker(ecr_image, hub_image) { sh( script: """ set -eux + . ci/scripts/retry.sh docker tag \ ${ecr_image} \ ${hub_image} - docker push ${hub_image} + retry 5 docker push ${hub_image} """, label: "Update ${hub_image} on Docker Hub", ) @@ -5457,26 +4225,7 @@ def deploy() { sh( script: """ set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - + . ci/scripts/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/docs/docs.tgz docs.tgz md5sum docs.tgz """, @@ -5555,9 +4304,10 @@ def deploy() { sh( script: """ set -eux + . ci/scripts/retry.sh docker pull tlcpackstaging/ci_arm:${tag} docker tag tlcpackstaging/ci_arm:${tag} tlcpack/ci-arm:${tag} - docker push tlcpack/ci-arm:${tag} + retry 5 docker push tlcpack/ci-arm:${tag} """, label: 'Tag tlcpackstaging/ci_arm image to tlcpack', ) @@ -5568,9 +4318,10 @@ def deploy() { sh( script: """ set -eux + . ci/scripts/retry.sh docker pull tlcpackstaging/ci_cortexm:${tag} docker tag tlcpackstaging/ci_cortexm:${tag} tlcpack/ci-cortexm:${tag} - docker push tlcpack/ci-cortexm:${tag} + retry 5 docker push tlcpack/ci-cortexm:${tag} """, label: 'Tag tlcpackstaging/ci_cortexm image to tlcpack', ) @@ -5581,9 +4332,10 @@ def deploy() { sh( script: """ set -eux + . ci/scripts/retry.sh docker pull tlcpackstaging/ci_cpu:${tag} docker tag tlcpackstaging/ci_cpu:${tag} tlcpack/ci-cpu:${tag} - docker push tlcpack/ci-cpu:${tag} + retry 5 docker push tlcpack/ci-cpu:${tag} """, label: 'Tag tlcpackstaging/ci_cpu image to tlcpack', ) @@ -5594,9 +4346,10 @@ def deploy() { sh( script: """ set -eux + . ci/scripts/retry.sh docker pull tlcpackstaging/ci_gpu:${tag} docker tag tlcpackstaging/ci_gpu:${tag} tlcpack/ci-gpu:${tag} - docker push tlcpack/ci-gpu:${tag} + retry 5 docker push tlcpack/ci-gpu:${tag} """, label: 'Tag tlcpackstaging/ci_gpu image to tlcpack', ) @@ -5607,9 +4360,10 @@ def deploy() { sh( script: """ set -eux + . ci/scripts/retry.sh docker pull tlcpackstaging/ci_hexagon:${tag} docker tag tlcpackstaging/ci_hexagon:${tag} tlcpack/ci-hexagon:${tag} - docker push tlcpack/ci-hexagon:${tag} + retry 5 docker push tlcpack/ci-hexagon:${tag} """, label: 'Tag tlcpackstaging/ci_hexagon image to tlcpack', ) @@ -5620,9 +4374,10 @@ def deploy() { sh( script: """ set -eux + . ci/scripts/retry.sh docker pull tlcpackstaging/ci_i386:${tag} docker tag tlcpackstaging/ci_i386:${tag} tlcpack/ci-i386:${tag} - docker push tlcpack/ci-i386:${tag} + retry 5 docker push tlcpack/ci-i386:${tag} """, label: 'Tag tlcpackstaging/ci_i386 image to tlcpack', ) @@ -5633,9 +4388,10 @@ def deploy() { sh( script: """ set -eux + . ci/scripts/retry.sh docker pull tlcpackstaging/ci_lint:${tag} docker tag tlcpackstaging/ci_lint:${tag} tlcpack/ci-lint:${tag} - docker push tlcpack/ci-lint:${tag} + retry 5 docker push tlcpack/ci-lint:${tag} """, label: 'Tag tlcpackstaging/ci_lint image to tlcpack', ) @@ -5646,9 +4402,10 @@ def deploy() { sh( script: """ set -eux + . ci/scripts/retry.sh docker pull tlcpackstaging/ci_minimal:${tag} docker tag tlcpackstaging/ci_minimal:${tag} tlcpack/ci-minimal:${tag} - docker push tlcpack/ci-minimal:${tag} + retry 5 docker push tlcpack/ci-minimal:${tag} """, label: 'Tag tlcpackstaging/ci_minimal image to tlcpack', ) @@ -5659,9 +4416,10 @@ def deploy() { sh( script: """ set -eux + . ci/scripts/retry.sh docker pull tlcpackstaging/ci_riscv:${tag} docker tag tlcpackstaging/ci_riscv:${tag} tlcpack/ci-riscv:${tag} - docker push tlcpack/ci-riscv:${tag} + retry 5 docker push tlcpack/ci-riscv:${tag} """, label: 'Tag tlcpackstaging/ci_riscv image to tlcpack', ) @@ -5672,9 +4430,10 @@ def deploy() { sh( script: """ set -eux + . ci/scripts/retry.sh docker pull tlcpackstaging/ci_wasm:${tag} docker tag tlcpackstaging/ci_wasm:${tag} tlcpack/ci-wasm:${tag} - docker push tlcpack/ci-wasm:${tag} + retry 5 docker push tlcpack/ci-wasm:${tag} """, label: 'Tag tlcpackstaging/ci_wasm image to tlcpack', ) diff --git a/ci/jenkins/Build.groovy.j2 b/ci/jenkins/Build.groovy.j2 index 51360b7d9c54..a083fe88ad80 100644 --- a/ci/jenkins/Build.groovy.j2 +++ b/ci/jenkins/Build.groovy.j2 @@ -84,8 +84,8 @@ stage('Build') { if (!skip_ci) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-gpu') }}) { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" make("${ci_gpu} --no-gpu", 'build', '-j2') {{ m.upload_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }} @@ -102,8 +102,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-cpu') }}) { - docker_init(ci_cpu) init_git() + docker_init(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build", label: 'Create CPU cmake config', @@ -126,8 +126,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-cpu-minimal') }}) { - docker_init(ci_minimal) init_git() + docker_init(ci_minimal) sh ( script: "${docker_run} ${ci_minimal} ./tests/scripts/task_config_build_minimal.sh build", label: 'Create CPU minimal cmake config', @@ -144,8 +144,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-wasm') }}) { - docker_init(ci_wasm) init_git() + docker_init(ci_wasm) sh ( script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build", label: 'Create WASM cmake config', @@ -169,8 +169,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-i386') }}) { - docker_init(ci_i386) init_git() + docker_init(ci_i386) sh ( script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build", label: 'Create i386 cmake config', @@ -187,8 +187,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('ARM-SMALL') { ws({{ m.per_exec_ws('tvm/build-arm') }}) { - docker_init(ci_arm) init_git() + docker_init(ci_arm) sh ( script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build", label: 'Create ARM cmake config', @@ -205,8 +205,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-cortexm') }}) { - docker_init(ci_cortexm) init_git() + docker_init(ci_cortexm) sh ( script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_config_build_cortexm.sh build", label: 'Create Cortex-M cmake config', @@ -223,8 +223,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-hexagon') }}) { - docker_init(ci_hexagon) init_git() + docker_init(ci_hexagon) sh ( script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build", label: 'Create Hexagon cmake config', @@ -245,8 +245,8 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-riscv') }}) { - docker_init(ci_riscv) init_git() + docker_init(ci_riscv) sh ( script: "${docker_run} ${ci_riscv} ./tests/scripts/task_config_build_riscv.sh build", label: 'Create RISC-V cmake config', diff --git a/ci/jenkins/Deploy.groovy.j2 b/ci/jenkins/Deploy.groovy.j2 index 08516da41b9f..d2ee4360da6b 100644 --- a/ci/jenkins/Deploy.groovy.j2 +++ b/ci/jenkins/Deploy.groovy.j2 @@ -30,10 +30,11 @@ def update_docker(ecr_image, hub_image) { sh( script: """ set -eux + . ci/scripts/retry.sh docker tag \ ${ecr_image} \ ${hub_image} - docker push ${hub_image} + retry 5 docker push ${hub_image} """, label: "Update ${hub_image} on Docker Hub", ) @@ -144,9 +145,10 @@ def deploy() { sh( script: """ set -eux + . ci/scripts/retry.sh docker pull tlcpackstaging/{{ image.name }}:${tag} docker tag tlcpackstaging/{{ image.name }}:${tag} tlcpack/{{ image.name.replace("_", "-") }}:${tag} - docker push tlcpack/{{ image.name.replace("_", "-") }}:${tag} + retry 5 docker push tlcpack/{{ image.name.replace("_", "-") }}:${tag} """, label: 'Tag tlcpackstaging/{{ image.name }} image to tlcpack', ) diff --git a/ci/jenkins/DockerBuild.groovy.j2 b/ci/jenkins/DockerBuild.groovy.j2 index 1f3bded86242..5ffbeded80fa 100644 --- a/ci/jenkins/DockerBuild.groovy.j2 +++ b/ci/jenkins/DockerBuild.groovy.j2 @@ -21,8 +21,9 @@ def ecr_push(full_name) { sh( script: """ set -x + . ci/scripts/retry.sh docker tag ${full_name} \$AWS_ECR_REPO/${full_name} - docker push \$AWS_ECR_REPO/${full_name} + retry 5 docker push \$AWS_ECR_REPO/${full_name} """, label: 'Upload image to ECR' ) @@ -63,7 +64,8 @@ def ecr_pull(full_name) { sh( script: """ set -eux - docker pull ${full_name} + . ci/scripts/retry.sh + retry 5 docker pull ${full_name} """, label: 'Pull image from ECR' ) diff --git a/ci/jenkins/Prepare.groovy.j2 b/ci/jenkins/Prepare.groovy.j2 index 6d0c0ec9c4b6..4464108968de 100644 --- a/ci/jenkins/Prepare.groovy.j2 +++ b/ci/jenkins/Prepare.groovy.j2 @@ -33,7 +33,7 @@ def init_git() { sh( script: """ set -eux - {{ m.bash_retry() }} + . ci/scripts/retry.sh retry 3 timeout 5m git submodule update --init -f --jobs 0 """, label: 'Update git submodules', @@ -65,8 +65,8 @@ def docker_init(image) { sh( script: """ set -eux - {{ m.bash_retry() }} - retry 3 docker pull ${image} + . ci/scripts/retry.sh + retry 5 docker pull ${image} """, label: 'Pull docker image', ) diff --git a/ci/jenkins/Test.groovy.j2 b/ci/jenkins/Test.groovy.j2 index 4ed149da9be0..52ed742d4cc0 100644 --- a/ci/jenkins/Test.groovy.j2 +++ b/ci/jenkins/Test.groovy.j2 @@ -294,8 +294,8 @@ stage('Test') { if (!skip_ci) { node('GPU') { ws({{ m.per_exec_ws('tvm/docs-python-gpu') }}) { - docker_init(ci_gpu) init_git() + docker_init(ci_gpu) {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }} add_microtvm_permissions() timeout(time: 180, unit: 'MINUTES') { diff --git a/ci/jenkins/macros.j2 b/ci/jenkins/macros.j2 index 9d02ad68d6da..e6e69097b076 100644 --- a/ci/jenkins/macros.j2 +++ b/ci/jenkins/macros.j2 @@ -39,8 +39,8 @@ def {{ method_name }}() { node('{{ node }}') { ws({{ per_exec_ws(ws) }}) { try { - docker_init({{ docker_image }}) init_git() + docker_init({{ docker_image }}) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM={{ platform }}', @@ -71,8 +71,8 @@ def {{ method_name }}() { '{{ name }} {{ shard_index }} of {{ num_shards }}': { node('{{ node }}') { ws({{ per_exec_ws(ws) }}) { - docker_init({{ docker_image }}) init_git() + docker_init({{ docker_image }}) timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'TVM_NUM_SHARDS={{ num_shards }}', @@ -95,8 +95,8 @@ def {{ method_name }}() { ws({{ per_exec_ws(ws) }}) { timeout(time: max_time, unit: 'MINUTES') { try { - docker_init({{ docker_image }}) init_git() + docker_init({{ docker_image }}) withEnv(['PLATFORM={{ platform }}'], { {{ caller() | indent(width=8) | trim }} }) @@ -120,8 +120,8 @@ def {{ method_name }}() { ws({{ per_exec_ws(ws) }}) { timeout(time: max_time, unit: 'MINUTES') { try { - docker_init({{ docker_image }}) init_git() + docker_init({{ docker_image }}) withEnv(['PLATFORM={{ platform }}', 'TEST_STEP_NAME={{ name }}', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -140,28 +140,6 @@ def {{ method_name }}() { }, {% endmacro %} -{% macro bash_retry() %} -retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done -} -{% endmacro %} - {% macro deploy_step(name, feature_flag, ws) %} '{{ name }}': { if ({{ feature_flag }}) { @@ -182,7 +160,7 @@ retry() { sh( script: """ set -eux - {{ bash_retry() | indent(width=14) }} + . ci/scripts/retry.sh {% for filename in filenames %} md5sum {{ filename }} retry 3 aws s3 cp --no-progress {{ filename }} s3://${s3_prefix}/{{ tag }}/{{ filename }} @@ -199,7 +177,7 @@ sh( sh( script: """ set -eux - {{ bash_retry() | indent(width=14) }} + . ci/scripts/retry.sh {% for filename in filenames %} retry 3 aws s3 cp --no-progress s3://${s3_prefix}/{{ tag }}/{{ filename }} {{ filename }} md5sum {{ filename }} diff --git a/ci/scripts/retry.sh b/ci/scripts/retry.sh new file mode 100644 index 000000000000..08958fedce89 --- /dev/null +++ b/ci/scripts/retry.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eux + +retry() { + local max_retries=$1 + shift + local n=0 + until [ "$n" -ge "$max_retries" ] + do + "$@" && break + n=$((n+1)) + if [ "$n" -eq "$max_retries" ]; then + echo "failed to update after attempt $n / $max_retries, giving up" + exit 1 + fi + + WAIT=$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update $n / $max_retries, waiting $WAIT to try again" + sleep "$WAIT" + done +}