From 6f60e8c095ebc70e4fb7ee97706c2e56e276cec3 Mon Sep 17 00:00:00 2001 From: Wei Chu Date: Wed, 24 Feb 2021 12:38:12 -0800 Subject: [PATCH 1/8] website version update --- ci/docker/Dockerfile.build.ubuntu | 1 + ci/docker/runtime_functions.sh | 20 ++++++ ci/jenkins/Jenkins_steps.groovy | 46 ++++++++++++++ .../Jenkinsfile_website_version_artifacts | 61 +++++++++++++++++++ 4 files changed, 128 insertions(+) create mode 100644 ci/jenkins/Jenkinsfile_website_version_artifacts diff --git a/ci/docker/Dockerfile.build.ubuntu b/ci/docker/Dockerfile.build.ubuntu index 34feb61149c7..7077cac976fa 100644 --- a/ci/docker/Dockerfile.build.ubuntu +++ b/ci/docker/Dockerfile.build.ubuntu @@ -47,6 +47,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get install -y \ ## Utilities curl \ + zip \ unzip \ pandoc \ ## Development tools diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index fb9783db4dd2..bde45b34ad9e 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1212,6 +1212,26 @@ build_docs_beta() { popd } +push_docs() { + folder_name=$1 + set -ex + pip3 install --user awscli + export PATH=~/.local/bin:$PATH + pushd docs/_build + tar -xzf full_website.tgz --strip-components 1 + # check if folder_name already exists in versions + pushd versions + if [ -d "$folder_name" ]; then + echo "Folder $folder_name already exists in versions. Please double check the FOLDER_NAME variable in Jenkens pipeline" + exit 1 + fi + mv master $folder_name + popd + zip -r9 versions-test.zip versions/. + aws s3 cp versions-test.zip s3://mxnet-website-static-artifacts --acl public-read + popd +} + create_repo() { repo_folder=$1 mxnet_url=$2 diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index b0244c114b36..ccf7c34f6dde 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1093,6 +1093,30 @@ def docs_prepare() { }] } +// This is for updateing the new version of website artifact +// Assumes you have run all of the docs generation functions +// Called from Jenkins_website_version_artifacts +def docs_full_website() { + return ['Build artifacts full_website.tgz': { + node(NODE_LINUX_CPU) { + ws('workspace/docs') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git() + + unstash 'jekyll-artifacts' + unstash 'python-artifacts' + + utils.docker_run('ubuntu_cpu_jekyll', 'build_docs', false) + + utils.pack_lib('full_website', 'docs/_build/full_website.tgz', false) + + // archive so the publish pipeline can access the artifact + archiveArtifacts 'docs/_build/full_website.tgz' + } + } + } + }] +} def docs_prepare_beta() { return ['Prepare for publication to the staging website': { @@ -1168,6 +1192,28 @@ def docs_publish_beta() { }] } +// This is for uploading website artifacts to S3 bucket +// Assumes you have run docs_full_website function +def docs_upload_s3() { + return ['Upload artifacts to s3 bucket': { + node(NODE_LINUX_CPU) { + ws('workspace/docs') { + timeout(time: max_time, unit: 'MINUTES') { + if(env.FOLDER_NAME) { + utils.unpack_and_init('full_website', 'docs/_build/full_website.tgz') + + utils.docker_run('ubuntu_cpu', "push_docs ${env.FOLDER_NAME}", false) + + archiveArtifacts 'docs/_build/versions-test.zip' + } else { + sh 'echo Can not find website version for release. Please specify env var FOLDER_NAME in Jenkins pipeline' + sh 'exit 1' + } + } + } + } + }] +} def sanity_lint() { return ['Lint': { diff --git a/ci/jenkins/Jenkinsfile_website_version_artifacts b/ci/jenkins/Jenkinsfile_website_version_artifacts new file mode 100644 index 000000000000..7f74f1a9e076 --- /dev/null +++ b/ci/jenkins/Jenkinsfile_website_version_artifacts @@ -0,0 +1,61 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 240 + +node('restricted-utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} + +utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_cpu_openblas('libmxnet') + ]) + + utils.parallel_stage('Build Docs', [ + custom_steps.docs_jekyll(), + custom_steps.docs_python('libmxnet'), + ]) + + utils.parallel_stage('Build Full Website', [ + custom_steps.docs_full_website() + ]) + + utils.parallel_stage('Upload Docs', [ + custom_steps.docs_upload_s3() + ]) +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) From 043bbc2eecae47cc45294c7bf5de222758fdcb2c Mon Sep 17 00:00:00 2001 From: Wei Chu Date: Fri, 26 Feb 2021 10:50:28 -0800 Subject: [PATCH 2/8] back up versions --- ci/docker/runtime_functions.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index bde45b34ad9e..79968cefc1c0 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1227,6 +1227,10 @@ push_docs() { fi mv master $folder_name popd + # back up versions folder + wget https://mxnet-website-static-artifacts.s3.us-east-2.amazonaws.com/versions.zip versions-backup + aws s3 cp versions-backup.zip s3://mxnet-website-static-artifacts --acl public-read + zip -r9 versions-test.zip versions/. aws s3 cp versions-test.zip s3://mxnet-website-static-artifacts --acl public-read popd From 4a758930475447497667614d1ba93881fb130589 Mon Sep 17 00:00:00 2001 From: Wei Chu Date: Fri, 26 Feb 2021 12:46:18 -0800 Subject: [PATCH 3/8] rename file --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 79968cefc1c0..662625f4a0f2 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1228,7 +1228,7 @@ push_docs() { mv master $folder_name popd # back up versions folder - wget https://mxnet-website-static-artifacts.s3.us-east-2.amazonaws.com/versions.zip versions-backup + wget https://mxnet-website-static-artifacts.s3.us-east-2.amazonaws.com/versions.zip -O versions-backup.zip aws s3 cp versions-backup.zip s3://mxnet-website-static-artifacts --acl public-read zip -r9 versions-test.zip versions/. From f3d1851ee00cd660b0dc7a83136e37a694eb49b8 Mon Sep 17 00:00:00 2001 From: Wei Chu Date: Fri, 26 Feb 2021 14:24:49 -0800 Subject: [PATCH 4/8] remove test --- ci/docker/runtime_functions.sh | 4 ++-- ci/jenkins/Jenkins_steps.groovy | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 662625f4a0f2..752c32bcc075 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1231,8 +1231,8 @@ push_docs() { wget https://mxnet-website-static-artifacts.s3.us-east-2.amazonaws.com/versions.zip -O versions-backup.zip aws s3 cp versions-backup.zip s3://mxnet-website-static-artifacts --acl public-read - zip -r9 versions-test.zip versions/. - aws s3 cp versions-test.zip s3://mxnet-website-static-artifacts --acl public-read + zip -r9 versions.zip versions/. + aws s3 cp versions.zip s3://mxnet-website-static-artifacts --acl public-read popd } diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index ccf7c34f6dde..03b325ac4a91 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1204,7 +1204,7 @@ def docs_upload_s3() { utils.docker_run('ubuntu_cpu', "push_docs ${env.FOLDER_NAME}", false) - archiveArtifacts 'docs/_build/versions-test.zip' + archiveArtifacts 'docs/_build/versions.zip' } else { sh 'echo Can not find website version for release. Please specify env var FOLDER_NAME in Jenkins pipeline' sh 'exit 1' From c7d791ba8a1610a56ab31e6649b238f098b137c5 Mon Sep 17 00:00:00 2001 From: Wei Chu Date: Mon, 1 Mar 2021 19:27:41 -0800 Subject: [PATCH 5/8] add backup version --- ci/docker/runtime_functions.sh | 12 ++++++------ ci/jenkins/Jenkins_steps.groovy | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 752c32bcc075..723f5420bea1 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1227,12 +1227,12 @@ push_docs() { fi mv master $folder_name popd - # back up versions folder - wget https://mxnet-website-static-artifacts.s3.us-east-2.amazonaws.com/versions.zip -O versions-backup.zip - aws s3 cp versions-backup.zip s3://mxnet-website-static-artifacts --acl public-read - - zip -r9 versions.zip versions/. - aws s3 cp versions.zip s3://mxnet-website-static-artifacts --acl public-read + zip -r9 versions-test.zip versions/. + # Upload versions folder + aws s3 cp versions-test.zip s3://mxnet-website-static-artifacts --acl public-read + # Backup versions folder with the latest version name + backup_file = "versions_backup_upto_$folder_name.zip" + aws s3 cp versions-test.zip s3://mxnet-website-static-artifacts/$backup_file --acl public-read popd } diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 03b325ac4a91..ccf7c34f6dde 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1204,7 +1204,7 @@ def docs_upload_s3() { utils.docker_run('ubuntu_cpu', "push_docs ${env.FOLDER_NAME}", false) - archiveArtifacts 'docs/_build/versions.zip' + archiveArtifacts 'docs/_build/versions-test.zip' } else { sh 'echo Can not find website version for release. Please specify env var FOLDER_NAME in Jenkins pipeline' sh 'exit 1' From 5154b99dfd94626b6f56d379b1157c98a3cab4f2 Mon Sep 17 00:00:00 2001 From: Wei Chu Date: Mon, 1 Mar 2021 20:44:25 -0800 Subject: [PATCH 6/8] remove space --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 723f5420bea1..ea7da868a94b 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1231,7 +1231,7 @@ push_docs() { # Upload versions folder aws s3 cp versions-test.zip s3://mxnet-website-static-artifacts --acl public-read # Backup versions folder with the latest version name - backup_file = "versions_backup_upto_$folder_name.zip" + backup_file="versions_backup_upto_$folder_name.zip" aws s3 cp versions-test.zip s3://mxnet-website-static-artifacts/$backup_file --acl public-read popd } From ee02c5e0121df63c6c4df6d78d12c40a2f2d519f Mon Sep 17 00:00:00 2001 From: Wei Chu Date: Tue, 2 Mar 2021 10:14:58 -0800 Subject: [PATCH 7/8] remove test --- ci/docker/runtime_functions.sh | 6 +++--- ci/jenkins/Jenkins_steps.groovy | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index ea7da868a94b..6659a6f63d6f 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1227,12 +1227,12 @@ push_docs() { fi mv master $folder_name popd - zip -r9 versions-test.zip versions/. + zip -r9 versions.zip versions/. # Upload versions folder - aws s3 cp versions-test.zip s3://mxnet-website-static-artifacts --acl public-read + aws s3 cp versions.zip s3://mxnet-website-static-artifacts --acl public-read # Backup versions folder with the latest version name backup_file="versions_backup_upto_$folder_name.zip" - aws s3 cp versions-test.zip s3://mxnet-website-static-artifacts/$backup_file --acl public-read + aws s3 cp versions.zip s3://mxnet-website-static-artifacts/$backup_file --acl public-read popd } diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index ccf7c34f6dde..03b325ac4a91 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1204,7 +1204,7 @@ def docs_upload_s3() { utils.docker_run('ubuntu_cpu', "push_docs ${env.FOLDER_NAME}", false) - archiveArtifacts 'docs/_build/versions-test.zip' + archiveArtifacts 'docs/_build/versions.zip' } else { sh 'echo Can not find website version for release. Please specify env var FOLDER_NAME in Jenkins pipeline' sh 'exit 1' From dd65c4eba072444b4c29c618d46db7450146ba68 Mon Sep 17 00:00:00 2001 From: Wei Chu Date: Fri, 5 Mar 2021 15:43:43 -0800 Subject: [PATCH 8/8] cp between s3 buckets --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 6659a6f63d6f..b8ba61643a39 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1232,7 +1232,7 @@ push_docs() { aws s3 cp versions.zip s3://mxnet-website-static-artifacts --acl public-read # Backup versions folder with the latest version name backup_file="versions_backup_upto_$folder_name.zip" - aws s3 cp versions.zip s3://mxnet-website-static-artifacts/$backup_file --acl public-read + aws s3 cp s3://mxnet-website-static-artifacts/versions.zip s3://mxnet-website-static-artifacts/$backup_file --acl public-read popd }