From 7716d768b9f4fe7b65f9dca50fe83883ae0ef17f Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Tue, 2 Mar 2021 21:46:35 +0000 Subject: [PATCH 01/10] Fix command for running rat check in v1.x branch. --- rat-excludes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rat-excludes b/rat-excludes index 787853668ba2..39d0cfefcb3a 100644 --- a/rat-excludes +++ b/rat-excludes @@ -21,7 +21,7 @@ # The following commands can be used to run a Apache RAT check locally - # Docker based 1-click-method: -# ci/build.py -p ubuntu_cpu test_rat_check +# ci/build.py -p ubuntu_rat /work/runtime_functions.sh nightly_test_rat_check # You can also manually download Apache RAT. For installation on Ubuntu: # sudo apt-get install maven -y #>/dev/null From 467402b2aa9a52c285385157666ca5005102baa5 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Tue, 2 Mar 2021 23:25:05 +0000 Subject: [PATCH 02/10] Add script for creating source archives. --- tools/create_source_archive.sh | 82 ++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100755 tools/create_source_archive.sh diff --git a/tools/create_source_archive.sh b/tools/create_source_archive.sh new file mode 100755 index 000000000000..98126a0202ce --- /dev/null +++ b/tools/create_source_archive.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +MXNET_BRANCH=$1 +MXNET_TAG=$2 +CMD=$(basename $0) + +if [[ -z "$MXNET_BRANCH" || -z "$MXNET_TAG" ]]; then + echo "Usage: $CMD " + echo " where is the branch the tag was cut from." + echo " is the tag you want to build a release for." + echo "" + echo " example: $CMD v1.8.x 1.8.0.rc3" + exit -1 +fi + +TAR=tar +if [[ $(uname) == "Darwin" ]]; then + TAR=gtar +fi + +# make sure gnu tar is installed +which $TAR > /dev/null +if [[ $? -ne 0 ]]; then + echo "It looks like you don't have GNU tar installed." + echo "" + echo "For OSX users, please install gnu-tar using the command 'brew install gnu-tar'" + exit -1 +fi + +SRCDIR=apache-mxnet-src-$MXNET_TAG-incubating +TARBALL=$SRCDIR.tar.gz + +# clone the repo and checkout the tag +echo "Cloning the MXNet repository..." +git clone -b $MXNET_BRANCH https://github.com/apache/incubator-mxnet.git $SRCDIR +pushd $SRCDIR +git submodule update --init --recursive +echo "Checking out tag $MXNET_TAG..." +git checkout $MXNET_TAG + +echo "Removing unwanted artifacts..." +#### IMPORTANT #### +# Remove artifacts which do not comply with the Apache Licensing Policy +rm -rf R-package +rm -rf 3rdparty/mkldnn/doc + +# Remove other artifacts we do not want contained in the source archive +rm -rf .DS_Store +rm -rf CODEOWNERS +find . -name ".git*" -print0 | xargs -0 rm -rf + +# run Apache RAT license checker to verify all source files are compliant +echo "Running Apache RAT License Checker..." +ci/build.py -p ubuntu_rat /work/runtime_functions.sh nightly_test_rat_check + +popd + +echo "Creating tarball $TARBALL..." +$TAR -czf $TARBALL $SRCDIR + +# sign the release tarball and create checksum file +gpg --armor --output $TARBALL.asc --detach-sig $TARBALL +shasum -a 512 $TARBALL > $TARBALL.sha512 + From 123cb72f0edf9a4572b16fa1b947578d5c5ad106 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 3 Mar 2021 17:33:58 +0000 Subject: [PATCH 03/10] Remove references to mx-theme and mathjax in LICENSE file, since we will not be distributing these in the source archive. --- LICENSE | 9 --------- 1 file changed, 9 deletions(-) diff --git a/LICENSE b/LICENSE index e862fd9d55bc..dee218603191 100644 --- a/LICENSE +++ b/LICENSE @@ -231,9 +231,6 @@ 3rdparty/onnx-tensorrt/third_party/onnx/third_party/benchmark 3rdparty/mkldnn/tests/benchdnn (Copy of the License available at top of current file) src/operator/special_functions-inl.h Cephes Library Functions (Copy of the License available at top of current file) - 3rdparty/mkldnn/doc/assets/mathjax (Copy of the License available at top of current file) - docs/python_docs/themes/mx-theme/mxtheme/static/material-design-icons-3.0.1 (Copy of the License available at top of current file) - docs/python_docs/themes/mx-theme/mxtheme/static/font/Roboto (Copy of the License available at top of current file) 3rdparty/tvm/3rdparty/bfloat16/bfloat16.cc (Copy of the License available at top of current file) ======================================================================================= @@ -245,7 +242,6 @@ 3rdparty/onnx-tensorrt 3rdparty/onnx-tensorrt/third_party/onnx docs/static_site/src/assets/js/clipboard.js - docs/python_docs/themes/mx-theme 3rdparty/intgemm 3rdparty/tvm/3rdparty/compiler-rt/builtin_fp16.h src/operator/nn/layer_norm.cc @@ -346,8 +342,3 @@ 3rdparty/tvm/3rdparty/rang - ======================================================================================= - SIL Open Font License (OFL) - ======================================================================================= - - docs/python_docs/themes/mx-theme/mxtheme/static/webfonts/ (Copy of the License available at licenses/OFL1_1) From 7acfe5be70337403ef344c356955469fcfa8b2b9 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 3 Mar 2021 17:48:03 +0000 Subject: [PATCH 04/10] Create a list of artifacts to be removed from source archive and use it in the create_source_archive.sh script. --- tools/create_source_archive.sh | 8 ++++++-- tools/source-exclude-artifacts.txt | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 tools/source-exclude-artifacts.txt diff --git a/tools/create_source_archive.sh b/tools/create_source_archive.sh index 98126a0202ce..78bcfb9fe13e 100755 --- a/tools/create_source_archive.sh +++ b/tools/create_source_archive.sh @@ -59,8 +59,12 @@ git checkout $MXNET_TAG echo "Removing unwanted artifacts..." #### IMPORTANT #### # Remove artifacts which do not comply with the Apache Licensing Policy -rm -rf R-package -rm -rf 3rdparty/mkldnn/doc +for d in $(cat tools/source-exclude-artifacts.txt | grep -v "^#"); do + if [[ -e $d ]]; then + echo "Removing $d from source archive..." + rm -rf $d + fi +done # Remove other artifacts we do not want contained in the source archive rm -rf .DS_Store diff --git a/tools/source-exclude-artifacts.txt b/tools/source-exclude-artifacts.txt new file mode 100644 index 000000000000..88f02b583a6e --- /dev/null +++ b/tools/source-exclude-artifacts.txt @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: +# This is a list of artifacts in the repository that should +# not be included in source release archives due to licensing +# restrictions. + +R-package +3rdparty/mkldnn/doc From fcee0019d7de6b1ba5445bedfaf72b0ebc1e17f1 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 3 Mar 2021 17:49:38 +0000 Subject: [PATCH 05/10] Revert "Remove references to mx-theme and mathjax in LICENSE file, since we will not be distributing these in the source archive." This reverts commit 123cb72f0edf9a4572b16fa1b947578d5c5ad106. --- LICENSE | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/LICENSE b/LICENSE index dee218603191..e862fd9d55bc 100644 --- a/LICENSE +++ b/LICENSE @@ -231,6 +231,9 @@ 3rdparty/onnx-tensorrt/third_party/onnx/third_party/benchmark 3rdparty/mkldnn/tests/benchdnn (Copy of the License available at top of current file) src/operator/special_functions-inl.h Cephes Library Functions (Copy of the License available at top of current file) + 3rdparty/mkldnn/doc/assets/mathjax (Copy of the License available at top of current file) + docs/python_docs/themes/mx-theme/mxtheme/static/material-design-icons-3.0.1 (Copy of the License available at top of current file) + docs/python_docs/themes/mx-theme/mxtheme/static/font/Roboto (Copy of the License available at top of current file) 3rdparty/tvm/3rdparty/bfloat16/bfloat16.cc (Copy of the License available at top of current file) ======================================================================================= @@ -242,6 +245,7 @@ 3rdparty/onnx-tensorrt 3rdparty/onnx-tensorrt/third_party/onnx docs/static_site/src/assets/js/clipboard.js + docs/python_docs/themes/mx-theme 3rdparty/intgemm 3rdparty/tvm/3rdparty/compiler-rt/builtin_fp16.h src/operator/nn/layer_norm.cc @@ -342,3 +346,8 @@ 3rdparty/tvm/3rdparty/rang + ======================================================================================= + SIL Open Font License (OFL) + ======================================================================================= + + docs/python_docs/themes/mx-theme/mxtheme/static/webfonts/ (Copy of the License available at licenses/OFL1_1) From 4f6170cc35848a9007a9ebffa2fa831d724a4949 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 3 Mar 2021 18:29:16 +0000 Subject: [PATCH 06/10] Use --exclude-vcs option to tar instead of find to remove most .git* directories. Also manually remove .github. --- tools/create_source_archive.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/create_source_archive.sh b/tools/create_source_archive.sh index 78bcfb9fe13e..edddd2662dfd 100755 --- a/tools/create_source_archive.sh +++ b/tools/create_source_archive.sh @@ -69,7 +69,7 @@ done # Remove other artifacts we do not want contained in the source archive rm -rf .DS_Store rm -rf CODEOWNERS -find . -name ".git*" -print0 | xargs -0 rm -rf +rm -rf .github # run Apache RAT license checker to verify all source files are compliant echo "Running Apache RAT License Checker..." @@ -78,7 +78,7 @@ ci/build.py -p ubuntu_rat /work/runtime_functions.sh nightly_test_rat_check popd echo "Creating tarball $TARBALL..." -$TAR -czf $TARBALL $SRCDIR +$TAR --exclude-vcs -czf $TARBALL $SRCDIR # sign the release tarball and create checksum file gpg --armor --output $TARBALL.asc --detach-sig $TARBALL From c79b063a49d8663b95c8fbfdcc847257ae7e0325 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 3 Mar 2021 19:09:02 +0000 Subject: [PATCH 07/10] Check out tag directly with submodules and limited depth to make it fast, remove branch parameter since we no longer need it. --- tools/create_source_archive.sh | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tools/create_source_archive.sh b/tools/create_source_archive.sh index edddd2662dfd..99f294ec0bff 100755 --- a/tools/create_source_archive.sh +++ b/tools/create_source_archive.sh @@ -18,16 +18,14 @@ set -e -MXNET_BRANCH=$1 -MXNET_TAG=$2 +MXNET_TAG=$1 CMD=$(basename $0) -if [[ -z "$MXNET_BRANCH" || -z "$MXNET_TAG" ]]; then - echo "Usage: $CMD " - echo " where is the branch the tag was cut from." - echo " is the tag you want to build a release for." +if [[ -z "$MXNET_TAG" ]]; then + echo "Usage: $CMD " + echo " where is the git tag you want to build a release for." echo "" - echo " example: $CMD v1.8.x 1.8.0.rc3" + echo " example: $CMD 1.8.0.rc3" exit -1 fi @@ -50,11 +48,10 @@ TARBALL=$SRCDIR.tar.gz # clone the repo and checkout the tag echo "Cloning the MXNet repository..." -git clone -b $MXNET_BRANCH https://github.com/apache/incubator-mxnet.git $SRCDIR +git clone -b $MXNET_TAG --depth 1 --recurse-submodules \ + --shallow-submodules https://github.com/apache/incubator-mxnet.git \ + $SRCDIR pushd $SRCDIR -git submodule update --init --recursive -echo "Checking out tag $MXNET_TAG..." -git checkout $MXNET_TAG echo "Removing unwanted artifacts..." #### IMPORTANT #### From 9e945609cb631cb25e5604eb7fbb926963cbc81c Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 3 Mar 2021 19:47:13 +0000 Subject: [PATCH 08/10] Verify that all files listed in the LICENSE file actually exist in the source tree. --- tools/create_source_archive.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/create_source_archive.sh b/tools/create_source_archive.sh index 99f294ec0bff..4c650de2c96f 100755 --- a/tools/create_source_archive.sh +++ b/tools/create_source_archive.sh @@ -68,10 +68,22 @@ rm -rf .DS_Store rm -rf CODEOWNERS rm -rf .github +# make sure all files referenced in LICENSE file still exist +echo "Making sure all paths referenced in LICENSE file exist..." +for f in $(cat LICENSE | grep "^\s*[0-9A-Za-z]*/[0-9A-Za-z]*" | awk '{print $1}'); do + echo "Checking if $f exists in source..." + if [[ ! -e $f ]]; then + echo -n "ERROR: Path $f is referenced in LICENSE file, but is not present " + echo "in source directory. Please update the LICENSE file." + exit -1 + fi +done + # run Apache RAT license checker to verify all source files are compliant echo "Running Apache RAT License Checker..." ci/build.py -p ubuntu_rat /work/runtime_functions.sh nightly_test_rat_check + popd echo "Creating tarball $TARBALL..." From ecb8e8895459e3838f42d9c906ed798636680bdd Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 3 Mar 2021 19:47:28 +0000 Subject: [PATCH 09/10] Remove 2 files from LICENSE that are no longer present in source tree. --- LICENSE | 2 -- 1 file changed, 2 deletions(-) diff --git a/LICENSE b/LICENSE index e862fd9d55bc..ff498644aa45 100644 --- a/LICENSE +++ b/LICENSE @@ -317,8 +317,6 @@ Apache-2.0 license + MIT License ======================================================================================= - src/serialization/cnpy.h (Copy of the AL2 License available at the top of this file, MIT License available at licenses/MIT) - src/serialization/cnpy.cc (Copy of the AL2 License available at the top of this file, MIT License available at licenses/MIT) 3rdparty/onnx-tensorrt/third_party/onnx/tools/protoc-gen-mypy.py (Copy of the referenced AL2 License available at top of current file) ======================================================================================= From 577558a001f990bea02606dfb436b661b6571ea7 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Thu, 4 Mar 2021 17:58:13 +0000 Subject: [PATCH 10/10] Remove lines from LICENSE file for items that are removed during source build process. --- tools/create_source_archive.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/create_source_archive.sh b/tools/create_source_archive.sh index 4c650de2c96f..6691a8b5cfc0 100755 --- a/tools/create_source_archive.sh +++ b/tools/create_source_archive.sh @@ -53,9 +53,9 @@ git clone -b $MXNET_TAG --depth 1 --recurse-submodules \ $SRCDIR pushd $SRCDIR -echo "Removing unwanted artifacts..." #### IMPORTANT #### # Remove artifacts which do not comply with the Apache Licensing Policy +echo "Removing unwanted artifacts..." for d in $(cat tools/source-exclude-artifacts.txt | grep -v "^#"); do if [[ -e $d ]]; then echo "Removing $d from source archive..." @@ -63,6 +63,17 @@ for d in $(cat tools/source-exclude-artifacts.txt | grep -v "^#"); do fi done +# Remove lines from LICENSE file for artifacts removed from source tree +echo "Removing lines from LICENSE for artifacts removed from source archive..." +for d in $(cat tools/source-exclude-artifacts.txt | grep -v "^#"); do + line=$(grep "$d" LICENSE) + if [[ $? -eq 0 && ! -z "$line" ]]; then + echo "Removing line from LICENSE: $line" + cat LICENSE | grep -v "$d" > LICENSE.new + mv -f LICENSE.new LICENSE + fi +done + # Remove other artifacts we do not want contained in the source archive rm -rf .DS_Store rm -rf CODEOWNERS