From 00a3b4fa27b3459d33a07ad0cd45277e9f1ab547 Mon Sep 17 00:00:00 2001 From: Matt Foley Date: Tue, 7 Apr 2020 15:51:12 -0700 Subject: [PATCH 01/14] Small changes to make dev-support/create-release tools less focused on hbase project only, while retaining special behaviors for hbase sub-projects which share the hbase dist and jira locations. --- dev-support/create-release/README.txt | 8 +-- .../create-release/do-release-docker.sh | 2 +- dev-support/create-release/release-build.sh | 46 +++++++++------- dev-support/create-release/release-tag.sh | 17 +++--- dev-support/create-release/release-util.sh | 52 ++++++++++++++----- dev-support/create-release/vote.tmpl | 13 ++--- 6 files changed, 86 insertions(+), 52 deletions(-) diff --git a/dev-support/create-release/README.txt b/dev-support/create-release/README.txt index 0d62671eb3ca..b3d1a7c65c14 100644 --- a/dev-support/create-release/README.txt +++ b/dev-support/create-release/README.txt @@ -19,7 +19,6 @@ Running a build on GCE is easy enough. Here are some notes if of use. Create an instance. 4CPU/15G/10G disk seems to work well enough. Once up, run the below to make your machine fit for RC building: - # Presuming debian-compatible OS $ sudo apt-get install -y git openjdk-8-jdk maven gnupg gnupg-agent # Install docker @@ -37,13 +36,14 @@ $ sudo add-apt-repository -y \ $ sudo apt-get update $ sudo apt-get install -y docker-ce docker-ce-cli containerd.io $ sudo usermod -a -G docker $USERID -# LOGOUT and then LOGIN again so $USERID shows as part of docker groupl +# LOGOUT and then LOGIN again so $USERID shows as part of docker group # Copy up private key for $USERID export from laptop and import on gce. $ gpg --import stack.duboce.net.asc $ export GPG_TTY=$(tty) # https://github.com/keybase/keybase-issues/issues/2798 $ eval $(gpg-agent --disable-scdaemon --daemon --no-grab --allow-preset-passphrase --default-cache-ttl=86400 --max-cache-ttl=86400) -$ git clone https://github.com/apache/hbase.git -$ cd hbase +$ export PROJECT="${PROJECT:-hbase}" +$ git clone https://github.com/apache/${PROJECT}.git +$ cd "${PROJECT}" $ mkdir ~/build $ ./dev-resources/create-release/do-release-docker.sh -d ~/build # etc. diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh index 3012d3cb2151..20680855f5f5 100755 --- a/dev-support/create-release/do-release-docker.sh +++ b/dev-support/create-release/do-release-docker.sh @@ -68,7 +68,7 @@ Options: -t [tag] tag for the hbase-rm docker image to use for building (default: "latest"). -j [path] path to local JDK installation to use building. By default the script will use openjdk8 installed in the docker image. - -p [project] project to build; default 'hbase'; alternatively, 'hbase-thirdparty', etc. + -p [project] project to build, such as 'hbase' or 'hbase-thirdparty'; defaults to $PROJECT env var -s [step] runs a single step of the process; valid steps are: tag, build, publish. if none specified, runs tag, then build, and then publish. EOF diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index 8d344104c09d..8182161d6e9d 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -29,7 +29,8 @@ function exit_with_usage { Usage: release-build.sh Creates build deliverables from a tag/commit. Arguments: - build Create binary packages and commit to dist.apache.org/repos/dist/dev/hbase/ + build Create binary packages and commit to dist.apache.org/repos/dist/dev/${PROJECT}/ + (or dist.apache.org/repos/dist/dev/hbase/${PROJECT}-${VERSION}/ in case of hbase sub-projects) publish-snapshot Publish snapshot release to Apache snapshots publish-release Publish a release to Apache release repo @@ -100,9 +101,13 @@ export LC_ALL=C.UTF-8 export LANG=C.UTF-8 # Commit ref to checkout when building -GIT_REF=${GIT_REF:-master} -RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/hbase" BASE_DIR=$(pwd) +GIT_REF=${GIT_REF:-master} +if [[ "$PROJECT" =~ ^hbase ]]; then + RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/hbase" +else + RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/${PROJECT}" +fi init_java init_mvn @@ -126,9 +131,6 @@ if [ -z "$VERSION" ]; then rm $TMP fi -# Profiles for publishing snapshots and release to Maven Central -PUBLISH_PROFILES="-P apache-release,release" - # This is a band-aid fix to avoid the failure of Maven nightly snapshot in some Jenkins # machines by explicitly calling /usr/sbin/lsof. Please see SPARK-22377 and the discussion # in its pull request. @@ -141,8 +143,6 @@ if [ -z "$PACKAGE_VERSION" ]; then PACKAGE_VERSION="${VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}" fi -DEST_DIR_NAME="$PACKAGE_VERSION" - git clean -d -f -x cd .. @@ -170,29 +170,35 @@ if [[ "$1" == "build" ]]; then echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Done building binary distribution" if ! is_dry_run; then - svn co --depth=empty $RELEASE_STAGING_LOCATION svn-hbase - rm -rf "svn-hbase/${DEST_DIR_NAME}" - mkdir -p "svn-hbase/${DEST_DIR_NAME}" + if [[ "$PROJECT" =~ ^hbase- ]]; then + DEST_DIR_NAME="${PROJECT}-${PACKAGE_VERSION}" + else + DEST_DIR_NAME="$PACKAGE_VERSION" + fi + svn_target="svn-${PROJECT}" + svn co --depth=empty "$RELEASE_STAGING_LOCATION" "$svn_target" + rm -rf "$svn_target/${DEST_DIR_NAME}" + mkdir -p "$svn_target/${DEST_DIR_NAME}" echo "Copying release tarballs" - cp ${PROJECT}-*.tar.* "svn-hbase/${DEST_DIR_NAME}/" - cp ${PROJECT}/CHANGES.md "svn-hbase/${DEST_DIR_NAME}/" - cp ${PROJECT}/RELEASENOTES.md "svn-hbase/${DEST_DIR_NAME}/" + cp ${PROJECT}-*.tar.* "$svn_target/${DEST_DIR_NAME}/" + cp ${PROJECT}/CHANGES.md "$svn_target/${DEST_DIR_NAME}/" + cp ${PROJECT}/RELEASENOTES.md "$svn_target/${DEST_DIR_NAME}/" shopt -s nocasematch # Generate api report only if project is hbase for now. if [ "${PROJECT}" == "hbase" ]; then # This script usually reports an errcode along w/ the report. generate_api_report ./${PROJECT} "${API_DIFF_TAG}" "${PACKAGE_VERSION}" || true - cp api*.html "svn-hbase/${DEST_DIR_NAME}/" + cp api*.html "$svn_target/${DEST_DIR_NAME}/" fi shopt -u nocasematch - svn add "svn-hbase/${DEST_DIR_NAME}" + svn add "$svn_target/${DEST_DIR_NAME}" - cd svn-hbase + cd "$svn_target" svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Apache ${PROJECT} $PACKAGE_VERSION" --no-auth-cache cd .. - rm -rf svn-hbase + rm -rf "$svn_target" fi exit 0 @@ -210,7 +216,7 @@ if [[ "$1" == "publish-snapshot" ]]; then fi # Coerce the requested version $MVN versions:set -DnewVersion=$VERSION - $MVN --settings $tmp_settings -DskipTests "$PUBLISH_PROFILES" deploy + $MVN --settings $tmp_settings -DskipTests -P "${PUBLISH_PROFILES}" deploy cd .. exit 0 fi @@ -230,7 +236,7 @@ if [[ "$1" == "publish-release" ]]; then fi echo "Staging release in nexus" if ! MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings "$tmp_settings" \ - -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES}" \ + -DskipTests -Dcheckstyle.skip=true -P "${PUBLISH_PROFILES}" \ -Dmaven.repo.local="${tmp_repo}" \ "${mvn_goals[@]}" > "${BASE_DIR}/mvn_deploy.log"; then echo "Staging build failed, see 'mvn_deploy.log' for details." >&2 diff --git a/dev-support/create-release/release-tag.sh b/dev-support/create-release/release-tag.sh index 28ade2c0915f..0aff44336ec9 100755 --- a/dev-support/create-release/release-tag.sh +++ b/dev-support/create-release/release-tag.sh @@ -71,16 +71,19 @@ ASF_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git" encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')") encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')") git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b $GIT_BRANCH -# NOTE: Here we are prepending project name on version for fetching -# changes from the HBASE JIRA. It has issues for hbase, hbase-conectors, -# hbase-operator-tools, etc. + +# 'update_releasenotes' searches the project's Jira for issues where 'Fix Version' matches specified +# $jira_fix_version. For most projects this is same as ${RELEASE_VERSION}. However, all the 'hbase-*' +# projects share the same HBASE jira name. To make this work, by convention, the HBASE jira "Fix Version" +# field values have the sub-project name pre-pended, as in "hbase-operator-tools-1.0.0". +# So, here we prepend the project name to the version, but only for the hbase sub-projects. +jira_fix_version="${RELEASE_VERSION}" shopt -s nocasematch -if [ "${PROJECT}" != "hbase" ]; then - # Needs the '-' on the end. - prefix="${PROJECT}-" +if [[ "${PROJECT}" =~ ^hbase- ]]; then + jira_fix_version="${PROJECT}-${RELEASE_VERSION}" fi shopt -u nocasematch -update_releasenotes `pwd`/${PROJECT} "${prefix}${RELEASE_VERSION}" +update_releasenotes `pwd`/${PROJECT} "${jira_fix_version}" cd ${PROJECT} diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index 99a8e902d0dc..1a88b1680ca7 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -21,6 +21,10 @@ GPG="gpg --pinentry-mode loopback --no-tty --batch" YETUS_VERSION=${YETUS_VERSION:-0.11.1} set -e +# Profiles for publishing snapshots and release to Maven Central and Dist +# Use with '-P' maven option +PUBLISH_PROFILES="apache-release,release" + function error { echo "$*" exit 1 @@ -289,34 +293,54 @@ function generate_api_report { cp ${project}/target/compat-check/report.html "./api_compare_${previous_version}_to_${release_tag}.html" } +# Look up the Jira name associated with project. +# Currently all the 'hbase-*' projects share the same HBASE jira name. This works because, +# by convention, the HBASE jira "Fix Version" field values have the sub-project name pre-pended, +# as in "hbase-operator-tools-1.0.0". +# TODO: For non-hbase-related projects, enhance this to use Jira API query instead of text lookup. +function get_jira_name { + local project="$1" + local jira_name + case "${project}" in + hbase*) jira_name="HBASE";; + *) jira_name="";; + esac + if [[ -z "$jira_name" ]]; then + echo "Sorry, can't determine the Jira name for project $project" >&2 + exit 1 + fi + echo "$jira_name" +} + # Update the CHANGES.md # DOES NOT DO COMMITS! Caller should do that. # yetus requires python2 to be on the path. function update_releasenotes { - local project="$1" - local release_version="$2" + local project_dir="$1" + local jira_fix_version="$2" local yetus="apache-yetus-${YETUS_VERSION}" + local jira_project="$(get_jira_name "$(basename "$project_dir")")" wget -qO- "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/yetus/${YETUS_VERSION}/${yetus}-bin.tar.gz" | \ tar xvz -C . || exit - cd ./${yetus} || exit - ./bin/releasedocmaker -p HBASE --fileversions -v ${release_version} -l --sortorder=newer --skip-credits + cd "./${yetus}" || exit + ./bin/releasedocmaker -p "${jira_project}" --fileversions -v "${jira_fix_version}" -l --sortorder=newer --skip-credits # First clear out the changes written by previous RCs. pwd - sed -i -e "/^## Release ${release_version}/,/^## Release/ {//!d; /^## Release ${release_version}/d;}" \ - ${project}/CHANGES.md || true - sed -i -e "/^# HBASE ${release_version} Release Notes/,/^# HBASE/{//!d; /^# HBASE ${release_version} Release Notes/d;}" \ - ${project}/RELEASENOTES.md || true + sed -i -e "/^## Release ${jira_fix_version}/,/^## Release/ {//!d; /^## Release ${jira_fix_version}/d;}" \ + "${project_dir}/CHANGES.md" || true + sed -i -e "/^# ${jira_project} ${jira_fix_version} Release Notes/,/^# ${jira_project}/{//!d; /^# ${jira_project} ${jira_fix_version} Release Notes/d;}" \ + "${project_dir}/RELEASENOTES.md" || true # The above generates RELEASENOTES.X.X.X.md and CHANGELOG.X.X.X.md. - # To insert into project CHANGES.me...need to cut the top off the + # To insert into project's CHANGES.md...need to cut the top off the # CHANGELOG.X.X.X.md file removing license and first line and then # insert it after the license comment closing where we have a # DO NOT REMOVE marker text! - sed -i -e '/## Release/,$!d' CHANGELOG.${release_version}.md - sed -i -e "/DO NOT REMOVE/r CHANGELOG.${release_version}.md" ${project}/CHANGES.md + sed -i -e '/## Release/,$!d' "CHANGELOG.${jira_fix_version}.md" + sed -i -e "/DO NOT REMOVE/r CHANGELOG.${jira_fix_version}.md" "${project_dir}/CHANGES.md" # Similar for RELEASENOTES but slightly different. - sed -i -e '/Release Notes/,$!d' RELEASENOTES.${release_version}.md - sed -i -e "/DO NOT REMOVE/r RELEASENOTES.${release_version}.md" ${project}/RELEASENOTES.md + sed -i -e '/Release Notes/,$!d' "RELEASENOTES.${jira_fix_version}.md" + sed -i -e "/DO NOT REMOVE/r RELEASENOTES.${jira_fix_version}.md" "${project_dir}/RELEASENOTES.md" cd .. || exit } @@ -375,7 +399,7 @@ make_binary_release() { MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings site -DskipTests \ -Dmaven.repo.local="${tmp_repo}" MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings install assembly:single -DskipTests \ - -Dcheckstyle.skip=true "${PUBLISH_PROFILES}" -Dmaven.repo.local="${tmp_repo}" + -Dcheckstyle.skip=true -P "${PUBLISH_PROFILES}" -Dmaven.repo.local="${tmp_repo}" # Check there is a bin gz output. The build may not produce one: e.g. hbase-thirdparty. f_bin_tgz="./${PROJECT}-assembly/target/${basename}*-bin.tar.gz" diff --git a/dev-support/create-release/vote.tmpl b/dev-support/create-release/vote.tmpl index c9c4b742e561..7810cd22d281 100644 --- a/dev-support/create-release/vote.tmpl +++ b/dev-support/create-release/vote.tmpl @@ -8,23 +8,24 @@ The VOTE will remain open for at least 72 hours. The tag to be voted on is ${RELEASE_TAG}: -https://github.com/apache/${PROJECT}/tree/${RELEASE_TAG} + https://github.com/apache/${PROJECT}/tree/${RELEASE_TAG} The release files, including signatures, digests, as well as CHANGES.md and RELEASENOTES.md included in this RC can be found at: - https://dist.apache.org/repos/dist/dev/hbase/${RELEASE_TAG}/ + https://dist.apache.org/repos/dist/dev/hbase/${RELEASE_TAG}/ Maven artifacts are available in a staging repository at: - https://repository.apache.org/content/repositories/${staged_repo_id}/ + https://repository.apache.org/content/repositories/${staged_repo_id}/ Artifacts were signed with the ${GPG_KEY} key which can be found in: - https://dist.apache.org/repos/dist/release/hbase/KEYS + https://dist.apache.org/repos/dist/release/hbase/KEYS - To learn more about apache ${PROJECT_TEXT}, please see -http://hbase.apache.org/ +To learn more about Apache ${PROJECT_TEXT}, please see + + http://hbase.apache.org/ Thanks, Your HBase Release Manager From db4bab6f57d61ecd89ff0656fa3d9890ce27de0c Mon Sep 17 00:00:00 2001 From: Matt Foley Date: Wed, 1 Apr 2020 16:14:09 -0700 Subject: [PATCH 02/14] narrow 'dry-run' limits so see svn activity up to just before check-in. --- dev-support/create-release/release-build.sh | 54 +++++++++++---------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index 8182161d6e9d..534a6835ef9e 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -169,36 +169,40 @@ if [[ "$1" == "build" ]]; then make_binary_release "${PROJECT}" "${VERSION}" echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Done building binary distribution" - if ! is_dry_run; then - if [[ "$PROJECT" =~ ^hbase- ]]; then - DEST_DIR_NAME="${PROJECT}-${PACKAGE_VERSION}" - else - DEST_DIR_NAME="$PACKAGE_VERSION" - fi - svn_target="svn-${PROJECT}" - svn co --depth=empty "$RELEASE_STAGING_LOCATION" "$svn_target" - rm -rf "$svn_target/${DEST_DIR_NAME}" - mkdir -p "$svn_target/${DEST_DIR_NAME}" - - echo "Copying release tarballs" - cp ${PROJECT}-*.tar.* "$svn_target/${DEST_DIR_NAME}/" - cp ${PROJECT}/CHANGES.md "$svn_target/${DEST_DIR_NAME}/" - cp ${PROJECT}/RELEASENOTES.md "$svn_target/${DEST_DIR_NAME}/" - shopt -s nocasematch - # Generate api report only if project is hbase for now. - if [ "${PROJECT}" == "hbase" ]; then - # This script usually reports an errcode along w/ the report. - generate_api_report ./${PROJECT} "${API_DIFF_TAG}" "${PACKAGE_VERSION}" || true - cp api*.html "$svn_target/${DEST_DIR_NAME}/" - fi - shopt -u nocasematch - - svn add "$svn_target/${DEST_DIR_NAME}" + if [[ "$PROJECT" =~ ^hbase- ]]; then + DEST_DIR_NAME="${PROJECT}-${PACKAGE_VERSION}" + else + DEST_DIR_NAME="$PACKAGE_VERSION" + fi + svn_target="svn-${PROJECT}" + svn co --depth=empty "$RELEASE_STAGING_LOCATION" "$svn_target" + rm -rf "$svn_target/${DEST_DIR_NAME}" + mkdir -p "$svn_target/${DEST_DIR_NAME}" + + echo "Copying release tarballs" + cp ${PROJECT}-*.tar.* "$svn_target/${DEST_DIR_NAME}/" + cp ${PROJECT}/CHANGES.md "$svn_target/${DEST_DIR_NAME}/" + cp ${PROJECT}/RELEASENOTES.md "$svn_target/${DEST_DIR_NAME}/" + shopt -s nocasematch + # Generate api report only if project is hbase for now. + if [ "${PROJECT}" == "hbase" ]; then + # This script usually reports an errcode along w/ the report. + generate_api_report ./${PROJECT} "${API_DIFF_TAG}" "${PACKAGE_VERSION}" || true + cp api*.html "$svn_target/${DEST_DIR_NAME}/" + fi + shopt -u nocasematch + svn add "$svn_target/${DEST_DIR_NAME}" + + if ! is_dry_run; then cd "$svn_target" svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Apache ${PROJECT} $PACKAGE_VERSION" --no-auth-cache cd .. rm -rf "$svn_target" + else + mv "$svn_target/${DEST_DIR_NAME}" "${svn_target}_${DEST_DIR_NAME}.dist" + echo "svn-managed 'dist' directory with release tarballs, CHANGES.md and RELEASENOTES.md available as $(pwd)/${svn_target}_${DEST_DIR_NAME}.dist" + rm -rf "$svn_target" fi exit 0 From afc35f67535f07c0de6b05201a545f0f59c4f7ac Mon Sep 17 00:00:00 2001 From: Matt Foley Date: Thu, 2 Apr 2020 14:45:51 -0700 Subject: [PATCH 03/14] Fix several typos and, in case of dry run, enable build step to chain from tag step by keeping the tagged git repo. --- dev-support/create-release/README.txt | 2 +- dev-support/create-release/do-release-docker.sh | 1 - dev-support/create-release/do-release.sh | 10 +++++++--- dev-support/create-release/release-build.sh | 11 ++++++++--- dev-support/create-release/release-util.sh | 3 ++- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/dev-support/create-release/README.txt b/dev-support/create-release/README.txt index b3d1a7c65c14..d5d499a8ab7f 100644 --- a/dev-support/create-release/README.txt +++ b/dev-support/create-release/README.txt @@ -5,7 +5,7 @@ For usage, pass '-h': $ ./do-release-docker.sh -h -To run a build w/o invoking docker (not recommeneded!), use +To run a build w/o invoking docker (not recommended!), use _do_release.sh_. It does not take parameters. It will ask you what commands to run with taking defaults from environment. diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh index 20680855f5f5..b37b0c8b55fa 100755 --- a/dev-support/create-release/do-release-docker.sh +++ b/dev-support/create-release/do-release-docker.sh @@ -153,7 +153,6 @@ GPG_KEY=$GPG_KEY ASF_PASSWORD=$ASF_PASSWORD GPG_PASSPHRASE=$GPG_PASSPHRASE RELEASE_STEP=$RELEASE_STEP -RELEASE_STEP=$RELEASE_STEP API_DIFF_TAG=$API_DIFF_TAG EOF diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh index 1c10dfc1dc0a..d5f91b7afd0f 100755 --- a/dev-support/create-release/do-release.sh +++ b/dev-support/create-release/do-release.sh @@ -69,9 +69,13 @@ function should_build { if should_build "tag" && [ $SKIP_TAG = 0 ]; then run_silent "Creating release tag $RELEASE_TAG..." "tag.log" \ "$SELF/release-tag.sh" - echo "It may take some time for the tag to be synchronized to github." - echo "Press enter when you've verified that the new tag ($RELEASE_TAG) is available." - read + if is_dry_run; then + export TAG_SAME_DRY_RUN="true"; + else + echo "It may take some time for the tag to be synchronized to github." + echo "Press enter when you've verified that the new tag ($RELEASE_TAG) is available." + read + fi else echo "Skipping tag creation for $RELEASE_TAG." fi diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index 534a6835ef9e..06379b62193f 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -115,9 +115,14 @@ init_python # Print out subset of perl version. perl --version | grep 'This is' -rm -rf ${PROJECT} -ASF_REPO="${ASF_REPO:-https://gitbox.apache.org/repos/asf/${PROJECT}.git}" -git clone "$ASF_REPO" +rm -rf "${PROJECT}" +# in case of dry run, enable build step to chain from tag step +if is_dry_run && [[ "${TAG_SAME_DRY_RUN:-}" == "true" && -d "${PROJECT}.tag" ]]; then + ln -s "${PROJECT}.tag" "${PROJECT}" +else + ASF_REPO="${ASF_REPO:-https://gitbox.apache.org/repos/asf/${PROJECT}.git}" + git clone "$ASF_REPO" "${PROJECT}" +fi cd ${PROJECT} git checkout $GIT_REF git_hash=`git rev-parse --short HEAD` diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index 1a88b1680ca7..e3c1aba2d95f 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -96,7 +96,7 @@ function get_api_diff_version { api_diff_tag="rel/$((major - 1)).0.0" fi fi - api_diff_tag=$(read_config "api_diff_tag", "$api_diff_tag") + api_diff_tag=$(read_config "api_diff_tag" "$api_diff_tag") echo $api_diff_tag } @@ -217,6 +217,7 @@ ASF_USERNAME: $ASF_USERNAME GPG_KEY: $GPG_KEY GIT_NAME: $GIT_NAME GIT_EMAIL: $GIT_EMAIL +DRY_RUN: $(is_dry_run && echo "yes" || echo "NO, THIS BUILD WILL BE PUBLISHED!") ================ EOF From 2e18e2885d385de7a48d82e17250bb96f466b96c Mon Sep 17 00:00:00 2001 From: Matt Foley Date: Tue, 7 Apr 2020 17:25:23 -0700 Subject: [PATCH 04/14] Changed terminology of release publish steps, from to . In fact, what was formerly called "build" built the distribution tarballs AND published them to dist, while "publish" built the maven artifacts AND published them to Nexus. The new terminology clarifies what's happening, and removes the appearance of order dependendency. --- .../create-release/do-release-docker.sh | 6 +++--- dev-support/create-release/do-release.sh | 12 ++++++------ dev-support/create-release/release-build.sh | 19 +++++++++---------- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh index b37b0c8b55fa..f12539ade0a2 100755 --- a/dev-support/create-release/do-release-docker.sh +++ b/dev-support/create-release/do-release-docker.sh @@ -47,7 +47,7 @@ # set -e -# Set this building other hbase repos: e.g. PROJECT=hbase-operator-tools +# Set this to build other hbase repos: e.g. PROJECT=hbase-operator-tools export PROJECT="${PROJECT:-hbase}" SELF=$(cd $(dirname "$0") && pwd) @@ -69,8 +69,8 @@ Options: -j [path] path to local JDK installation to use building. By default the script will use openjdk8 installed in the docker image. -p [project] project to build, such as 'hbase' or 'hbase-thirdparty'; defaults to $PROJECT env var - -s [step] runs a single step of the process; valid steps are: tag, build, publish. if - none specified, runs tag, then build, and then publish. + -s [step] runs a single step of the process; valid steps are: tag|publish-dist|publish-release. + If none specified, runs tag, then publish-dist, and then publish-release. EOF } diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh index d5f91b7afd0f..17db1cd69b4c 100755 --- a/dev-support/create-release/do-release.sh +++ b/dev-support/create-release/do-release.sh @@ -80,16 +80,16 @@ else echo "Skipping tag creation for $RELEASE_TAG." fi -if should_build "build"; then - run_silent "Building ${PROJECT}..." "build.log" \ - "$SELF/release-build.sh" build +if should_build "publish-dist"; then + run_silent "Publishing distribution packages (tarballs)" "build.log" \ + "$SELF/release-build.sh" publish-dist else - echo "Skipping build step." + echo "Skipping publish-dist step." fi -if should_build "publish"; then +if should_build "publish-release"; then run_silent "Publishing release" "publish.log" \ "$SELF/release-build.sh" publish-release else - echo "Skipping publish step." + echo "Skipping publish-release step." fi diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index 06379b62193f..72626d6449ae 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -26,13 +26,12 @@ SELF=$(cd $(dirname $0) && pwd) # Print usage and exit. function exit_with_usage { cat << EOF -Usage: release-build.sh -Creates build deliverables from a tag/commit. +Usage: release-build.sh +Creates release deliverables from a tag or commit. Arguments: - build Create binary packages and commit to dist.apache.org/repos/dist/dev/${PROJECT}/ - (or dist.apache.org/repos/dist/dev/hbase/${PROJECT}-${VERSION}/ in case of hbase sub-projects) - publish-snapshot Publish snapshot release to Apache snapshots - publish-release Publish a release to Apache release repo + publish-dist Build and publish distribution packages (tarballs) to Apache dist repo + publish-snapshot Build and publish maven artifacts snapshot release to Apache snapshots repo + publish-release Build and publish maven artifacts release to Apache release repo All other inputs are environment variables: GIT_REF - Release tag or commit to build from @@ -48,7 +47,7 @@ Set REPO environment to full path to repo to use to avoid re-downloading dependencies on each run. For example: - $ PROJECT="hbase-operator-tools" ASF_USERNAME=NAME ASF_PASSWORD=PASSWORD GPG_PASSPHRASE=PASSWORD GPG_KEY=stack@apache.org ./release-build.sh build + $ PROJECT="hbase-operator-tools" ASF_USERNAME=NAME ASF_PASSWORD=PASSWORD GPG_PASSPHRASE=PASSWORD GPG_KEY=stack@apache.org ./release-build.sh publish-dist EOF exit 1 } @@ -116,7 +115,7 @@ init_python perl --version | grep 'This is' rm -rf "${PROJECT}" -# in case of dry run, enable build step to chain from tag step +# in case of dry run, enable publish steps to chain from tag step if is_dry_run && [[ "${TAG_SAME_DRY_RUN:-}" == "true" && -d "${PROJECT}.tag" ]]; then ln -s "${PROJECT}.tag" "${PROJECT}" else @@ -162,7 +161,7 @@ echo "" >> "$tmp_settings" echo "" >> "$tmp_settings" export tmp_settings -if [[ "$1" == "build" ]]; then +if [[ "$1" == "publish-dist" ]]; then # Source and binary tarballs echo "Packaging release source tarballs" make_src_release "${PROJECT}" "${VERSION}" @@ -215,7 +214,7 @@ fi if [[ "$1" == "publish-snapshot" ]]; then cd "${PROJECT}" - # Publish ${PROJECT} to Maven release repo + # Publish ${PROJECT} to Maven snapshot repo echo "Deploying ${PROJECT} SNAPSHOT at '$GIT_REF' ($git_hash)" echo "Publish version is $VERSION" if [[ ! $VERSION == *"SNAPSHOT"* ]]; then From 3c11815ac2047f94c29ff1e6d4794f6ab717352a Mon Sep 17 00:00:00 2001 From: Matt Foley Date: Fri, 3 Apr 2020 12:34:39 -0700 Subject: [PATCH 05/14] Improve Maven settings.xml file, and names of variables related to it. Remove unnecessary use of "-Dmaven.repo.local", put it in settings.xml instead. Stop putting password literals in settings.xml. (It's therefore no longer necessary to delete settings.xml in cleanup, but left that since there is no harm.) --- dev-support/create-release/release-build.sh | 43 ++++++++++++--------- dev-support/create-release/release-util.sh | 10 ++--- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index 72626d6449ae..7fed9bc281c6 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -43,7 +43,7 @@ All other inputs are environment variables: GPG_PASSPHRASE - Passphrase for GPG key PROJECT - The project to build. No default. -Set REPO environment to full path to repo to use +Set REPO env variable to full path of a directory to use as local mvn repo (dependencies cache) to avoid re-downloading dependencies on each run. For example: @@ -56,11 +56,11 @@ set -e function cleanup { echo "Cleaning up temp settings file." >&2 - rm "${tmp_settings}" &> /dev/null || true + rm "${MAVEN_SETTINGS_FILE}" &> /dev/null || true # If REPO was set, then leave things be. Otherwise if we defined a repo clean it out. - if [[ -z "${REPO}" ]] && [[ -n "${tmp_repo}" ]]; then - echo "Cleaning up temp repo in '${tmp_repo}'. set REPO to reuse downloads." >&2 - rm -rf "${tmp_repo}" &> /dev/null || true + if [[ -z "${REPO}" ]] && [[ -n "${MAVEN_LOCAL_REPO}" ]]; then + echo "Cleaning up temp repo in '${MAVEN_LOCAL_REPO}'. set REPO to reuse downloads." >&2 + rm -rf "${MAVEN_LOCAL_REPO}" &> /dev/null || true fi } @@ -150,16 +150,24 @@ fi git clean -d -f -x cd .. -tmp_repo="${REPO:-`pwd`/$(mktemp -d hbase-repo-XXXXX)}" -tmp_settings="/${tmp_repo}/tmp-settings.xml" -echo "" > "$tmp_settings" -echo "apache.snapshots.https$ASF_USERNAME" >> "$tmp_settings" -echo "$ASF_PASSWORD" >> "$tmp_settings" -echo "apache.releases.https$ASF_USERNAME" >> "$tmp_settings" -echo "$ASF_PASSWORD" >> "$tmp_settings" -echo "" >> "$tmp_settings" -echo "" >> "$tmp_settings" -export tmp_settings +MAVEN_LOCAL_REPO="${REPO:-$(pwd)/$(mktemp -d hbase-repo-XXXXX)}" +MAVEN_SETTINGS_FILE="/${MAVEN_LOCAL_REPO}/tmp-settings.xml" +export MAVEN_SETTINGS_FILE MAVEN_LOCAL_REPO ASF_USERNAME ASF_PASSWORD +# reference passwords from env rather than storing in the settings.xml file. +cat <<'EOF' > "$MAVEN_SETTINGS_FILE" + + + /${env.MAVEN_LOCAL_REPO} + + apache.snapshots.https${env.ASF_USERNAME} + ${env.ASF_PASSWORD} + apache.releases.https${env.ASF_USERNAME} + ${env.ASF_PASSWORD} + + +EOF if [[ "$1" == "publish-dist" ]]; then # Source and binary tarballs @@ -224,7 +232,7 @@ if [[ "$1" == "publish-snapshot" ]]; then fi # Coerce the requested version $MVN versions:set -DnewVersion=$VERSION - $MVN --settings $tmp_settings -DskipTests -P "${PUBLISH_PROFILES}" deploy + $MVN --settings $MAVEN_SETTINGS_FILE -DskipTests -P "${PUBLISH_PROFILES}" deploy cd .. exit 0 fi @@ -243,9 +251,8 @@ if [[ "$1" == "publish-release" ]]; then mvn_goals=("${mvn_goals[@]}" deploy) fi echo "Staging release in nexus" - if ! MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings "$tmp_settings" \ + if ! MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings "$MAVEN_SETTINGS_FILE" \ -DskipTests -Dcheckstyle.skip=true -P "${PUBLISH_PROFILES}" \ - -Dmaven.repo.local="${tmp_repo}" \ "${mvn_goals[@]}" > "${BASE_DIR}/mvn_deploy.log"; then echo "Staging build failed, see 'mvn_deploy.log' for details." >&2 exit 1 diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index e3c1aba2d95f..fcb7523651a7 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -395,12 +395,10 @@ make_binary_release() { # a third to assemble the binary artifact. Trying to do # all in the one invocation fails; a problem in our # assembly spec to in maven. TODO. Meantime, three invocations. - MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings clean install -DskipTests \ - -Dmaven.repo.local="${tmp_repo}" - MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings site -DskipTests \ - -Dmaven.repo.local="${tmp_repo}" - MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings install assembly:single -DskipTests \ - -Dcheckstyle.skip=true -P "${PUBLISH_PROFILES}" -Dmaven.repo.local="${tmp_repo}" + MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $MAVEN_SETTINGS_FILE clean install -DskipTests + MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $MAVEN_SETTINGS_FILE site -DskipTests + MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $MAVEN_SETTINGS_FILE install assembly:single -DskipTests \ + -Dcheckstyle.skip=true -P "${PUBLISH_PROFILES}" # Check there is a bin gz output. The build may not produce one: e.g. hbase-thirdparty. f_bin_tgz="./${PROJECT}-assembly/target/${basename}*-bin.tar.gz" From 8ae7456ec9e9e4556bef4bb9f3587cc0353b8aaa Mon Sep 17 00:00:00 2001 From: Matt Foley Date: Fri, 3 Apr 2020 16:20:04 -0700 Subject: [PATCH 06/14] Merge 'tag' into release-build.sh as another stage. Also found and fixed a couple bugs in the current release-tag.sh. --- dev-support/create-release/do-release.sh | 6 +- dev-support/create-release/release-build.sh | 143 ++++++++++++++------ dev-support/create-release/release-tag.sh | 5 +- dev-support/create-release/release-util.sh | 25 ++++ 4 files changed, 129 insertions(+), 50 deletions(-) diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh index 17db1cd69b4c..10f57321da65 100755 --- a/dev-support/create-release/do-release.sh +++ b/dev-support/create-release/do-release.sh @@ -68,7 +68,7 @@ function should_build { if should_build "tag" && [ $SKIP_TAG = 0 ]; then run_silent "Creating release tag $RELEASE_TAG..." "tag.log" \ - "$SELF/release-tag.sh" + "$SELF/release-build.sh" tag if is_dry_run; then export TAG_SAME_DRY_RUN="true"; else @@ -81,14 +81,14 @@ else fi if should_build "publish-dist"; then - run_silent "Publishing distribution packages (tarballs)" "build.log" \ + run_silent "Publishing distribution packages (tarballs)" "publish-dist.log" \ "$SELF/release-build.sh" publish-dist else echo "Skipping publish-dist step." fi if should_build "publish-release"; then - run_silent "Publishing release" "publish.log" \ + run_silent "Publishing release" "publish-release.log" \ "$SELF/release-build.sh" publish-release else echo "Skipping publish-release step." diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index 7fed9bc281c6..64d8309165e9 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -25,26 +25,39 @@ SELF=$(cd $(dirname $0) && pwd) # Print usage and exit. function exit_with_usage { - cat << EOF -Usage: release-build.sh + cat <<'EOF' +Usage: release-build.sh Creates release deliverables from a tag or commit. Arguments: + tag Prepares for release on specified git branch: Set release version, create release tag, + increment version for ongoing dev, and publish to Apache git repo publish-dist Build and publish distribution packages (tarballs) to Apache dist repo publish-snapshot Build and publish maven artifacts snapshot release to Apache snapshots repo publish-release Build and publish maven artifacts release to Apache release repo All other inputs are environment variables: - GIT_REF - Release tag or commit to build from - PACKAGE_VERSION - Release identifier in top level package directory (e.g. 2.1.2RC1) - VERSION - (optional) Version of project being built (e.g. 2.1.2) - ASF_USERNAME - Username of ASF committer account - ASF_PASSWORD - Password of ASF committer account - GPG_KEY - GPG key used to sign release artifacts - GPG_PASSPHRASE - Passphrase for GPG key - PROJECT - The project to build. No default. - -Set REPO env variable to full path of a directory to use as local mvn repo (dependencies cache) -to avoid re-downloading dependencies on each run. +Used for 'tag' and 'publish': + PROJECT - The project to build. No default. + ASF_USERNAME - Username of ASF committer account + ASF_PASSWORD - Password of ASF committer account + +Used only for 'tag': + GIT_NAME - Name to use with git + GIT_EMAIL - E-mail address to use with git + GIT_BRANCH - Git branch on which to make release + RELEASE_VERSION - Version used in pom files for release + RELEASE_TAG - Name of release tag + NEXT_VERSION - Development version after release + +Used only for 'publish': + GIT_REF - Release tag or commit to build from + VERSION - Version of project to be built (e.g. 2.1.2). + Optional for 'publish', as it defaults to the version in pom at GIT_REF. + PACKAGE_VERSION - Release identifier in top level dist directory (e.g. 2.1.2RC1) + GPG_KEY - GPG key id (usually email addr) used to sign release artifacts + GPG_PASSPHRASE - Passphrase for GPG key + REPO - Set to full path of a directory to use as maven local repo (dependencies cache) + to avoid re-downloading dependencies for each stage. For example: $ PROJECT="hbase-operator-tools" ASF_USERNAME=NAME ASF_PASSWORD=PASSWORD GPG_PASSPHRASE=PASSWORD GPG_KEY=stack@apache.org ./release-build.sh publish-dist @@ -56,7 +69,7 @@ set -e function cleanup { echo "Cleaning up temp settings file." >&2 - rm "${MAVEN_SETTINGS_FILE}" &> /dev/null || true + rm -f "${MAVEN_SETTINGS_FILE}" &> /dev/null || true # If REPO was set, then leave things be. Otherwise if we defined a repo clean it out. if [[ -z "${REPO}" ]] && [[ -n "${MAVEN_LOCAL_REPO}" ]]; then echo "Cleaning up temp repo in '${MAVEN_LOCAL_REPO}'. set REPO to reuse downloads." >&2 @@ -64,7 +77,7 @@ function cleanup { fi } -if [ $# -eq 0 ]; then +if [ $# -ne 1 ]; then exit_with_usage fi @@ -72,32 +85,78 @@ if [[ $@ == *"help"* ]]; then exit_with_usage fi -# Read in the ASF password. -if [[ -z "$ASF_PASSWORD" ]]; then - echo 'The environment variable ASF_PASSWORD is not set. Enter the password.' - echo - stty -echo && printf "ASF password: " && read ASF_PASSWORD && printf '\n' && stty echo -fi +export LC_ALL=C.UTF-8 +export LANG=C.UTF-8 +export GPG_TTY=$(tty) -# Read in the GPG passphrase -if [[ -z "$GPG_PASSPHRASE" ]]; then - echo 'The environment variable GPG_PASSPHRASE is not set. Enter the passphrase to' - echo 'unlock the GPG signing key that will be used to sign the release!' - echo - stty -echo && printf "GPG passphrase: " && read GPG_PASSPHRASE && printf '\n' && stty echo - export GPG_PASSPHRASE - export GPG_TTY=$(tty) -fi +init_java +init_mvn +init_python +# Print out subset of perl version (used in git hooks) +perl --version | grep 'This is' + +rm -rf "${PROJECT}" -for env in ASF_USERNAME GPG_PASSPHRASE GPG_KEY; do - if [ -z "${!env}" ]; then - echo "ERROR: $env must be set to run this script" - exit_with_usage +if [[ "$1" == "tag" ]]; then + # for 'tag' stage + set -o pipefail + check_get_passwords ASF_PASSWORD + check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION \ + GIT_EMAIL GIT_NAME GIT_BRANCH + ASF_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git" + encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')") + encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')") + git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b $GIT_BRANCH + + # 'update_releasenotes' searches the project's Jira for issues where 'Fix Version' matches specified + # $jira_fix_version. For most projects this is same as ${RELEASE_VERSION}. However, all the 'hbase-*' + # projects share the same HBASE jira name. To make this work, by convention, the HBASE jira "Fix Version" + # field values have the sub-project name pre-pended, as in "hbase-operator-tools-1.0.0". + # So, here we prepend the project name to the version, but only for the hbase sub-projects. + jira_fix_version="${RELEASE_VERSION}" + shopt -s nocasematch + if [[ "${PROJECT}" =~ ^hbase- ]]; then + jira_fix_version="${PROJECT}-${RELEASE_VERSION}" fi -done + shopt -u nocasematch + update_releasenotes `pwd`/${PROJECT} "${jira_fix_version}" -export LC_ALL=C.UTF-8 -export LANG=C.UTF-8 + cd ${PROJECT} + + git config user.name "$GIT_NAME" + git config user.email $GIT_EMAIL + + # Create release version + $MVN versions:set -DnewVersion=$RELEASE_VERSION | grep -v "no value" # silence logs + git add RELEASENOTES.md CHANGES.md + + git commit -a -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md" + echo "Creating tag $RELEASE_TAG at the head of $GIT_BRANCH" + git tag $RELEASE_TAG + + # Create next version + $MVN versions:set -DnewVersion=$NEXT_VERSION | grep -v "no value" # silence logs + + git commit -a -m "Preparing development version $NEXT_VERSION" + + if ! is_dry_run; then + # Push changes + git push origin $RELEASE_TAG + git push origin HEAD:$GIT_BRANCH + cd .. + rm -rf ${PROJECT} + else + cd .. + mv ${PROJECT} ${PROJECT}.tag + echo "Clone with version changes and tag available as ${PROJECT}.tag in the output directory." + fi + exit 0 +fi + +### Below is for 'publish-*' stages ### + +check_get_passwords ASF_PASSWORD GPG_PASSPHRASE +check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD GPG_KEY GPG_PASSPHRASE # Commit ref to checkout when building BASE_DIR=$(pwd) @@ -108,13 +167,6 @@ else RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/${PROJECT}" fi -init_java -init_mvn -init_python -# Print out subset of perl version. -perl --version | grep 'This is' - -rm -rf "${PROJECT}" # in case of dry run, enable publish steps to chain from tag step if is_dry_run && [[ "${TAG_SAME_DRY_RUN:-}" == "true" && -d "${PROJECT}.tag" ]]; then ln -s "${PROJECT}.tag" "${PROJECT}" @@ -274,4 +326,5 @@ fi cd .. rm -rf "${PROJECT}" -echo "ERROR: expects to be called with 'install', 'publish-release' or 'publish-snapshot'" +echo "ERROR: expects to be called with 'tag', 'publish-dist', 'publish-release', or 'publish-snapshot'" >&2 +exit_with_usage diff --git a/dev-support/create-release/release-tag.sh b/dev-support/create-release/release-tag.sh index 0aff44336ec9..5fe3da056cf6 100755 --- a/dev-support/create-release/release-tag.sh +++ b/dev-support/create-release/release-tag.sh @@ -25,7 +25,8 @@ function exit_with_usage { local NAME=$(basename $0) cat << EOF usage: $NAME -Tags an $PROJECT release on a particular branch. +Prepares for release on specified git branch: Set release version, create release tag, +increment version for ongoing dev, and publish to Apache git repo. Inputs are specified with the following environment variables: ASF_USERNAME - Apache Username @@ -54,7 +55,7 @@ if [[ -z "$ASF_PASSWORD" ]]; then fi for env in ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION GIT_EMAIL \ - GIT_NAME GIT_BRANCH GPG_KEY; do + GIT_NAME GIT_BRANCH; do if [ -z "${!env}" ]; then echo "$env must be set to run this script" exit 1 diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index fcb7523651a7..88ec35a89b18 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -248,6 +248,31 @@ function is_dry_run { [[ $DRY_RUN = 1 ]] } +function check_get_passwords { + for env in "$@"; do + if [ -z "${!env}" ]; then + echo "The environment variable $env is not set. Please enter the password or passphrase." + echo + stty -echo && printf "$env : " && read $env && printf '\n' && stty echo + export $env + fi + done +} + +function check_needed_vars { + local missing=0 + for env in "$@"; do + if [ -z "${!env}" ]; then + echo "$env must be set to run this script" + (( missing++ )) + else + export $env + fi + done + (( missing > 0 )) && exit_with_usage + return 0 +} + # Initializes JAVA_VERSION to the version of the JVM in use. function init_java { if [ -z "$JAVA_HOME" ]; then From 911ba47c3bbc19d331125741b96179ed3637a584 Mon Sep 17 00:00:00 2001 From: Matt Foley Date: Fri, 3 Apr 2020 16:22:36 -0700 Subject: [PATCH 07/14] Delete redundant release-tag.sh script. --- dev-support/create-release/release-tag.sh | 118 ---------------------- 1 file changed, 118 deletions(-) delete mode 100755 dev-support/create-release/release-tag.sh diff --git a/dev-support/create-release/release-tag.sh b/dev-support/create-release/release-tag.sh deleted file mode 100755 index 5fe3da056cf6..000000000000 --- a/dev-support/create-release/release-tag.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Tags release. Updates releasenotes and changes. -SELF=$(cd $(dirname $0) && pwd) -. "$SELF/release-util.sh" - -function exit_with_usage { - local NAME=$(basename $0) - cat << EOF -usage: $NAME -Prepares for release on specified git branch: Set release version, create release tag, -increment version for ongoing dev, and publish to Apache git repo. - -Inputs are specified with the following environment variables: -ASF_USERNAME - Apache Username -ASF_PASSWORD - Apache Password -GIT_NAME - Name to use with git -GIT_EMAIL - E-mail address to use with git -GIT_BRANCH - Git branch on which to make release -RELEASE_VERSION - Version used in pom files for release -RELEASE_TAG - Name of release tag -NEXT_VERSION - Development version after release -EOF - exit 1 -} - -set -e -set -o pipefail - -if [[ $@ == *"help"* ]]; then - exit_with_usage -fi - -if [[ -z "$ASF_PASSWORD" ]]; then - echo 'The environment variable ASF_PASSWORD is not set. Enter the password.' - echo - stty -echo && printf "ASF password: " && read ASF_PASSWORD && printf '\n' && stty echo -fi - -for env in ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION GIT_EMAIL \ - GIT_NAME GIT_BRANCH; do - if [ -z "${!env}" ]; then - echo "$env must be set to run this script" - exit 1 - fi -done - -init_java -init_mvn - -rm -rf ${PROJECT} - -ASF_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git" -# Ugly! -encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')") -encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')") -git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b $GIT_BRANCH - -# 'update_releasenotes' searches the project's Jira for issues where 'Fix Version' matches specified -# $jira_fix_version. For most projects this is same as ${RELEASE_VERSION}. However, all the 'hbase-*' -# projects share the same HBASE jira name. To make this work, by convention, the HBASE jira "Fix Version" -# field values have the sub-project name pre-pended, as in "hbase-operator-tools-1.0.0". -# So, here we prepend the project name to the version, but only for the hbase sub-projects. -jira_fix_version="${RELEASE_VERSION}" -shopt -s nocasematch -if [[ "${PROJECT}" =~ ^hbase- ]]; then - jira_fix_version="${PROJECT}-${RELEASE_VERSION}" -fi -shopt -u nocasematch -update_releasenotes `pwd`/${PROJECT} "${jira_fix_version}" - -cd ${PROJECT} - -git config user.name "$GIT_NAME" -git config user.email $GIT_EMAIL - -# Create release version -$MVN versions:set -DnewVersion=$RELEASE_VERSION | grep -v "no value" # silence logs -git add RELEASENOTES.md CHANGES.md - -git commit -a -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md" -echo "Creating tag $RELEASE_TAG at the head of $GIT_BRANCH" -git tag $RELEASE_TAG - -# Create next version -$MVN versions:set -DnewVersion=$NEXT_VERSION | grep -v "no value" # silence logs - -git commit -a -m "Preparing development version $NEXT_VERSION" - -if ! is_dry_run; then - # Push changes - git push origin $RELEASE_TAG - git push origin HEAD:$GIT_BRANCH - cd .. - rm -rf ${PROJECT} -else - cd .. - mv ${PROJECT} ${PROJECT}.tag - echo "Clone with version changes and tag available as ${PROJECT}.tag in the output directory." -fi - From 88ab8854f0aa44a60ba5c39c047de16891a9cf49 Mon Sep 17 00:00:00 2001 From: Matt Foley Date: Tue, 7 Apr 2020 18:22:22 -0700 Subject: [PATCH 08/14] Multiple small improvements: - Fix publish-snapshot so it does same checks as publish-release. - Factor out common maven usages, and move them to build-util.sh. - Change default polarity of DRY_RUN to default to true. Change -n ("no-publish") to -f ("force publish") to actually publish. - Fix problems in do-release.sh so it runs correctly outside of docker, including DRY_RUN being exported. - Have do-release.sh set REPO (shared maven local repository) if doing all three stages. - Cleaned up REPO directory creation. - General cleanup of comments and usage. --- .gitignore | 2 + .../create-release/do-release-docker.sh | 8 +- dev-support/create-release/do-release.sh | 11 +- dev-support/create-release/release-build.sh | 75 +++---------- dev-support/create-release/release-util.sh | 100 ++++++++++++++---- 5 files changed, 113 insertions(+), 83 deletions(-) diff --git a/.gitignore b/.gitignore index 0ae87d9d0854..5b33192f8c93 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ linklint/ .checkstyle **/.checkstyle .java-version +*.log +**/*.log diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh index f12539ade0a2..190dc3be0b33 100755 --- a/dev-support/create-release/do-release-docker.sh +++ b/dev-support/create-release/do-release-docker.sh @@ -64,13 +64,15 @@ This script runs the release scripts inside a docker image. Options: -d [path] required. working directory. output will be written to "output" in here. - -n dry run mode. Checks and local builds, but does not upload anything. + -f "force" -- actually publish this release. Unless you specify '-f', it will + default to dry run mode, which checks and does local builds, but does not upload anything. -t [tag] tag for the hbase-rm docker image to use for building (default: "latest"). -j [path] path to local JDK installation to use building. By default the script will use openjdk8 installed in the docker image. -p [project] project to build, such as 'hbase' or 'hbase-thirdparty'; defaults to $PROJECT env var -s [step] runs a single step of the process; valid steps are: tag|publish-dist|publish-release. If none specified, runs tag, then publish-dist, and then publish-release. + 'publish-snapshot' is also an allowed, less used, option. EOF } @@ -78,10 +80,10 @@ WORKDIR= IMGTAG=latest JAVA= RELEASE_STEP= -while getopts "d:hj:np:s:t:" opt; do +while getopts "d:fhj:p:s:t:" opt; do case $opt in d) WORKDIR="$OPTARG" ;; - n) DRY_RUN=1 ;; + f) DRY_RUN=0 ;; t) IMGTAG="$OPTARG" ;; j) JAVA="$OPTARG" ;; p) PROJECT="$OPTARG" ;; diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh index 10f57321da65..c75c3649bf55 100755 --- a/dev-support/create-release/do-release.sh +++ b/dev-support/create-release/do-release.sh @@ -26,10 +26,10 @@ export PROJECT="${PROJECT:-hbase}" SELF=$(cd $(dirname $0) && pwd) . "$SELF/release-util.sh" -while getopts "bn" opt; do +while getopts "b:f" opt; do case $opt in - b) GIT_BRANCH=$OPTARG ;; - n) DRY_RUN=1 ;; + b) export GIT_BRANCH=$OPTARG ;; + f) export DRY_RUN=0 ;; # "force", ie actually publish this release (otherwise defaults to dry run) ?) error "Invalid option: $OPTARG" ;; esac done @@ -66,6 +66,11 @@ function should_build { fi } +# If doing all stages, use shared maven local repo +if [[ -z "$RELEASE_STEP" ]]; then + export REPO="${REPO:-$(pwd)/$(mktemp -d hbase-repo-XXXXX)}" +fi + if should_build "tag" && [ $SKIP_TAG = 0 ]; then run_silent "Creating release tag $RELEASE_TAG..." "tag.log" \ "$SELF/release-build.sh" tag diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index 64d8309165e9..d13b3c761579 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -29,11 +29,13 @@ function exit_with_usage { Usage: release-build.sh Creates release deliverables from a tag or commit. Arguments: - tag Prepares for release on specified git branch: Set release version, create release tag, - increment version for ongoing dev, and publish to Apache git repo + tag Prepares for release on specified git branch: Set release version, + update CHANGES and RELEASENOTES, create release tag, + increment version for ongoing dev, and publish to Apache git repo. publish-dist Build and publish distribution packages (tarballs) to Apache dist repo publish-snapshot Build and publish maven artifacts snapshot release to Apache snapshots repo - publish-release Build and publish maven artifacts release to Apache release repo + publish-release Build and publish maven artifacts release to Apache release repo, and + construct vote email from template All other inputs are environment variables: Used for 'tag' and 'publish': @@ -52,7 +54,8 @@ Used only for 'tag': Used only for 'publish': GIT_REF - Release tag or commit to build from VERSION - Version of project to be built (e.g. 2.1.2). - Optional for 'publish', as it defaults to the version in pom at GIT_REF. + Optional for 'publish', as it defaults to the version in pom at GIT_REF, + which typically will have been set by 'tag'. PACKAGE_VERSION - Release identifier in top level dist directory (e.g. 2.1.2RC1) GPG_KEY - GPG key id (usually email addr) used to sign release artifacts GPG_PASSPHRASE - Passphrase for GPG key @@ -127,7 +130,7 @@ if [[ "$1" == "tag" ]]; then git config user.email $GIT_EMAIL # Create release version - $MVN versions:set -DnewVersion=$RELEASE_VERSION | grep -v "no value" # silence logs + maven_set_version $RELEASE_VERSION git add RELEASENOTES.md CHANGES.md git commit -a -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md" @@ -135,7 +138,7 @@ if [[ "$1" == "tag" ]]; then git tag $RELEASE_TAG # Create next version - $MVN versions:set -DnewVersion=$NEXT_VERSION | grep -v "no value" # silence logs + maven_set_version $NEXT_VERSION git commit -a -m "Preparing development version $NEXT_VERSION" @@ -202,33 +205,11 @@ fi git clean -d -f -x cd .. -MAVEN_LOCAL_REPO="${REPO:-$(pwd)/$(mktemp -d hbase-repo-XXXXX)}" -MAVEN_SETTINGS_FILE="/${MAVEN_LOCAL_REPO}/tmp-settings.xml" -export MAVEN_SETTINGS_FILE MAVEN_LOCAL_REPO ASF_USERNAME ASF_PASSWORD -# reference passwords from env rather than storing in the settings.xml file. -cat <<'EOF' > "$MAVEN_SETTINGS_FILE" - - - /${env.MAVEN_LOCAL_REPO} - - apache.snapshots.https${env.ASF_USERNAME} - ${env.ASF_PASSWORD} - apache.releases.https${env.ASF_USERNAME} - ${env.ASF_PASSWORD} - - -EOF - if [[ "$1" == "publish-dist" ]]; then # Source and binary tarballs echo "Packaging release source tarballs" make_src_release "${PROJECT}" "${VERSION}" - # Add timestamps to mvn logs. - MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss ${MAVEN_OPTS}" - echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Building binary dist" make_binary_release "${PROJECT}" "${VERSION}" echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Done building binary distribution" @@ -273,53 +254,29 @@ if [[ "$1" == "publish-dist" ]]; then fi if [[ "$1" == "publish-snapshot" ]]; then - cd "${PROJECT}" - # Publish ${PROJECT} to Maven snapshot repo - echo "Deploying ${PROJECT} SNAPSHOT at '$GIT_REF' ($git_hash)" - echo "Publish version is $VERSION" - if [[ ! $VERSION == *"SNAPSHOT"* ]]; then - echo "ERROR: Snapshots must have a version containing SNAPSHOT" - echo "ERROR: You gave version '$VERSION'" - exit 1 - fi - # Coerce the requested version - $MVN versions:set -DnewVersion=$VERSION - $MVN --settings $MAVEN_SETTINGS_FILE -DskipTests -P "${PUBLISH_PROFILES}" deploy - cd .. + pushd "${PROJECT}" + maven_deploy snapshot "${BASE_DIR}/mvn_deploy.log" + popd exit 0 fi if [[ "$1" == "publish-release" ]]; then ( cd "${PROJECT}" - # Publish ${PROJECT} to Maven release repo - echo "Publishing ${PROJECT} checkout at '$GIT_REF' ($git_hash)" - echo "Publish version is $VERSION" - # Coerce the requested version - $MVN versions:set -DnewVersion=$VERSION - declare -a mvn_goals=(clean install) - declare staged_repo_id="dryrun-no-repo" - if ! is_dry_run; then - mvn_goals=("${mvn_goals[@]}" deploy) - fi echo "Staging release in nexus" - if ! MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings "$MAVEN_SETTINGS_FILE" \ - -DskipTests -Dcheckstyle.skip=true -P "${PUBLISH_PROFILES}" \ - "${mvn_goals[@]}" > "${BASE_DIR}/mvn_deploy.log"; then - echo "Staging build failed, see 'mvn_deploy.log' for details." >&2 - exit 1 - fi + maven_deploy release "${BASE_DIR}/mvn_deploy.log" + declare staged_repo_id="dryrun-no-repo" if ! is_dry_run; then staged_repo_id=$(grep -o "Closing staging repository with ID .*" "${BASE_DIR}/mvn_deploy.log" \ | sed -e 's/Closing staging repository with ID "\([^"]*\)"./\1/') echo "Artifacts successfully staged to repo ${staged_repo_id}" else - echo "Artifacts successfully built. not staged due to dry run." + echo "Artifacts successfully built. Not staged due to dry run." fi # Dump out email to send. Where we find vote.tmpl depends # on where this script is run from export PROJECT_TEXT=$(echo "${PROJECT}" | sed "s/-/ /g") - eval "echo \"$(< ${SELF}/vote.tmpl)\"" |tee "${BASE_DIR}/vote.txt" + eval "echo \"$(< "${SELF}/vote.tmpl")\"" |tee "${BASE_DIR}/vote.txt" ) exit 0 fi diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index 88ec35a89b18..fd3a63ccc6bd 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -16,17 +16,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # -DRY_RUN=${DRY_RUN:-0} +DRY_RUN=${DRY_RUN:-1} #default to dry run GPG="gpg --pinentry-mode loopback --no-tty --batch" YETUS_VERSION=${YETUS_VERSION:-0.11.1} -set -e +# Maven Profiles for publishing snapshots and release to Maven Central and Dist +PUBLISH_PROFILES=("-P" "apache-release,release") -# Profiles for publishing snapshots and release to Maven Central and Dist -# Use with '-P' maven option -PUBLISH_PROFILES="apache-release,release" +set -e function error { - echo "$*" + echo "Error: $*" >&2 exit 1 } @@ -38,7 +37,7 @@ function read_config { read -p "$PROMPT [$DEFAULT]: " REPLY local RETVAL="${REPLY:-$DEFAULT}" if [ -z "$RETVAL" ]; then - error "$PROMPT is must be provided." + error "$PROMPT must be provided." fi echo "$RETVAL" } @@ -54,7 +53,7 @@ function run_silent { shift 2 echo "========================" - echo "= $BANNER" + echo "=== $BANNER" echo "Command: $*" echo "Log file: $LOG_FILE" @@ -66,6 +65,7 @@ function run_silent { tail "$LOG_FILE" exit $EC fi + echo "=== SUCCESS" } function fcreate_secure { @@ -179,7 +179,8 @@ function get_release_info { if check_for_tag "$RELEASE_TAG"; then read -p "$RELEASE_TAG already exists. Continue anyway [y/n]? " ANSWER if [ "$ANSWER" != "y" ]; then - error "Exiting." + echo "Exiting." + exit 1 fi SKIP_TAG=1 fi @@ -285,8 +286,7 @@ function init_java { function init_python { if ! [ -x "$(command -v python2)" ]; then - echo 'Error: python2 needed by yetus. Install or add link? E.g: sudo ln -sf /usr/bin/python2.7 /usr/local/bin/python2' >&2 - exit 1 + error 'python2 needed by yetus. Install or add link? E.g: sudo ln -sf /usr/bin/python2.7 /usr/local/bin/python2' fi echo "python version: `python2 --version`" } @@ -304,6 +304,32 @@ function init_mvn { # Add timestamped logging. MVN="${MVN} -B" export MVN + configure_maven +} + +function configure_maven { + # Add timestamps to mvn logs. + MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss ${MAVEN_OPTS}" + MAVEN_LOCAL_REPO="${REPO:-$(pwd)/$(mktemp -d hbase-repo-XXXXX)}" + [[ -d "$MAVEN_LOCAL_REPO" ]] || mkdir -p "$MAVEN_LOCAL_REPO" + MAVEN_SETTINGS_FILE="${MAVEN_LOCAL_REPO}/tmp-settings.xml" + export MAVEN_OPTS MAVEN_SETTINGS_FILE MAVEN_LOCAL_REPO + export ASF_USERNAME ASF_PASSWORD + # reference passwords from env rather than storing in the settings.xml file. + cat <<'EOF' > "$MAVEN_SETTINGS_FILE" + + + /${env.MAVEN_LOCAL_REPO} + + apache.snapshots.https${env.ASF_USERNAME} + ${env.ASF_PASSWORD} + apache.releases.https${env.ASF_USERNAME} + ${env.ASF_PASSWORD} + + +EOF } # Writes report into cwd! @@ -332,8 +358,7 @@ function get_jira_name { *) jira_name="";; esac if [[ -z "$jira_name" ]]; then - echo "Sorry, can't determine the Jira name for project $project" >&2 - exit 1 + error "Sorry, can't determine the Jira name for project $project" fi echo "$jira_name" } @@ -404,7 +429,7 @@ make_src_release() { # - GPG needs to be defined, with the path to GPG: defaults 'gpg'. # - The passphrase in the GPG_PASSPHRASE variable: no default (we don't make .asc file). # - GIT_REF which is the tag to create the tgz from: defaults to 'master'. -# - MVN Default is 'mvn'. +# - MVN Default is 'mvn -B'. # For example: # $ GPG_PASSPHRASE="XYZ" GIT_REF="master" make_src_release hbase-operator-tools 1.0.0 make_binary_release() { @@ -420,10 +445,10 @@ make_binary_release() { # a third to assemble the binary artifact. Trying to do # all in the one invocation fails; a problem in our # assembly spec to in maven. TODO. Meantime, three invocations. - MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $MAVEN_SETTINGS_FILE clean install -DskipTests - MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $MAVEN_SETTINGS_FILE site -DskipTests - MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $MAVEN_SETTINGS_FILE install assembly:single -DskipTests \ - -Dcheckstyle.skip=true -P "${PUBLISH_PROFILES}" + ${MVN} --settings $MAVEN_SETTINGS_FILE clean install -DskipTests + ${MVN} --settings $MAVEN_SETTINGS_FILE site -DskipTests + ${MVN} --settings $MAVEN_SETTINGS_FILE install assembly:single -DskipTests \ + -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" # Check there is a bin gz output. The build may not produce one: e.g. hbase-thirdparty. f_bin_tgz="./${PROJECT}-assembly/target/${basename}*-bin.tar.gz" @@ -439,3 +464,42 @@ make_binary_release() { echo "No ${f_bin_tgz} product; expected?" fi } + +# Do maven command to set version into local pom +function maven_set_version { #input: + local this_version="$1" + $MVN versions:set -DnewVersion="$this_version" | grep -v "no value" # silence logs +} + +# Do maven deploy to snapshot or release artifact repository, with checks. +function maven_deploy { #inputs: + # Invoke with cwd=$PROJECT + local deploy_type="$1" + local log_file="$2" + if [[ "$deploy_type" != "snapshot" && "$deploy_type" != "release" ]]; then + error "unrecognized deploy type, must be 'snapshot'|'release'" + fi + if [[ -z "$log_file" ]] || ! touch "$log_file"; then + error "must provide writable maven log output filepath" + fi + if [[ $deploy_type == "snapshot" ]] && ! [[ $VERSION =~ -SNAPSHOT$ ]]; then + error "Snapshots must have a version with suffix '-SNAPSHOT'; you gave version '$VERSION'" + elif [[ $deploy_type == "release" ]] && [[ $VERSION =~ SNAPSHOT ]]; then + error "Non-snapshot release version must not include the word 'SNAPSHOT'; you gave version '$VERSION'" + fi + # Publish ${PROJECT} to Maven release repo + echo "Publishing ${PROJECT} checkout at '$GIT_REF' ($git_hash)" + echo "Publish version is $VERSION" + # Coerce the requested version + maven_set_version "$VERSION" + declare -a mvn_goals=(clean install) + if ! is_dry_run; then + mvn_goals=("${mvn_goals[@]}" deploy) + fi + if ! ${MVN} --settings "$MAVEN_SETTINGS_FILE" \ + -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \ + "${mvn_goals[@]}" > "$log_file"; then + error "Deploy build failed, for details see log at '$log_file'." + fi + return 0 +} From 2e4c4e8073d6fa0f5502ad18339a5e92b019ed0d Mon Sep 17 00:00:00 2001 From: Matt Foley Date: Thu, 9 Apr 2020 13:07:58 -0700 Subject: [PATCH 09/14] - fix all 'shellcheck' errors - use ${BASH_SOURCE[0]} instead of $0 to determine script directory path - smarter way to read version from pom with mvn - minor bug fixes --- .../create-release/do-release-docker.sh | 21 +- dev-support/create-release/do-release.sh | 21 +- dev-support/create-release/release-build.sh | 67 ++++--- dev-support/create-release/release-util.sh | 183 ++++++++++-------- 4 files changed, 163 insertions(+), 129 deletions(-) diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh index 190dc3be0b33..10bb1298567f 100755 --- a/dev-support/create-release/do-release-docker.sh +++ b/dev-support/create-release/do-release-docker.sh @@ -50,12 +50,13 @@ set -e # Set this to build other hbase repos: e.g. PROJECT=hbase-operator-tools export PROJECT="${PROJECT:-hbase}" -SELF=$(cd $(dirname "$0") && pwd) +SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=SCRIPTDIR/release-util.sh . "$SELF/release-util.sh" function usage { local NAME - NAME="$(basename "$0")" + NAME="$(basename "${BASH_SOURCE[0]}")" cat < 0 )); then + error "Arguments can only be provided with option flags, invalid args: $*" +fi if [ -z "$WORKDIR" ] || [ ! -d "$WORKDIR" ]; then error "Work directory (-d) must be defined and exist. Run with -h for help." @@ -158,15 +163,17 @@ RELEASE_STEP=$RELEASE_STEP API_DIFF_TAG=$API_DIFF_TAG EOF -JAVA_VOL= +JAVA_VOL=() if [ -n "$JAVA" ]; then echo "JAVA_HOME=/opt/hbase-java" >> "$ENVFILE" - JAVA_VOL="--volume $JAVA:/opt/hbase-java" + JAVA_VOL=(--volume "$JAVA:/opt/hbase-java") fi echo "Building $RELEASE_TAG; output will be at $WORKDIR/output" -docker run -ti \ +cmd=(docker run -ti \ --env-file "$ENVFILE" \ --volume "$WORKDIR:/opt/hbase-rm" \ - $JAVA_VOL \ - "hbase-rm:$IMGTAG" + "${JAVA_VOL[@]}" \ + "hbase-rm:$IMGTAG") +echo "${cmd[*]}" +"${cmd[@]}" diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh index c75c3649bf55..d859045887bd 100755 --- a/dev-support/create-release/do-release.sh +++ b/dev-support/create-release/do-release.sh @@ -23,7 +23,8 @@ # and passwords to use building. export PROJECT="${PROJECT:-hbase}" -SELF=$(cd $(dirname $0) && pwd) +SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=SCRIPTDIR/release-util.sh . "$SELF/release-util.sh" while getopts "b:f" opt; do @@ -33,14 +34,19 @@ while getopts "b:f" opt; do ?) error "Invalid option: $OPTARG" ;; esac done +shift $((OPTIND-1)) +if (( $# > 0 )); then + error "Arguments can only be provided with option flags, invalid args: $*" +fi # If running in docker, import and then cache keys. if [ "$RUNNING_IN_DOCKER" = "1" ]; then # Run gpg agent. - eval $(gpg-agent --disable-scdaemon --daemon --no-grab --allow-preset-passphrase --default-cache-ttl=86400 --max-cache-ttl=86400) - echo "GPG Version: `gpg --version`" + eval "$(gpg-agent --disable-scdaemon --daemon --no-grab --allow-preset-passphrase \ + --default-cache-ttl=86400 --max-cache-ttl=86400)" + echo "GPG Version: $(gpg --version)" # Inside docker, need to import the GPG key stored in the current directory. - echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --import "$SELF/gpg.key" + echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --import "$SELF/gpg.key" # We may need to adjust the path since JAVA_HOME may be overridden by the driver script. if [ -n "$JAVA_HOME" ]; then @@ -53,7 +59,8 @@ else # Outside docker, need to ask for information about the release. get_release_info fi -export GPG_TTY=$(tty) +GPG_TTY="$(tty)" +export GPG_TTY function should_build { local WHAT=$1 @@ -71,7 +78,7 @@ if [[ -z "$RELEASE_STEP" ]]; then export REPO="${REPO:-$(pwd)/$(mktemp -d hbase-repo-XXXXX)}" fi -if should_build "tag" && [ $SKIP_TAG = 0 ]; then +if should_build "tag" && [ "$SKIP_TAG" = 0 ]; then run_silent "Creating release tag $RELEASE_TAG..." "tag.log" \ "$SELF/release-build.sh" tag if is_dry_run; then @@ -79,7 +86,7 @@ if should_build "tag" && [ $SKIP_TAG = 0 ]; then else echo "It may take some time for the tag to be synchronized to github." echo "Press enter when you've verified that the new tag ($RELEASE_TAG) is available." - read + read -r fi else echo "Skipping tag creation for $RELEASE_TAG." diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index d13b3c761579..982f24a31d49 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -20,7 +20,8 @@ trap cleanup EXIT # Source in utils. -SELF=$(cd $(dirname $0) && pwd) +SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=SCRIPTDIR/release-util.sh . "$SELF/release-util.sh" # Print usage and exit. @@ -84,13 +85,14 @@ if [ $# -ne 1 ]; then exit_with_usage fi -if [[ $@ == *"help"* ]]; then +if [[ "$*" == *"help"* ]]; then exit_with_usage fi export LC_ALL=C.UTF-8 export LANG=C.UTF-8 -export GPG_TTY=$(tty) +GPG_TTY="$(tty)" +export GPG_TTY init_java init_mvn @@ -107,9 +109,9 @@ if [[ "$1" == "tag" ]]; then check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION \ GIT_EMAIL GIT_NAME GIT_BRANCH ASF_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git" - encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')") - encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')") - git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b $GIT_BRANCH + encoded_username="$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')")" + encoded_password="$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')")" + git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b "$GIT_BRANCH" "${PROJECT}" # 'update_releasenotes' searches the project's Jira for issues where 'Fix Version' matches specified # $jira_fix_version. For most projects this is same as ${RELEASE_VERSION}. However, all the 'hbase-*' @@ -122,35 +124,35 @@ if [[ "$1" == "tag" ]]; then jira_fix_version="${PROJECT}-${RELEASE_VERSION}" fi shopt -u nocasematch - update_releasenotes `pwd`/${PROJECT} "${jira_fix_version}" + update_releasenotes "$(pwd)/${PROJECT}" "${jira_fix_version}" - cd ${PROJECT} + cd "${PROJECT}" git config user.name "$GIT_NAME" - git config user.email $GIT_EMAIL + git config user.email "$GIT_EMAIL" # Create release version - maven_set_version $RELEASE_VERSION + maven_set_version "$RELEASE_VERSION" git add RELEASENOTES.md CHANGES.md git commit -a -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md" echo "Creating tag $RELEASE_TAG at the head of $GIT_BRANCH" - git tag $RELEASE_TAG + git tag "$RELEASE_TAG" # Create next version - maven_set_version $NEXT_VERSION + maven_set_version "$NEXT_VERSION" git commit -a -m "Preparing development version $NEXT_VERSION" if ! is_dry_run; then # Push changes - git push origin $RELEASE_TAG - git push origin HEAD:$GIT_BRANCH + git push origin "$RELEASE_TAG" + git push origin "HEAD:$GIT_BRANCH" cd .. - rm -rf ${PROJECT} + rm -rf "${PROJECT}" else cd .. - mv ${PROJECT} ${PROJECT}.tag + mv "${PROJECT}" "${PROJECT}.tag" echo "Clone with version changes and tag available as ${PROJECT}.tag in the output directory." fi exit 0 @@ -177,17 +179,13 @@ else ASF_REPO="${ASF_REPO:-https://gitbox.apache.org/repos/asf/${PROJECT}.git}" git clone "$ASF_REPO" "${PROJECT}" fi -cd ${PROJECT} -git checkout $GIT_REF -git_hash=`git rev-parse --short HEAD` -echo "Checked out ${PROJECT} git hash $git_hash" +cd "${PROJECT}" +git checkout "$GIT_REF" +git_hash="$(git rev-parse --short HEAD)" +echo "Checked out ${PROJECT} at ${GIT_REF} commit $git_hash" if [ -z "$VERSION" ]; then - # Run $MVN in a separate command so that 'set -e' does the right thing. - TMP=$(mktemp) - $MVN help:evaluate -Dexpression=project.version > $TMP - VERSION=$(cat $TMP | grep -v INFO | grep -v WARNING | grep -v Download) - rm $TMP + VERSION="$(maven_get_version)" fi # This is a band-aid fix to avoid the failure of Maven nightly snapshot in some Jenkins @@ -210,9 +208,9 @@ if [[ "$1" == "publish-dist" ]]; then echo "Packaging release source tarballs" make_src_release "${PROJECT}" "${VERSION}" - echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Building binary dist" + echo "$(date -u +'%Y-%m-%dT%H:%M:%SZ') Building binary dist" make_binary_release "${PROJECT}" "${VERSION}" - echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Done building binary distribution" + echo "$(date -u +'%Y-%m-%dT%H:%M:%SZ') Done building binary distribution" if [[ "$PROJECT" =~ ^hbase- ]]; then DEST_DIR_NAME="${PROJECT}-${PACKAGE_VERSION}" @@ -221,18 +219,18 @@ if [[ "$1" == "publish-dist" ]]; then fi svn_target="svn-${PROJECT}" svn co --depth=empty "$RELEASE_STAGING_LOCATION" "$svn_target" - rm -rf "$svn_target/${DEST_DIR_NAME}" + rm -rf "${svn_target:?}/${DEST_DIR_NAME}" mkdir -p "$svn_target/${DEST_DIR_NAME}" echo "Copying release tarballs" - cp ${PROJECT}-*.tar.* "$svn_target/${DEST_DIR_NAME}/" - cp ${PROJECT}/CHANGES.md "$svn_target/${DEST_DIR_NAME}/" - cp ${PROJECT}/RELEASENOTES.md "$svn_target/${DEST_DIR_NAME}/" + cp "${PROJECT}"-*.tar.* "$svn_target/${DEST_DIR_NAME}/" + cp "${PROJECT}/CHANGES.md" "$svn_target/${DEST_DIR_NAME}/" + cp "${PROJECT}/RELEASENOTES.md" "$svn_target/${DEST_DIR_NAME}/" shopt -s nocasematch # Generate api report only if project is hbase for now. if [ "${PROJECT}" == "hbase" ]; then # This script usually reports an errcode along w/ the report. - generate_api_report ./${PROJECT} "${API_DIFF_TAG}" "${PACKAGE_VERSION}" || true + generate_api_report "./${PROJECT}" "${API_DIFF_TAG}" "${PACKAGE_VERSION}" || true cp api*.html "$svn_target/${DEST_DIR_NAME}/" fi shopt -u nocasematch @@ -241,7 +239,7 @@ if [[ "$1" == "publish-dist" ]]; then if ! is_dry_run; then cd "$svn_target" - svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Apache ${PROJECT} $PACKAGE_VERSION" --no-auth-cache + svn ci --username "$ASF_USERNAME" --password "$ASF_PASSWORD" -m"Apache ${PROJECT} $PACKAGE_VERSION" --no-auth-cache cd .. rm -rf "$svn_target" else @@ -275,7 +273,8 @@ if [[ "$1" == "publish-release" ]]; then fi # Dump out email to send. Where we find vote.tmpl depends # on where this script is run from - export PROJECT_TEXT=$(echo "${PROJECT}" | sed "s/-/ /g") + PROJECT_TEXT="${PROJECT//-/ }" #substitute like 's/-/ /g' + export PROJECT_TEXT eval "echo \"$(< "${SELF}/vote.tmpl")\"" |tee "${BASE_DIR}/vote.txt" ) exit 0 diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index fd3a63ccc6bd..3ae1904f58c0 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -34,7 +34,7 @@ function read_config { local DEFAULT="$2" local REPLY= - read -p "$PROMPT [$DEFAULT]: " REPLY + read -r -p "$PROMPT [$DEFAULT]: " REPLY local RETVAL="${REPLY:-$DEFAULT}" if [ -z "$RETVAL" ]; then error "$PROMPT must be provided." @@ -81,29 +81,33 @@ function check_for_tag { # API compare version. function get_api_diff_version { - local version=$1 - local rev=$(echo "$version" | cut -d . -f 3) + local version="$1" + local rev local api_diff_tag - if [ $rev != 0 ]; then - local short_version=$(echo "$version" | cut -d . -f 1-2) + rev=$(echo "$version" | cut -d . -f 3) + if [ "$rev" != 0 ]; then + local short_version + short_version="$(echo "$version" | cut -d . -f 1-2)" api_diff_tag="rel/${short_version}.$((rev - 1))" else - local major=$(echo "$version" | cut -d . -f 1) - local minor=$(echo "$version" | cut -d . -f 2) - if [ $minor != 0 ]; then + local major minor + major="$(echo "$version" | cut -d . -f 1)" + minor="$(echo "$version" | cut -d . -f 2)" + if [ "$minor" != 0 ]; then api_diff_tag="rel/${major}.$((minor - 1)).0" else api_diff_tag="rel/$((major - 1)).0.0" fi fi - api_diff_tag=$(read_config "api_diff_tag" "$api_diff_tag") - echo $api_diff_tag + api_diff_tag="$(read_config "api_diff_tag" "$api_diff_tag")" + echo "$api_diff_tag" } # Get all branches that begin with 'branch-', the hbase convention for # release branches, sort them and then pop off the most recent. function get_release_info { - export PROJECT=$(read_config "PROJECT" "$PROJECT") + PROJECT="$(read_config "PROJECT" "$PROJECT")" + export PROJECT if [[ -z "${ASF_REPO}" ]]; then ASF_REPO="https://gitbox.apache.org/repos/asf/${PROJECT}.git" @@ -116,31 +120,34 @@ function get_release_info { fi if [ -z "$GIT_BRANCH" ]; then # If no branch is specified, find out the latest branch from the repo. - GIT_BRANCH=$(git ls-remote --heads "$ASF_REPO" | + GIT_BRANCH="$(git ls-remote --heads "$ASF_REPO" | grep refs/heads/branch- | awk '{print $2}' | sort -r | head -n 1 | - cut -d/ -f3) + cut -d/ -f3)" fi - export GIT_BRANCH=$(read_config "GIT_BRANCH" "$GIT_BRANCH") + GIT_BRANCH="$(read_config "GIT_BRANCH" "$GIT_BRANCH")" + export GIT_BRANCH # Find the current version for the branch. - local VERSION=$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" | - parse_version) - echo "Current branch VERSION is $VERSION." + local version + version="$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" | + parse_version)" + echo "Current branch VERSION is $version." - NEXT_VERSION="$VERSION" + NEXT_VERSION="$version" RELEASE_VERSION="" - SHORT_VERSION=$(echo "$VERSION" | cut -d . -f 1-2) - if [[ ! $VERSION =~ .*-SNAPSHOT ]]; then - RELEASE_VERSION="$VERSION" + SHORT_VERSION="$(echo "$version" | cut -d . -f 1-2)" + if [[ ! "$version" =~ .*-SNAPSHOT ]]; then + RELEASE_VERSION="$version" else - RELEASE_VERSION="${VERSION/-SNAPSHOT/}" + RELEASE_VERSION="${version/-SNAPSHOT/}" fi - local REV=$(echo "${RELEASE_VERSION}" | cut -d . -f 3) + local REV + REV="$(echo "${RELEASE_VERSION}" | cut -d . -f 3)" # Find out what RC is being prepared. # - If the current version is "x.y.0", then this is RC0 of the "x.y.0" release. @@ -148,7 +155,7 @@ function get_release_info { # - If it has, then we're building RC0 of the current version. # - If it has not, we're building the next RC of the previous version. local RC_COUNT - if [ $REV != 0 ]; then + if [ "$REV" != 0 ]; then local PREV_REL_REV=$((REV - 1)) PREV_REL_TAG="rel/${SHORT_VERSION}.${PREV_REL_REV}" if check_for_tag "$PREV_REL_TAG"; then @@ -157,7 +164,7 @@ function get_release_info { NEXT_VERSION="${SHORT_VERSION}.${REV}-SNAPSHOT" else RELEASE_VERSION="${SHORT_VERSION}.${PREV_REL_REV}" - RC_COUNT=$(git ls-remote --tags "$ASF_REPO" "${RELEASE_VERSION}RC*" | wc -l) + RC_COUNT="$(git ls-remote --tags "$ASF_REPO" "${RELEASE_VERSION}RC*" | wc -l)" # This makes a 'number' of it. RC_COUNT=$((RC_COUNT)) fi @@ -167,17 +174,17 @@ function get_release_info { RC_COUNT=0 fi - export RELEASE_VERSION=$(read_config "RELEASE_VERSION" "$RELEASE_VERSION") - export NEXT_VERSION=$(read_config "NEXT_VERSION" "$NEXT_VERSION") + RELEASE_VERSION="$(read_config "RELEASE_VERSION" "$RELEASE_VERSION")" + NEXT_VERSION="$(read_config "NEXT_VERSION" "$NEXT_VERSION")" + export RELEASE_VERSION NEXT_VERSION - - RC_COUNT=$(read_config "RC_COUNT" "$RC_COUNT") + RC_COUNT="$(read_config "RC_COUNT" "$RC_COUNT")" # Check if the RC already exists, and if re-creating the RC, skip tag creation. RELEASE_TAG="${RELEASE_VERSION}RC${RC_COUNT}" SKIP_TAG=0 if check_for_tag "$RELEASE_TAG"; then - read -p "$RELEASE_TAG already exists. Continue anyway [y/n]? " ANSWER + read -r -p "$RELEASE_TAG already exists. Continue anyway [y/n]? " ANSWER if [ "$ANSWER" != "y" ]; then echo "Exiting." exit 1 @@ -185,26 +192,27 @@ function get_release_info { SKIP_TAG=1 fi - export RELEASE_TAG + export RELEASE_TAG SKIP_TAG GIT_REF="$RELEASE_TAG" if is_dry_run; then echo "This is a dry run. Please confirm the ref that will be built for testing." - GIT_REF=$(read_config "GIT_REF" "$GIT_REF") + GIT_REF="$(read_config "GIT_REF" "$GIT_REF")" fi export GIT_REF export PACKAGE_VERSION="$RELEASE_TAG" - export API_DIFF_TAG=$(get_api_diff_version $RELEASE_VERSION) + API_DIFF_TAG="$(get_api_diff_version "$RELEASE_VERSION")" # Gather some user information. - export ASF_USERNAME=$(read_config "ASF_USERNAME" "$LOGNAME") + ASF_USERNAME="$(read_config "ASF_USERNAME" "$LOGNAME")" - GIT_NAME=$(git config user.name || echo "") - export GIT_NAME=$(read_config "GIT_NAME" "$GIT_NAME") + GIT_NAME="$(git config user.name || echo "")" + GIT_NAME="$(read_config "GIT_NAME" "$GIT_NAME")" - export GIT_EMAIL="$ASF_USERNAME@apache.org" - export GPG_KEY=$(read_config "GPG_KEY" "$GIT_EMAIL") + GIT_EMAIL="$ASF_USERNAME@apache.org" + GPG_KEY="$(read_config "GPG_KEY" "$GIT_EMAIL")" + export API_DIFF_TAG ASF_USERNAME GIT_NAME GIT_EMAIL GPG_KEY cat < 0 )) && exit_with_usage @@ -288,21 +300,21 @@ function init_python { if ! [ -x "$(command -v python2)" ]; then error 'python2 needed by yetus. Install or add link? E.g: sudo ln -sf /usr/bin/python2.7 /usr/local/bin/python2' fi - echo "python version: `python2 --version`" + echo "python version: $(python2 --version)" } # Set MVN function init_mvn { if [ -n "$MAVEN_HOME" ]; then - MVN=${MAVEN_HOME}/bin/mvn - elif [ $(type -P mvn) ]; then - MVN=mvn + MVN=("${MAVEN_HOME}/bin/mvn") + elif [ "$(type -P mvn)" ]; then + MVN=(mvn) else error "MAVEN_HOME is not set nor is mvn on the current path." fi - echo "mvn version: `$MVN --version`" - # Add timestamped logging. - MVN="${MVN} -B" + echo "mvn version: $("${MVN[@]}" --version)" + # Add batch mode. + MVN=("${MVN[@]}" -B) export MVN configure_maven } @@ -313,7 +325,8 @@ function configure_maven { MAVEN_LOCAL_REPO="${REPO:-$(pwd)/$(mktemp -d hbase-repo-XXXXX)}" [[ -d "$MAVEN_LOCAL_REPO" ]] || mkdir -p "$MAVEN_LOCAL_REPO" MAVEN_SETTINGS_FILE="${MAVEN_LOCAL_REPO}/tmp-settings.xml" - export MAVEN_OPTS MAVEN_SETTINGS_FILE MAVEN_LOCAL_REPO + MVN=("${MVN[@]}" --settings "${MAVEN_SETTINGS_FILE}") + export MVN MAVEN_OPTS MAVEN_SETTINGS_FILE MAVEN_LOCAL_REPO export ASF_USERNAME ASF_PASSWORD # reference passwords from env rather than storing in the settings.xml file. cat <<'EOF' > "$MAVEN_SETTINGS_FILE" @@ -337,12 +350,13 @@ function generate_api_report { local project="$1" local previous_tag="$2" local release_tag="$3" + local previous_version # Generate api report. - ${project}/dev-support/checkcompatibility.py --annotation \ + "${project}"/dev-support/checkcompatibility.py --annotation \ org.apache.yetus.audience.InterfaceAudience.Public \ - $previous_tag $release_tag - local previous_version=$(echo ${previous_tag} | sed -e 's/rel\///') - cp ${project}/target/compat-check/report.html "./api_compare_${previous_version}_to_${release_tag}.html" + "$previous_tag" "$release_tag" + previous_version="$(echo "${previous_tag}" | sed -e 's/rel\///')" + cp "${project}/target/compat-check/report.html" "./api_compare_${previous_version}_to_${release_tag}.html" } # Look up the Jira name associated with project. @@ -370,7 +384,8 @@ function update_releasenotes { local project_dir="$1" local jira_fix_version="$2" local yetus="apache-yetus-${YETUS_VERSION}" - local jira_project="$(get_jira_name "$(basename "$project_dir")")" + local jira_project + jira_project="$(get_jira_name "$(basename "$project_dir")")" wget -qO- "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/yetus/${YETUS_VERSION}/${yetus}-bin.tar.gz" | \ tar xvz -C . || exit cd "./${yetus}" || exit @@ -409,12 +424,12 @@ make_src_release() { # Tar up the src and sign and hash it. project="${1}" version="${2}" - basename="${project}-${version}" - rm -rf "${basename}-src*" - tgz="${basename}-src.tar.gz" + base_name="${project}-${version}" + rm -rf "${base_name}"-src* + tgz="${base_name}-src.tar.gz" cd "${project}" || exit git clean -d -f -x - git archive --format=tar.gz --output="../${tgz}" --prefix="${basename}/" "${GIT_REF:-master}" + git archive --format=tar.gz --output="../${tgz}" --prefix="${base_name}/" "${GIT_REF:-master}" cd .. || exit echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --armour --output "${tgz}.asc" \ --detach-sig "${tgz}" @@ -429,15 +444,15 @@ make_src_release() { # - GPG needs to be defined, with the path to GPG: defaults 'gpg'. # - The passphrase in the GPG_PASSPHRASE variable: no default (we don't make .asc file). # - GIT_REF which is the tag to create the tgz from: defaults to 'master'. -# - MVN Default is 'mvn -B'. +# - MVN Default is "mvn -B --settings $MAVEN_SETTINGS_FILE". # For example: # $ GPG_PASSPHRASE="XYZ" GIT_REF="master" make_src_release hbase-operator-tools 1.0.0 make_binary_release() { project="${1}" version="${2}" - basename="${project}-${version}" - rm -rf "${basename}-bin*" - cd $project || exit + base_name="${project}-${version}" + rm -rf "${base_name}"-bin* + cd "$project" || exit git clean -d -f -x # Three invocations of maven. This seems to work. One to @@ -445,30 +460,35 @@ make_binary_release() { # a third to assemble the binary artifact. Trying to do # all in the one invocation fails; a problem in our # assembly spec to in maven. TODO. Meantime, three invocations. - ${MVN} --settings $MAVEN_SETTINGS_FILE clean install -DskipTests - ${MVN} --settings $MAVEN_SETTINGS_FILE site -DskipTests - ${MVN} --settings $MAVEN_SETTINGS_FILE install assembly:single -DskipTests \ - -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" + "${MVN[@]}" clean install -DskipTests + "${MVN[@]}" site -DskipTests + "${MVN[@]}" install assembly:single -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" # Check there is a bin gz output. The build may not produce one: e.g. hbase-thirdparty. - f_bin_tgz="./${PROJECT}-assembly/target/${basename}*-bin.tar.gz" - if ls ${f_bin_tgz} &>/dev/null; then - cp ${f_bin_tgz} .. + local f_bin_prefix="./${PROJECT}-assembly/target/${base_name}" + if ls "${f_bin_prefix}"*-bin.tar.gz &>/dev/null; then + cp "${f_bin_prefix}"*-bin.tar.gz .. cd .. || exit - for i in "${basename}"*-bin.tar.gz; do + for i in "${base_name}"*-bin.tar.gz; do echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --armour --output "$i.asc" --detach-sig "$i" echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --print-md SHA512 "${i}" > "$i.sha512" done else cd .. || exit - echo "No ${f_bin_tgz} product; expected?" + echo "No ${f_bin_prefix}*-bin.tar.gz product; expected?" fi } # Do maven command to set version into local pom function maven_set_version { #input: local this_version="$1" - $MVN versions:set -DnewVersion="$this_version" | grep -v "no value" # silence logs + "${MVN[@]}" versions:set -DnewVersion="$this_version" | grep -v "no value" # silence logs +} + +# Do maven command to read version from local pom +function maven_get_version { + # shellcheck disable=SC2016 + "${MVN[@]}" -q -N -Dexec.executable="echo" -Dexec.args='${project.version}' exec:exec } # Do maven deploy to snapshot or release artifact repository, with checks. @@ -482,12 +502,14 @@ function maven_deploy { #inputs: if [[ -z "$log_file" ]] || ! touch "$log_file"; then error "must provide writable maven log output filepath" fi - if [[ $deploy_type == "snapshot" ]] && ! [[ $VERSION =~ -SNAPSHOT$ ]]; then + # shellcheck disable=SC2153 + if [[ "$deploy_type" == "snapshot" ]] && ! [[ "$VERSION" =~ -SNAPSHOT$ ]]; then error "Snapshots must have a version with suffix '-SNAPSHOT'; you gave version '$VERSION'" - elif [[ $deploy_type == "release" ]] && [[ $VERSION =~ SNAPSHOT ]]; then + elif [[ "$deploy_type" == "release" ]] && [[ "$VERSION" =~ SNAPSHOT ]]; then error "Non-snapshot release version must not include the word 'SNAPSHOT'; you gave version '$VERSION'" fi # Publish ${PROJECT} to Maven release repo + # shellcheck disable=SC2154 echo "Publishing ${PROJECT} checkout at '$GIT_REF' ($git_hash)" echo "Publish version is $VERSION" # Coerce the requested version @@ -496,8 +518,7 @@ function maven_deploy { #inputs: if ! is_dry_run; then mvn_goals=("${mvn_goals[@]}" deploy) fi - if ! ${MVN} --settings "$MAVEN_SETTINGS_FILE" \ - -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \ + if ! "${MVN[@]}" -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \ "${mvn_goals[@]}" > "$log_file"; then error "Deploy build failed, for details see log at '$log_file'." fi From c78261205da8ece89a2f641f4f369612e752bc62 Mon Sep 17 00:00:00 2001 From: Matt Foley Date: Sat, 18 Apr 2020 14:42:14 -0700 Subject: [PATCH 10/14] A few minor improvements, and some debug code that will be removed later. --- dev-support/create-release/do-release.sh | 4 +++- dev-support/create-release/release-build.sh | 16 +++++++------ dev-support/create-release/release-util.sh | 26 +++++++++++++++++++-- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh index d859045887bd..1ed5150179e9 100755 --- a/dev-support/create-release/do-release.sh +++ b/dev-support/create-release/do-release.sh @@ -45,7 +45,8 @@ if [ "$RUNNING_IN_DOCKER" = "1" ]; then eval "$(gpg-agent --disable-scdaemon --daemon --no-grab --allow-preset-passphrase \ --default-cache-ttl=86400 --max-cache-ttl=86400)" echo "GPG Version: $(gpg --version)" - # Inside docker, need to import the GPG key stored in the current directory. + # Inside docker, need to import the GPG keyfile stored in the current directory. + # (On workstation, assume GPG has access to keychain/cache with key_id already imported.) echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --import "$SELF/gpg.key" # We may need to adjust the path since JAVA_HOME may be overridden by the driver script. @@ -93,6 +94,7 @@ else fi if should_build "publish-dist"; then + debug_show_gpg_params run_silent "Publishing distribution packages (tarballs)" "publish-dist.log" \ "$SELF/release-build.sh" publish-dist else diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index 982f24a31d49..6b6b98543602 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -89,11 +89,7 @@ if [[ "$*" == *"help"* ]]; then exit_with_usage fi -export LC_ALL=C.UTF-8 -export LANG=C.UTF-8 -GPG_TTY="$(tty)" -export GPG_TTY - +init_locale init_java init_mvn init_python @@ -159,8 +155,13 @@ if [[ "$1" == "tag" ]]; then fi ### Below is for 'publish-*' stages ### - -check_get_passwords ASF_PASSWORD GPG_PASSPHRASE +set -x +check_get_passwords ASF_PASSWORD +if [[ -z "$GPG_PASSPHRASE" ]]; then + check_get_passwords GPG_PASSPHRASE + GPG_TTY="$(tty)" + export GPG_TTY +fi check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD GPG_KEY GPG_PASSPHRASE # Commit ref to checkout when building @@ -209,6 +210,7 @@ if [[ "$1" == "publish-dist" ]]; then make_src_release "${PROJECT}" "${VERSION}" echo "$(date -u +'%Y-%m-%dT%H:%M:%SZ') Building binary dist" + debug_show_gpg_params make_binary_release "${PROJECT}" "${VERSION}" echo "$(date -u +'%Y-%m-%dT%H:%M:%SZ') Done building binary distribution" diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index 3ae1904f58c0..32fe81889257 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -219,8 +219,8 @@ function get_release_info { Release details: GIT_BRANCH: $GIT_BRANCH RELEASE_VERSION: $RELEASE_VERSION -RELEASE_TAG: $RELEASE_TAG NEXT_VERSION: $NEXT_VERSION +RELEASE_TAG: $RELEASE_TAG $([[ "$GIT_REF" != "$RELEASE_TAG" ]] && printf "\n%s\n" "GIT_REF: $GIT_REF") API_DIFF_TAG: $API_DIFF_TAG ASF_USERNAME: $ASF_USERNAME GPG_KEY: $GPG_KEY @@ -286,6 +286,28 @@ function check_needed_vars { return 0 } +function debug_show_gpg_params { + echo "===================" + echo "GPG-related env variables:" + env | grep -i gpg | sort + echo "===================" + echo "Available key ids:" + gpg --list-keys + echo "===================" +} + +function init_locale { + local locale_value + OS=`uname -s` + case "${OS}" in + Darwin*) locale_value="en_US.UTF-8";; + Linux*) locale_value="C.UTF-8";; + *) error "unknown OS";; + esac + export LC_ALL="$locale_value" + export LANG="$locale_value" +} + # Initializes JAVA_VERSION to the version of the JVM in use. function init_java { if [ -z "$JAVA_HOME" ]; then @@ -462,7 +484,7 @@ make_binary_release() { # assembly spec to in maven. TODO. Meantime, three invocations. "${MVN[@]}" clean install -DskipTests "${MVN[@]}" site -DskipTests - "${MVN[@]}" install assembly:single -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" + "${MVN[@]}" -e -X install assembly:single -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" # Check there is a bin gz output. The build may not produce one: e.g. hbase-thirdparty. local f_bin_prefix="./${PROJECT}-assembly/target/${base_name}" From 8463b28ccc243bcf2a4433e7c1510fb970715d0e Mon Sep 17 00:00:00 2001 From: Matt Foley Date: Tue, 21 Apr 2020 18:40:08 -0700 Subject: [PATCH 11/14] - stop deleting maven settings file, now that it doesn't contain password strings - do maven-gpg-plugin config settings in maven settings file correctly as documented - fix gpg signing failure on Mac due to gpg-agent timeout - remove gpg debug code --- dev-support/create-release/do-release.sh | 1 - dev-support/create-release/release-build.sh | 7 ++-- dev-support/create-release/release-util.sh | 38 +++++++++++++++------ 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh index 1ed5150179e9..9497d1b281e3 100755 --- a/dev-support/create-release/do-release.sh +++ b/dev-support/create-release/do-release.sh @@ -94,7 +94,6 @@ else fi if should_build "publish-dist"; then - debug_show_gpg_params run_silent "Publishing distribution packages (tarballs)" "publish-dist.log" \ "$SELF/release-build.sh" publish-dist else diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index 6b6b98543602..4bbb863897d1 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -72,11 +72,10 @@ EOF set -e function cleanup { - echo "Cleaning up temp settings file." >&2 - rm -f "${MAVEN_SETTINGS_FILE}" &> /dev/null || true # If REPO was set, then leave things be. Otherwise if we defined a repo clean it out. if [[ -z "${REPO}" ]] && [[ -n "${MAVEN_LOCAL_REPO}" ]]; then - echo "Cleaning up temp repo in '${MAVEN_LOCAL_REPO}'. set REPO to reuse downloads." >&2 + echo "Cleaning up temp repo in '${MAVEN_LOCAL_REPO}'. Set REPO to reuse downloads." >&2 + rm -f "${MAVEN_SETTINGS_FILE}" &> /dev/null || true rm -rf "${MAVEN_LOCAL_REPO}" &> /dev/null || true fi } @@ -155,7 +154,6 @@ if [[ "$1" == "tag" ]]; then fi ### Below is for 'publish-*' stages ### -set -x check_get_passwords ASF_PASSWORD if [[ -z "$GPG_PASSPHRASE" ]]; then check_get_passwords GPG_PASSPHRASE @@ -210,7 +208,6 @@ if [[ "$1" == "publish-dist" ]]; then make_src_release "${PROJECT}" "${VERSION}" echo "$(date -u +'%Y-%m-%dT%H:%M:%SZ') Building binary dist" - debug_show_gpg_params make_binary_release "${PROJECT}" "${VERSION}" echo "$(date -u +'%Y-%m-%dT%H:%M:%SZ') Done building binary distribution" diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index 32fe81889257..808faf757666 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -286,16 +286,6 @@ function check_needed_vars { return 0 } -function debug_show_gpg_params { - echo "===================" - echo "GPG-related env variables:" - env | grep -i gpg | sort - echo "===================" - echo "Available key ids:" - gpg --list-keys - echo "===================" -} - function init_locale { local locale_value OS=`uname -s` @@ -362,7 +352,19 @@ function configure_maven { ${env.ASF_PASSWORD} apache.releases.https${env.ASF_USERNAME} ${env.ASF_PASSWORD} + gpg.passphrase + ${env.GPG_PASSPHRASE} + + + + true + + + ${env.GPG_KEY} + + + EOF } @@ -484,7 +486,8 @@ make_binary_release() { # assembly spec to in maven. TODO. Meantime, three invocations. "${MVN[@]}" clean install -DskipTests "${MVN[@]}" site -DskipTests - "${MVN[@]}" -e -X install assembly:single -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" + kick_gpg_agent + "${MVN[@]}" install assembly:single -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" # Check there is a bin gz output. The build may not produce one: e.g. hbase-thirdparty. local f_bin_prefix="./${PROJECT}-assembly/target/${base_name}" @@ -501,6 +504,19 @@ make_binary_release() { fi } +# "Wake up" the gpg agent so it responds properly to maven-gpg-plugin, and doesn't cause timeout. +# Specifically this is done between invocation of 'mvn site' and 'mvn assembly:single', because +# the 'site' build takes long enough that the gpg-agent does become unresponsive and the following +# 'assembly' build (where gpg signing occurs) experiences timeout, without this "kick". +function kick_gpg_agent { + # All that's needed is to run gpg on a random file + local i + i="$(mktemp)" + echo "This is a test file" > "$i" + echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --armour --output "$i.asc" --detach-sig "$i" + rm "$i" "$i.asc" +} + # Do maven command to set version into local pom function maven_set_version { #input: local this_version="$1" From 556e5114754ace5d29581ba332c9539c4c655179 Mon Sep 17 00:00:00 2001 From: Matt Foley Date: Thu, 30 Apr 2020 21:27:14 -0700 Subject: [PATCH 12/14] - fix various bugs to enable publish-dist, publish-snapshot, and publish-release to work correctly as individual steps and/or without docker - fix another gpg timeout - improve log reporting from publish-release step --- dev-support/create-release/do-release.sh | 6 ++- dev-support/create-release/release-build.sh | 43 +++++++++++++-------- dev-support/create-release/release-util.sh | 15 ++++--- 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh index 9497d1b281e3..67c7869eb33b 100755 --- a/dev-support/create-release/do-release.sh +++ b/dev-support/create-release/do-release.sh @@ -100,7 +100,11 @@ else echo "Skipping publish-dist step." fi -if should_build "publish-release"; then +if should_build "publish-snapshot"; then + run_silent "Publishing snapshot" "publish-snapshot.log" \ + "$SELF/release-build.sh" publish-snapshot + +elif should_build "publish-release"; then run_silent "Publishing release" "publish-release.log" \ "$SELF/release-build.sh" publish-release else diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index 4bbb863897d1..cf2273424694 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -48,16 +48,16 @@ Used only for 'tag': GIT_NAME - Name to use with git GIT_EMAIL - E-mail address to use with git GIT_BRANCH - Git branch on which to make release - RELEASE_VERSION - Version used in pom files for release - RELEASE_TAG - Name of release tag - NEXT_VERSION - Development version after release + RELEASE_VERSION - Version used in pom files for release (e.g. 2.1.2) + RELEASE_TAG - Name of release tag (e.g. 2.1.2RC0) + NEXT_VERSION - Development version after release (e.g. 2.1.3-SNAPSHOT) Used only for 'publish': - GIT_REF - Release tag or commit to build from + GIT_REF - Release tag or commit to build from (default $RELEASE_TAG) VERSION - Version of project to be built (e.g. 2.1.2). Optional for 'publish', as it defaults to the version in pom at GIT_REF, which typically will have been set by 'tag'. - PACKAGE_VERSION - Release identifier in top level dist directory (e.g. 2.1.2RC1) + PACKAGE_VERSION - Release identifier in top level dist directory (default $RELEASE_TAG) GPG_KEY - GPG key id (usually email addr) used to sign release artifacts GPG_PASSPHRASE - Passphrase for GPG key REPO - Set to full path of a directory to use as maven local repo (dependencies cache) @@ -148,7 +148,7 @@ if [[ "$1" == "tag" ]]; then else cd .. mv "${PROJECT}" "${PROJECT}.tag" - echo "Clone with version changes and tag available as ${PROJECT}.tag in the output directory." + echo "Dry run: Clone with version changes and tag available as ${PROJECT}.tag in the output directory." fi exit 0 fi @@ -243,7 +243,7 @@ if [[ "$1" == "publish-dist" ]]; then rm -rf "$svn_target" else mv "$svn_target/${DEST_DIR_NAME}" "${svn_target}_${DEST_DIR_NAME}.dist" - echo "svn-managed 'dist' directory with release tarballs, CHANGES.md and RELEASENOTES.md available as $(pwd)/${svn_target}_${DEST_DIR_NAME}.dist" + echo "Dry run: svn-managed 'dist' directory with release tarballs, CHANGES.md and RELEASENOTES.md available as $(pwd)/${svn_target}_${DEST_DIR_NAME}.dist" rm -rf "$svn_target" fi @@ -251,24 +251,35 @@ if [[ "$1" == "publish-dist" ]]; then fi if [[ "$1" == "publish-snapshot" ]]; then - pushd "${PROJECT}" - maven_deploy snapshot "${BASE_DIR}/mvn_deploy.log" - popd - exit 0 + ( + cd "${PROJECT}" + mvn_log="${BASE_DIR}/mvn_deploy_snapshot.log" + echo "Publishing snapshot to nexus" + maven_deploy snapshot "$mvn_log" + if ! is_dry_run; then + echo "Snapshot artifacts successfully published to repo." + rm "$mvn_log" + else + echo "Dry run: Snapshot artifacts successfully built, but not published due to dry run." + fi + ) + exit $? fi if [[ "$1" == "publish-release" ]]; then ( cd "${PROJECT}" + mvn_log="${BASE_DIR}/mvn_deploy_release.log" echo "Staging release in nexus" - maven_deploy release "${BASE_DIR}/mvn_deploy.log" + maven_deploy release "$mvn_log" declare staged_repo_id="dryrun-no-repo" if ! is_dry_run; then - staged_repo_id=$(grep -o "Closing staging repository with ID .*" "${BASE_DIR}/mvn_deploy.log" \ + staged_repo_id=$(grep -o "Closing staging repository with ID .*" "$mvn_log" \ | sed -e 's/Closing staging repository with ID "\([^"]*\)"./\1/') - echo "Artifacts successfully staged to repo ${staged_repo_id}" + echo "Release artifacts successfully published to repo ${staged_repo_id}" + rm "$mvn_log" else - echo "Artifacts successfully built. Not staged due to dry run." + echo "Dry run: Release artifacts successfully built, but not published due to dry run." fi # Dump out email to send. Where we find vote.tmpl depends # on where this script is run from @@ -276,7 +287,7 @@ if [[ "$1" == "publish-release" ]]; then export PROJECT_TEXT eval "echo \"$(< "${SELF}/vote.tmpl")\"" |tee "${BASE_DIR}/vote.txt" ) - exit 0 + exit $? fi cd .. diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index 808faf757666..1cbbef7c68b3 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -520,6 +520,7 @@ function kick_gpg_agent { # Do maven command to set version into local pom function maven_set_version { #input: local this_version="$1" + echo "${MVN[@]}" versions:set -DnewVersion="$this_version" "${MVN[@]}" versions:set -DnewVersion="$this_version" | grep -v "no value" # silence logs } @@ -533,11 +534,11 @@ function maven_get_version { function maven_deploy { #inputs: # Invoke with cwd=$PROJECT local deploy_type="$1" - local log_file="$2" + local mvn_log_file="$2" #secondary log file used later to extract staged_repo_id if [[ "$deploy_type" != "snapshot" && "$deploy_type" != "release" ]]; then error "unrecognized deploy type, must be 'snapshot'|'release'" fi - if [[ -z "$log_file" ]] || ! touch "$log_file"; then + if [[ -z "$mvn_log_file" ]] || ! touch "$mvn_log_file"; then error "must provide writable maven log output filepath" fi # shellcheck disable=SC2153 @@ -546,19 +547,23 @@ function maven_deploy { #inputs: elif [[ "$deploy_type" == "release" ]] && [[ "$VERSION" =~ SNAPSHOT ]]; then error "Non-snapshot release version must not include the word 'SNAPSHOT'; you gave version '$VERSION'" fi - # Publish ${PROJECT} to Maven release repo + # Publish ${PROJECT} to Maven repo # shellcheck disable=SC2154 echo "Publishing ${PROJECT} checkout at '$GIT_REF' ($git_hash)" echo "Publish version is $VERSION" # Coerce the requested version maven_set_version "$VERSION" + # Prepare for signing + kick_gpg_agent declare -a mvn_goals=(clean install) if ! is_dry_run; then mvn_goals=("${mvn_goals[@]}" deploy) fi + echo "${MVN[@]}" -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \ + "${mvn_goals[@]}" if ! "${MVN[@]}" -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \ - "${mvn_goals[@]}" > "$log_file"; then - error "Deploy build failed, for details see log at '$log_file'." + "${mvn_goals[@]}" | tee "$mvn_log_file"; then + error "Deploy build failed, for details see log at '$mvn_log_file'." fi return 0 } From 0ed5931ff23cd5e097ace40c8f83545d83865eee Mon Sep 17 00:00:00 2001 From: Matt Foley Date: Fri, 1 May 2020 14:17:12 -0700 Subject: [PATCH 13/14] - fix bug in argument to checkcompatibility.py: replace PACKAGE_VERSION with GIT_REF - demote "PACKAGE_VERSION" to "package_version_name" and undocument it in favor of RELEASE_TAG. Still enable appropriate defaulting in case RELEASE_TAG is undefined. - unify RELEASE_VERSION with VERSION, to remove ambiguity and allow it to be set when only running 'publish' step without 'tag' - query confirm RELEASE_TAG - emphasize that release-build.sh is called for a single action at a time, and should be called from do-release.sh. - add '-s' option to do-release.sh - suppress maven "Download from central:" messages --- dev-support/create-release/README.txt | 8 ++- .../create-release/do-release-docker.sh | 1 - dev-support/create-release/do-release.sh | 15 ++-- dev-support/create-release/release-build.sh | 71 +++++++++++-------- dev-support/create-release/release-util.sh | 45 +++++++----- 5 files changed, 82 insertions(+), 58 deletions(-) diff --git a/dev-support/create-release/README.txt b/dev-support/create-release/README.txt index d5d499a8ab7f..62703a176bdc 100644 --- a/dev-support/create-release/README.txt +++ b/dev-support/create-release/README.txt @@ -5,9 +5,11 @@ For usage, pass '-h': $ ./do-release-docker.sh -h -To run a build w/o invoking docker (not recommended!), use -_do_release.sh_. It does not take parameters. It will ask -you what commands to run with taking defaults from environment. +To run a build w/o invoking docker (not recommended!), use _do_release.sh_. + +Both scripts will query interactively for needed parameters and passphrases. +For explanation of the parameters, execute: + $ release-build.sh --help Before starting the RC build, run a reconciliation of what is in JIRA with what is in the commit log. Make sure they align and that diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh index 10bb1298567f..645e8f236d65 100755 --- a/dev-support/create-release/do-release-docker.sh +++ b/dev-support/create-release/do-release-docker.sh @@ -152,7 +152,6 @@ NEXT_VERSION=$NEXT_VERSION RELEASE_VERSION=$RELEASE_VERSION RELEASE_TAG=$RELEASE_TAG GIT_REF=$GIT_REF -PACKAGE_VERSION=$PACKAGE_VERSION ASF_USERNAME=$ASF_USERNAME GIT_NAME=$GIT_NAME GIT_EMAIL=$GIT_EMAIL diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh index 67c7869eb33b..94da4a775ebd 100755 --- a/dev-support/create-release/do-release.sh +++ b/dev-support/create-release/do-release.sh @@ -27,10 +27,11 @@ SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=SCRIPTDIR/release-util.sh . "$SELF/release-util.sh" -while getopts "b:f" opt; do +while getopts "b:fs:" opt; do case $opt in b) export GIT_BRANCH=$OPTARG ;; f) export DRY_RUN=0 ;; # "force", ie actually publish this release (otherwise defaults to dry run) + s) RELEASE_STEP="$OPTARG" ;; ?) error "Invalid option: $OPTARG" ;; esac done @@ -63,6 +64,13 @@ fi GPG_TTY="$(tty)" export GPG_TTY +if [[ -z "$RELEASE_STEP" ]]; then + # If doing all stages, leave out 'publish-snapshot' + RELEASE_STEP="tag_publish-dist_publish-release" + # and use shared maven local repo for efficiency + export REPO="${REPO:-$(pwd)/$(mktemp -d hbase-repo-XXXXX)}" +fi + function should_build { local WHAT=$1 if [[ -z "$RELEASE_STEP" ]]; then @@ -74,11 +82,6 @@ function should_build { fi } -# If doing all stages, use shared maven local repo -if [[ -z "$RELEASE_STEP" ]]; then - export REPO="${REPO:-$(pwd)/$(mktemp -d hbase-repo-XXXXX)}" -fi - if should_build "tag" && [ "$SKIP_TAG" = 0 ]; then run_silent "Creating release tag $RELEASE_TAG..." "tag.log" \ "$SELF/release-build.sh" tag diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index cf2273424694..1b378f45fa50 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -29,39 +29,46 @@ function exit_with_usage { cat <<'EOF' Usage: release-build.sh Creates release deliverables from a tag or commit. -Arguments: - tag Prepares for release on specified git branch: Set release version, - update CHANGES and RELEASENOTES, create release tag, - increment version for ongoing dev, and publish to Apache git repo. - publish-dist Build and publish distribution packages (tarballs) to Apache dist repo - publish-snapshot Build and publish maven artifacts snapshot release to Apache snapshots repo - publish-release Build and publish maven artifacts release to Apache release repo, and - construct vote email from template - -All other inputs are environment variables: -Used for 'tag' and 'publish': +Argument: one of 'tag', 'publish-dist', 'publish-snapshot', or 'publish-release' + tag Prepares for release on specified git branch: Set release version, + update CHANGES and RELEASENOTES, create release tag, + increment version for ongoing dev, and publish to Apache git repo. + publish-dist Build and publish distribution packages (tarballs) to Apache dist repo + publish-snapshot Build and publish maven artifacts snapshot release to Apache snapshots repo + publish-release Build and publish maven artifacts release to Apache release repo, and + construct vote email from template + +All other inputs are environment variables. Please use do-release-docker.sh or +do-release.sh to set up the needed environment variables. This script, release-build.sh, +is not intended to be called stand-alone, and such use is untested. The env variables used are: + +Used for 'tag' and 'publish' stages: PROJECT - The project to build. No default. + RELEASE_VERSION - Version used in pom files for release (e.g. 2.1.2) + Required for 'tag'; defaults for 'publish' to the version in pom at GIT_REF + RELEASE_TAG - Name of release tag (e.g. 2.1.2RC0), also used by + publish-dist as package version name in dist directory path ASF_USERNAME - Username of ASF committer account ASF_PASSWORD - Password of ASF committer account + DRY_RUN - 1:true (default), 0:false. If "1", does almost all the work, but doesn't actually + publish anything to upstream source or object repositories. It defaults to "1", so if you want + to actually publish you have to set '-f' (force) flag in do-release.sh or do-release-docker.sh. Used only for 'tag': GIT_NAME - Name to use with git GIT_EMAIL - E-mail address to use with git - GIT_BRANCH - Git branch on which to make release - RELEASE_VERSION - Version used in pom files for release (e.g. 2.1.2) - RELEASE_TAG - Name of release tag (e.g. 2.1.2RC0) + GIT_BRANCH - Git branch on which to make release. Tag is always placed at HEAD of this branch. NEXT_VERSION - Development version after release (e.g. 2.1.3-SNAPSHOT) Used only for 'publish': - GIT_REF - Release tag or commit to build from (default $RELEASE_TAG) - VERSION - Version of project to be built (e.g. 2.1.2). - Optional for 'publish', as it defaults to the version in pom at GIT_REF, - which typically will have been set by 'tag'. - PACKAGE_VERSION - Release identifier in top level dist directory (default $RELEASE_TAG) + GIT_REF - Release tag or commit to build from (defaults to $RELEASE_TAG; only need to + separately define GIT_REF if RELEASE_TAG is not actually present as a tag at publish time) + If both RELEASE_TAG and GIT_REF are undefined it will default to HEAD of master. GPG_KEY - GPG key id (usually email addr) used to sign release artifacts GPG_PASSPHRASE - Passphrase for GPG key REPO - Set to full path of a directory to use as maven local repo (dependencies cache) - to avoid re-downloading dependencies for each stage. + to avoid re-downloading dependencies for each stage. It is automatically set if you + request full sequence of stages (tag, publish-dist, publish-release) in do-release.sh. For example: $ PROJECT="hbase-operator-tools" ASF_USERNAME=NAME ASF_PASSWORD=PASSWORD GPG_PASSPHRASE=PASSWORD GPG_KEY=stack@apache.org ./release-build.sh publish-dist @@ -100,6 +107,7 @@ rm -rf "${PROJECT}" if [[ "$1" == "tag" ]]; then # for 'tag' stage set -o pipefail + set -x # detailed logging during action check_get_passwords ASF_PASSWORD check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION \ GIT_EMAIL GIT_NAME GIT_BRANCH @@ -183,8 +191,8 @@ git checkout "$GIT_REF" git_hash="$(git rev-parse --short HEAD)" echo "Checked out ${PROJECT} at ${GIT_REF} commit $git_hash" -if [ -z "$VERSION" ]; then - VERSION="$(maven_get_version)" +if [ -z "${RELEASE_VERSION}" ]; then + RELEASE_VERSION="$(maven_get_version)" fi # This is a band-aid fix to avoid the failure of Maven nightly snapshot in some Jenkins @@ -195,26 +203,28 @@ if ! hash $LSOF 2>/dev/null; then LSOF=/usr/sbin/lsof fi -if [ -z "$PACKAGE_VERSION" ]; then - PACKAGE_VERSION="${VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}" +package_version_name="$RELEASE_TAG" +if [ -z "$package_version_name" ]; then + package_version_name="${RELEASE_VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}" fi git clean -d -f -x cd .. +set -x # detailed logging during action if [[ "$1" == "publish-dist" ]]; then # Source and binary tarballs echo "Packaging release source tarballs" - make_src_release "${PROJECT}" "${VERSION}" + make_src_release "${PROJECT}" "${RELEASE_VERSION}" echo "$(date -u +'%Y-%m-%dT%H:%M:%SZ') Building binary dist" - make_binary_release "${PROJECT}" "${VERSION}" + make_binary_release "${PROJECT}" "${RELEASE_VERSION}" echo "$(date -u +'%Y-%m-%dT%H:%M:%SZ') Done building binary distribution" if [[ "$PROJECT" =~ ^hbase- ]]; then - DEST_DIR_NAME="${PROJECT}-${PACKAGE_VERSION}" + DEST_DIR_NAME="${PROJECT}-${package_version_name}" else - DEST_DIR_NAME="$PACKAGE_VERSION" + DEST_DIR_NAME="$package_version_name" fi svn_target="svn-${PROJECT}" svn co --depth=empty "$RELEASE_STAGING_LOCATION" "$svn_target" @@ -229,7 +239,7 @@ if [[ "$1" == "publish-dist" ]]; then # Generate api report only if project is hbase for now. if [ "${PROJECT}" == "hbase" ]; then # This script usually reports an errcode along w/ the report. - generate_api_report "./${PROJECT}" "${API_DIFF_TAG}" "${PACKAGE_VERSION}" || true + generate_api_report "./${PROJECT}" "${API_DIFF_TAG}" "${GIT_REF}" || true cp api*.html "$svn_target/${DEST_DIR_NAME}/" fi shopt -u nocasematch @@ -238,7 +248,7 @@ if [[ "$1" == "publish-dist" ]]; then if ! is_dry_run; then cd "$svn_target" - svn ci --username "$ASF_USERNAME" --password "$ASF_PASSWORD" -m"Apache ${PROJECT} $PACKAGE_VERSION" --no-auth-cache + svn ci --username "$ASF_USERNAME" --password "$ASF_PASSWORD" -m"Apache ${PROJECT} $package_version_name" --no-auth-cache cd .. rm -rf "$svn_target" else @@ -290,6 +300,7 @@ if [[ "$1" == "publish-release" ]]; then exit $? fi +set +x # done with detailed logging cd .. rm -rf "${PROJECT}" echo "ERROR: expects to be called with 'tag', 'publish-dist', 'publish-release', or 'publish-snapshot'" >&2 diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index 1cbbef7c68b3..830c640bff99 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -179,9 +179,10 @@ function get_release_info { export RELEASE_VERSION NEXT_VERSION RC_COUNT="$(read_config "RC_COUNT" "$RC_COUNT")" + RELEASE_TAG="${RELEASE_VERSION}RC${RC_COUNT}" + RELEASE_TAG="$(read_config "RELEASE_TAG" "$RELEASE_TAG")" # Check if the RC already exists, and if re-creating the RC, skip tag creation. - RELEASE_TAG="${RELEASE_VERSION}RC${RC_COUNT}" SKIP_TAG=0 if check_for_tag "$RELEASE_TAG"; then read -r -p "$RELEASE_TAG already exists. Continue anyway [y/n]? " ANSWER @@ -196,11 +197,10 @@ function get_release_info { GIT_REF="$RELEASE_TAG" if is_dry_run; then - echo "This is a dry run. Please confirm the ref that will be built for testing." + echo "This is a dry run. If tag does not actually exist, please confirm the ref that will be built for testing." GIT_REF="$(read_config "GIT_REF" "$GIT_REF")" fi export GIT_REF - export PACKAGE_VERSION="$RELEASE_TAG" API_DIFF_TAG="$(get_api_diff_version "$RELEASE_VERSION")" @@ -288,7 +288,7 @@ function check_needed_vars { function init_locale { local locale_value - OS=`uname -s` + OS="$(uname -s)" case "${OS}" in Darwin*) locale_value="en_US.UTF-8";; Linux*) locale_value="C.UTF-8";; @@ -324,16 +324,19 @@ function init_mvn { else error "MAVEN_HOME is not set nor is mvn on the current path." fi - echo "mvn version: $("${MVN[@]}" --version)" # Add batch mode. MVN=("${MVN[@]}" -B) export MVN + echo -n "mvn version: " + "${MVN[@]}" --version configure_maven } function configure_maven { # Add timestamps to mvn logs. MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss ${MAVEN_OPTS}" + # Suppress gobs of "Download from central:" messages + MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn ${MAVEN_OPTS}" MAVEN_LOCAL_REPO="${REPO:-$(pwd)/$(mktemp -d hbase-repo-XXXXX)}" [[ -d "$MAVEN_LOCAL_REPO" ]] || mkdir -p "$MAVEN_LOCAL_REPO" MAVEN_SETTINGS_FILE="${MAVEN_LOCAL_REPO}/tmp-settings.xml" @@ -446,9 +449,9 @@ function update_releasenotes { # $ GPG_PASSPHRASE="XYZ" GIT_REF="master" make_src_release hbase-operator-tools 1.0.0 make_src_release() { # Tar up the src and sign and hash it. - project="${1}" - version="${2}" - base_name="${project}-${version}" + local project="${1}" + local version="${2}" + local base_name="${project}-${version}" rm -rf "${base_name}"-src* tgz="${base_name}-src.tar.gz" cd "${project}" || exit @@ -472,9 +475,9 @@ make_src_release() { # For example: # $ GPG_PASSPHRASE="XYZ" GIT_REF="master" make_src_release hbase-operator-tools 1.0.0 make_binary_release() { - project="${1}" - version="${2}" - base_name="${project}-${version}" + local project="${1}" + local version="${2}" + local base_name="${project}-${version}" rm -rf "${base_name}"-bin* cd "$project" || exit @@ -542,17 +545,17 @@ function maven_deploy { #inputs: error "must provide writable maven log output filepath" fi # shellcheck disable=SC2153 - if [[ "$deploy_type" == "snapshot" ]] && ! [[ "$VERSION" =~ -SNAPSHOT$ ]]; then - error "Snapshots must have a version with suffix '-SNAPSHOT'; you gave version '$VERSION'" - elif [[ "$deploy_type" == "release" ]] && [[ "$VERSION" =~ SNAPSHOT ]]; then - error "Non-snapshot release version must not include the word 'SNAPSHOT'; you gave version '$VERSION'" + if [[ "$deploy_type" == "snapshot" ]] && ! [[ "$RELEASE_VERSION" =~ -SNAPSHOT$ ]]; then + error "Snapshots must have a version with suffix '-SNAPSHOT'; you gave version '$RELEASE_VERSION'" + elif [[ "$deploy_type" == "release" ]] && [[ "$RELEASE_VERSION" =~ SNAPSHOT ]]; then + error "Non-snapshot release version must not include the word 'SNAPSHOT'; you gave version '$RELEASE_VERSION'" fi # Publish ${PROJECT} to Maven repo # shellcheck disable=SC2154 echo "Publishing ${PROJECT} checkout at '$GIT_REF' ($git_hash)" - echo "Publish version is $VERSION" + echo "Publish version is $RELEASE_VERSION" # Coerce the requested version - maven_set_version "$VERSION" + maven_set_version "$RELEASE_VERSION" # Prepare for signing kick_gpg_agent declare -a mvn_goals=(clean install) @@ -561,9 +564,15 @@ function maven_deploy { #inputs: fi echo "${MVN[@]}" -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \ "${mvn_goals[@]}" + echo "Logging to ${mvn_log_file}. This will take a while..." + rm -f "$mvn_log_file" + # The tortuous redirect in the next command allows mvn's stdout and stderr to go to mvn_log_file, + # while also sending stderr back to the caller. + # shellcheck disable=SC2094 if ! "${MVN[@]}" -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \ - "${mvn_goals[@]}" | tee "$mvn_log_file"; then + "${mvn_goals[@]}" 1>> "$mvn_log_file" 2> >( tee -a "$mvn_log_file" >&2 ); then error "Deploy build failed, for details see log at '$mvn_log_file'." fi + echo "BUILD SUCCESS." return 0 } From 954e0646b129bbaa6aac9c2910600e6b4b082269 Mon Sep 17 00:00:00 2001 From: Matt Foley Date: Tue, 5 May 2020 22:30:21 -0700 Subject: [PATCH 14/14] Replace human wait for tag propagation --- dev-support/create-release/do-release.sh | 4 ---- dev-support/create-release/release-build.sh | 3 ++- dev-support/create-release/release-util.sh | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh index 94da4a775ebd..313d277f6aec 100755 --- a/dev-support/create-release/do-release.sh +++ b/dev-support/create-release/do-release.sh @@ -87,10 +87,6 @@ if should_build "tag" && [ "$SKIP_TAG" = 0 ]; then "$SELF/release-build.sh" tag if is_dry_run; then export TAG_SAME_DRY_RUN="true"; - else - echo "It may take some time for the tag to be synchronized to github." - echo "Press enter when you've verified that the new tag ($RELEASE_TAG) is available." - read -r fi else echo "Skipping tag creation for $RELEASE_TAG." diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index 1b378f45fa50..ba6f3e817ae6 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -99,7 +99,7 @@ init_locale init_java init_mvn init_python -# Print out subset of perl version (used in git hooks) +# Print out subset of perl version (used in git hooks and japi-compliance-checker) perl --version | grep 'This is' rm -rf "${PROJECT}" @@ -151,6 +151,7 @@ if [[ "$1" == "tag" ]]; then # Push changes git push origin "$RELEASE_TAG" git push origin "HEAD:$GIT_BRANCH" + wait_for_tag "$RELEASE_TAG" cd .. rm -rf "${PROJECT}" else diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index 830c640bff99..397a20c77835 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -79,6 +79,23 @@ function check_for_tag { curl -s --head --fail "$ASF_GITHUB_REPO/releases/tag/$1" > /dev/null } +function wait_for_tag { + # Confirm the tag synchronizes to github. This can take a couple minutes, + # but usually it just takes a few seconds. + local max_propagation_time=300 + local prop_delay=30 + while ! check_for_tag "$1"; do + if (( max_propagation_time <= 0 )); then + echo "ERROR: Taking more than 5 minutes to propagate Release Tag $1 to github mirror." >&2 + echo "Please wait and resume other create-release steps when $1 is available in github." >&2 + exit 1 + fi + echo "Waiting up to $max_propagation_time seconds for tag to propagate to github mirror..." + sleep $prop_delay + max_propagation_time=$((max_propagation_time - prop_delay)) + done +} + # API compare version. function get_api_diff_version { local version="$1"