From 6141a2656266c14581e9be3d77a1d184feffb2a4 Mon Sep 17 00:00:00 2001 From: Parth Chandra Date: Fri, 13 Sep 2024 17:22:13 -0700 Subject: [PATCH 1/2] feat: publish artifacts to staging repository --- dev/release/README.md | 44 +++++++- dev/release/publish-to-maven.sh | 178 ++++++++++++++++++++++++++++++++ 2 files changed, 220 insertions(+), 2 deletions(-) create mode 100755 dev/release/publish-to-maven.sh diff --git a/dev/release/README.md b/dev/release/README.md index d113452948..058a920fa1 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -156,7 +156,47 @@ Setting up your project in the ASF Nexus Repository from here: https://infra.apa ##### Release Manager Setup Set up your development environment from here: https://infra.apache.org/publishing-maven-artifacts.html -TODO: build and publish a release candidate to nexus. +##### Build and publish a release candidate to nexus. +The script `publish-to-maven.sh` will publish the artifacts created by the `build-release-comet.sh` script. +The artifacts will be signed using the gpg key of the release manager and uploaded to the maven staging repository. + +Note: This script needs `xmllint` to be installed. On MacOS xmllint is available by default. + +On Ubuntu `apt-get install -y libxml2-utils` + +On RedHat `yum install -y xmlstarlet` + +```shell + +/comet:$./dev/release/publish-to-maven.sh -h +usage: publish-to-maven.sh options + +Publish signed artifacts to Maven. + +Options +-u ASF_USERNAME - Username of ASF committer account +-r LOCAL_REPO - path to temporary local maven repo (created and written to by 'build-release-comet.sh') + +The following will be prompted for - +ASF_PASSWORD - Password of ASF committer account +GPG_KEY - GPG key used to sign release artifacts +GPG_PASSPHRASE - Passphrase for GPG key +``` + +example +```shell +/comet:$./dev/release/publish-to-maven.sh -u release_manager_asf_id -r /tmp/comet-staging-repo-VsYOX +ASF Password : +GPG Key (Optional): +GPG Passphrase : +Creating Nexus staging repository +... +``` + +In the Nexus repository UI (https://repository.apache.org/) locate and verify the artifacts in +staging (https://central.sonatype.org/publish/release/#locate-and-examine-your-staging-repository). + +If the artifacts appear to be correct, then close and release the repository so it is made visible. ### Start an Email Voting Thread @@ -227,7 +267,7 @@ svn delete -m "delete old DataFusion Comet release" https://dist.apache.org/repo ### Publishing JAR Files to Maven -The process for publishing JAR files to Maven is not defined yet. +Once the vote has passed, promote the staged release candidate to production in the Nexus repository UI (https://repository.apache.org/). ### Publishing to crates.io diff --git a/dev/release/publish-to-maven.sh b/dev/release/publish-to-maven.sh new file mode 100755 index 0000000000..c0cc4652a4 --- /dev/null +++ b/dev/release/publish-to-maven.sh @@ -0,0 +1,178 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### +# This file is based on the script release-build.sh in Spark +### + +function usage { + local NAME=$(basename $0) + cat << EOF +usage: $NAME options + +Publish signed artifacts to Maven. + +Options + -u ASF_USERNAME - Username of ASF committer account + -r LOCAL_REPO - path to temporary local maven repo (created and written to by 'build-release-comet.sh') + + One of -p or -v must be specified + +The following will be prompted for - + ASF_PASSWORD - Password of ASF committer account + GPG_KEY - GPG key used to sign release artifacts + GPG_PASSPHRASE - Passphrase for GPG key +EOF + exit 1 +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +COMET_HOME_DIR=$SCRIPT_DIR/../.. + +ASF_USERNAME="" +ASF_PASSWORD="" + +NEXUS_ROOT=https://repository.apache.org/service/local/staging +NEXUS_PROFILE=789e15c00fd47 + +while getopts "u:r:h" opt; do + case $opt in + u) ASF_USERNAME="$OPTARG" ;; + r) LOCAL_REPO="$OPTARG" ;; + h) usage ;; + \?) error "Invalid option. Run with -h for help." ;; + esac +done + +if [ "$LOCAL_REPO" == "" ] +then + "Please provide the local Maven repository (from 'build-release-comet.sh')" + usage +fi + +if [ "$ASF_USERNAME" == "" ] +then + read -p "ASF Username : " ASF_USERNAME && echo "" +fi +# Read some secret information +read -s -p "ASF Password : " ASF_PASSWORD && echo "" +read -s -p "GPG Key (Optional): " GPG_KEY && echo "" +read -s -p "GPG Passphrase : " GPG_PASSPHRASE && echo "" + +if [ "$ASF_USERNAME" == "" ] || [ "$ASF_PASSWORD" == "" ] || [ "$GPG_PASSPHRASE" = "" ] +then + echo "Missing credentials" + exit 1 +fi + +# Default GPG command to use +GPG="gpg" +if [ "$GPG_KEY" != "" ] +then + GPG="$GPG -u $GPG_KEY" +fi + +# sha1sum on linux, shasum on macos +SHA1SUM=$( which sha1sum || which shasum) + +GIT_HASH=$(git rev-parse --short HEAD) + +# REF: https://support.sonatype.com/hc/en-us/articles/213465818-How-can-I-programmatically-upload-an-artifact-into-Nexus-Repo-2 +# REF: https://support.sonatype.com/hc/en-us/articles/213465868-Uploading-to-a-Nexus-Repository-2-Staging-Repository-via-REST-API +echo "Creating Nexus staging repository" +# check permission +PERMITTED_REQUEST="-u $ASF_USERNAME:$ASF_PASSWORD \ + -H "Content-Type:application/xml" \ + $NEXUS_ROOT/profiles/$NEXUS_PROFILE/start" +PERMITTED=$(curl -s -o /dev/null -w "%{http_code}" $PERMITTED_REQUEST) + +if [ "$PERMITTED" != "200" ] +then + echo "Nexus replied with a status code: $PERMITTED" + echo "You may not be authorized to perform this action" + exit 1 +fi + +REPO_REQUEST="Apache Datafusion Comet $COMET_VERSION (commit $GIT_HASH)" +REPO_REQUEST_RESPONSE=$(curl -I -X POST -d "$REPO_REQUEST" -u $ASF_USERNAME:$ASF_PASSWORD \ + -H "Content-Type:application/xml" \ + $NEXUS_ROOT/profiles/$NEXUS_PROFILE/start) +if [ $? -ne 0 ] +then + echo "Error creating staged repository" + echo "$REPO_REQUEST_RESPONSE" + exit 1 +fi + +STAGED_REPO_ID=$(echo $REPO_REQUEST_RESPONSE | xmllint -xpath "//stagedRepositoryId/text()" ) +echo "Created Nexus staging repository: $STAGED_REPO_ID" + +if [ "$STAGED_REPO_ID" == "" ] +then + echo "Error creating staged repository" + echo "$REPO_REQUEST_RESPONSE" + exit 1 +fi + +echo "Deploying artifacts from $LOCAL_REPO" + +pushd $LOCAL_REPO/org/apache/datafusion + +# Remove any extra files generated during install +find . -type f |grep -v \.jar |grep -v \.pom | xargs rm + +echo "Creating hash and signature files" +for file in $(find . -type f) +do +echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --output $file.asc \ + --detach-sig --armour $file; +if [ $(command -v md5) ]; then + # Available on OS X; -q to keep only hash + md5 -q $file > $file.md5 +else + # Available on Linux; cut to keep only hash + md5sum $file | cut -f1 -d' ' > $file.md5 +fi +$SHA1SUM $file | cut -f1 -d' ' > $file.sha1 +done + +NEXUS_UPLOAD=$NEXUS_ROOT/deployByRepositoryId/$STAGED_REPO_ID +echo "Uploading files to $NEXUS_UPLOAD" +for file in $(find . -type f) +do + # strip leading ./ + FILE_SHORT=$(echo $file | sed -e "s/\.\///") + DEST_URL="$NEXUS_UPLOAD/org/apache/datafusion/$FILE_SHORT" + echo " Uploading $FILE_SHORT" + curl -u $ASF_USERNAME:$ASF_PASSWORD --upload-file $FILE_SHORT $DEST_URL + if [ $? -ne 0 ] + then + echo " - Failed" + exit 2 + fi +done + +echo "Closing nexus staging repository" +REPO_REQUEST="$STAGED_REPO_IDApache Datafusion Comet $COMET_VERSION (commit $GIT_HASH)" +REPO_REQUEST_RESPONSE=$(curl -X POST -d "$REPO_REQUEST" -u $ASF_USERNAME:$ASF_PASSWORD \ + -H "Content-Type:application/xml" -v \ + $NEXUS_ROOT/profiles/$NEXUS_PROFILE/finish) +echo "Closed Nexus staging repository: $STAGED_REPO_ID" + +popd \ No newline at end of file From 49fd948a3f7546e7393243148ebc258be75971a4 Mon Sep 17 00:00:00 2001 From: Parth Chandra Date: Fri, 20 Sep 2024 15:55:16 -0700 Subject: [PATCH 2/2] address review comments --- dev/release/publish-to-maven.sh | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/dev/release/publish-to-maven.sh b/dev/release/publish-to-maven.sh index c0cc4652a4..9895c8e1e1 100755 --- a/dev/release/publish-to-maven.sh +++ b/dev/release/publish-to-maven.sh @@ -32,8 +32,6 @@ Options -u ASF_USERNAME - Username of ASF committer account -r LOCAL_REPO - path to temporary local maven repo (created and written to by 'build-release-comet.sh') - One of -p or -v must be specified - The following will be prompted for - ASF_PASSWORD - Password of ASF committer account GPG_KEY - GPG key used to sign release artifacts @@ -96,18 +94,6 @@ GIT_HASH=$(git rev-parse --short HEAD) # REF: https://support.sonatype.com/hc/en-us/articles/213465818-How-can-I-programmatically-upload-an-artifact-into-Nexus-Repo-2 # REF: https://support.sonatype.com/hc/en-us/articles/213465868-Uploading-to-a-Nexus-Repository-2-Staging-Repository-via-REST-API echo "Creating Nexus staging repository" -# check permission -PERMITTED_REQUEST="-u $ASF_USERNAME:$ASF_PASSWORD \ - -H "Content-Type:application/xml" \ - $NEXUS_ROOT/profiles/$NEXUS_PROFILE/start" -PERMITTED=$(curl -s -o /dev/null -w "%{http_code}" $PERMITTED_REQUEST) - -if [ "$PERMITTED" != "200" ] -then - echo "Nexus replied with a status code: $PERMITTED" - echo "You may not be authorized to perform this action" - exit 1 -fi REPO_REQUEST="Apache Datafusion Comet $COMET_VERSION (commit $GIT_HASH)" REPO_REQUEST_RESPONSE=$(curl -I -X POST -d "$REPO_REQUEST" -u $ASF_USERNAME:$ASF_PASSWORD \ @@ -120,7 +106,7 @@ then exit 1 fi -STAGED_REPO_ID=$(echo $REPO_REQUEST_RESPONSE | xmllint -xpath "//stagedRepositoryId/text()" ) +STAGED_REPO_ID=$(echo $REPO_REQUEST_RESPONSE | xmllint --xpath "//stagedRepositoryId/text()" -) echo "Created Nexus staging repository: $STAGED_REPO_ID" if [ "$STAGED_REPO_ID" == "" ]