From af8d649d3fd891d8c421630799aae5bb5acd826b Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Tue, 7 Nov 2023 14:55:32 -0600 Subject: [PATCH 1/5] Build source and docs artifacts in GitHub Actions --- .github/workflows/source-and-docs-release.yml | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 .github/workflows/source-and-docs-release.yml diff --git a/.github/workflows/source-and-docs-release.yml b/.github/workflows/source-and-docs-release.yml new file mode 100644 index 00000000..9e771ac3 --- /dev/null +++ b/.github/workflows/source-and-docs-release.yml @@ -0,0 +1,100 @@ +on: + workflow_dispatch: + inputs: + git_remote: + type: choice + description: "Git remote to checkout" + options: + - python + - Yhg1s + - pablogsal + - ambv + git_commit: + type: string + description: "Git commit to target for the release. Must use the full commit SHA, not the short ID" + cpython_release: + type: string + description: "CPython release number (ie '3.11.5', note without the 'v' prefix)" + +name: "Build Python source and docs artifacts" + +jobs: + source-and-docs: + runs-on: ubuntu-22.04 + steps: + - name: "Checkout python/release-tools" + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: "Checkout ${{ inputs.git_remote }}/cpython" + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: "${{ inputs.git_remote }}/cpython" + ref: "v${{ inputs.cpython_release }}" + path: "cpython" + + - name: "Verify CPython commit matches tag" + run: | + if [[ "${{ inputs.git_commit }}" != "$(cd cpython && git rev-parse HEAD)" ]]; then + echo "expected git commit ('${{ inputs.git_commit }}') didn't match tagged commit ('$(git rev-parse HEAD)')" + exit 1 + fi + + - name: "Setup Python" + uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 + with: + python-version: 3.11 + + - name: "Install source dependencies" + run: | + python -m pip install --no-deps \ + -r requirements.txt + + - name: "Install docs dependencies" + # Docs aren't built for alpha or beta releases. + if: ${{ !(contains(inputs.cpython_release, 'a') || contains(inputs.cpython_release, 'b')) }} + run: | + python -m pip install \ + -r cpython/Doc/requirements.txt + + sudo apt-get update + sudo apt-get install --yes --no-install-recommends \ + latexmk texlive-xetex xindy texinfo texlive-latex-base \ + texlive-fonts-recommended texlive-fonts-extra \ + texlive-full + + - name: "Build Python release artifacts" + run: | + cd cpython + python ../release.py --export ${{ inputs.cpython_release }} + + - name: "Test Python source tarballs" + run: | + mkdir -p ./tmp/installation/ + cp cpython/${{ inputs.cpython_release }}/src/Python-${{ inputs.cpython_release }}.tgz ./tmp/ + cd tmp/ + tar xvf Python-${{ inputs.cpython_release }}.tgz + cd Python-${{ inputs.cpython_release }} + + ./configure --prefix=$(realpath '../installation/') + make -j + make install -j + + cd ../installation + ./bin/python3 -m test -uall + + - name: "Upload the source artifacts" + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: source + path: | + cpython/${{ inputs.cpython_release }}/src + + - name: "Upload the docs artifacts" + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + # Conditionally run this step if there is a 'docs/' directory. + # Docs aren't built for alpha or beta releases. + if: ${{ hashFiles('cpython/${{ inputs.cpython_release }}/docs') != '' }} + with: + name: docs + path: | + cpython/${{ inputs.cpython_release }}/docs From 6aa0eda72a24f864d673eec37f305b294de975e6 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Tue, 14 Nov 2023 15:15:10 -0600 Subject: [PATCH 2/5] Move signing to run_release.py, instructions for GitHub Actions builds --- release.py | 13 ----- run_release.py | 150 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 109 insertions(+), 54 deletions(-) diff --git a/release.py b/release.py index 54ab8521..21f9b568 100755 --- a/release.py +++ b/release.py @@ -279,19 +279,6 @@ def tarball(source, clamp_mtime): print(' %s %8s %s' % ( checksum_xz.hexdigest(), int(os.path.getsize(xz)), xz)) - print('Signing tarballs with GPG') - uid = os.environ.get("GPG_KEY_FOR_RELEASE") - if not uid: - print('List of available private keys:') - run_cmd(['gpg -K | grep -A 1 "^sec"'], shell=True) - uid = input('Please enter key ID to use for signing: ') - run_cmd(['gpg', '-bas', '-u', uid, tgz]) - run_cmd(['gpg', '-bas', '-u', uid, xz]) - - print('Signing tarballs with Sigstore') - run_cmd(['python3', '-m', 'sigstore', 'sign', - '--oidc-disable-ambient-providers', tgz, xz], shell=False) - def export(tag, silent=False): make_dist(tag.text) diff --git a/run_release.py b/run_release.py index 59e793f7..e32d204b 100755 --- a/run_release.py +++ b/run_release.py @@ -295,7 +295,6 @@ def check_tool(db: DbfilenameShelf, tool: str) -> None: check_git = functools.partial(check_tool, tool="git") -check_latexmk = functools.partial(check_tool, tool="latexmk") check_make = functools.partial(check_tool, tool="make") check_blurb = functools.partial(check_tool, tool="blurb") check_autoconf = functools.partial(check_tool, tool="autoconf") @@ -460,39 +459,62 @@ def create_tag(db: DbfilenameShelf) -> None: ) -def build_release_artifacts(db: DbfilenameShelf) -> None: - with cd(db["git_repo"]): - release_mod.export(db["release"]) - - -def test_release_artifacts(db: DbfilenameShelf) -> None: - with tempfile.TemporaryDirectory() as tmppath: - the_dir = pathlib.Path(tmppath) - the_dir.mkdir(exist_ok=True) - filename = f"Python-{db['release']}" - tarball = f"Python-{db['release']}.tgz" - shutil.copy2( - db["git_repo"] / str(db["release"]) / "src" / tarball, - the_dir / tarball, - ) - subprocess.check_call(["tar", "xvf", tarball], cwd=the_dir) - subprocess.check_call( - ["./configure", "--prefix", str(the_dir / "installation")], - cwd=the_dir / filename, - ) - subprocess.check_call(["make", "-j"], cwd=the_dir / filename) - subprocess.check_call(["make", "install", "-j"], cwd=the_dir / filename) - process = subprocess.run( - ["./bin/python3", "-m", "test", "-uall"], - cwd=str(the_dir / "installation"), - text=True, - ) +def wait_for_source_and_docs_artifacts(db: DbfilenameShelf) -> None: + # Determine if we need to wait for docs or only source artifacts. + release_tag = db["release"] + should_wait_for_docs = release_tag.is_final or release_tag.is_release_candiate - if process.returncode == 0: - return + # Create the directory so it's easier to place the artifacts there. + release_path = pathlib.Path(db["git_repo"] / str(db["release"])) + release_path.mkdir(parents=True, exist_ok=True) - if not ask_question("Some test_failed! Do you want to continue?"): - raise ReleaseException("Test failed!") + # Build the list of filepaths we're expecting. + wait_for_paths = [ + release_path / "src" / f"Python-{db['release']}.tgz", + release_path / "src" / f"Python-{db['release']}.tar.xz" + ] + if should_wait_for_docs: + wait_for_paths.extend([ + release_path / "docs" / f"python-{db['release']}-docs.epub", + release_path / "docs" / f"python-{db['release']}-docs-html.tar.bz2", + release_path / "docs" / f"python-{db['release']}-docs-html.zip", + release_path / "docs" / f"python-{db['release']}-docs-pdf-a4.tar.bz2", + release_path / "docs" / f"python-{db['release']}-docs-pdf-a4.zip", + release_path / "docs" / f"python-{db['release']}-docs-pdf-letter.tar.bz2", + release_path / "docs" / f"python-{db['release']}-docs-pdf-letter.zip", + release_path / "docs" / f"python-{db['release']}-docs-texinfo.tar.bz2", + release_path / "docs" / f"python-{db['release']}-docs-texinfo.zip", + release_path / "docs" / f"python-{db['release']}-docs-text.tar.bz2", + release_path / "docs" / f"python-{db['release']}-docs-text.zip", + ]) + + print(f"Waiting for source{' and docs' if should_wait_for_docs else ''} artifacts to be built") + print(f"Artifacts should be placed at '{release_path}':") + for path in wait_for_paths: + print(f"- '{os.path.relpath(path, release_path)}'") + + while not all(path.exists() for path in wait_for_paths): + time.sleep(1) + + +def sign_source_artifacts(db: DbfilenameShelf) -> None: + print('Signing tarballs with GPG') + uid = os.environ.get("GPG_KEY_FOR_RELEASE") + if not uid: + print('List of available private keys:') + subprocess.check_call('gpg -K | grep -A 1 "^sec"', shell=True) + uid = input('Please enter key ID to use for signing: ') + + tarballs_path = pathlib.Path(db["git_repo"] / str(db["release"]) / "src") + tgz = str(tarballs_path / ("Python-%s.tgz" % db["release"])) + xz = str(tarballs_path / ("Python-%s.tar.xz" % db["release"])) + + subprocess.check_call(['gpg', '-bas', '-u', uid, tgz]) + subprocess.check_call(['gpg', '-bas', '-u', uid, xz]) + + print('Signing tarballs with Sigstore') + subprocess.check_call(['python3', '-m', 'sigstore', 'sign', + '--oidc-disable-ambient-providers', tgz, xz]) class MySFTPClient(paramiko.SFTPClient): @@ -664,9 +686,55 @@ def execute_command(command): execute_command(f"find {destination} -type f -exec chmod 664 {{}} \\;") +def start_build_of_source_and_docs(db: DbfilenameShelf) -> None: + # Get the git commit SHA for the tag + commit_sha = subprocess.check_output( + ["git", "rev-list", "-n", "1", db["release"].gitname], + cwd=db["git_repo"] + ).decode().strip() + + # Get the owner of the GitHub repo (first path segment in a 'github.com' remote URL) + # This works for both 'https' and 'ssh' style remote URLs. + origin_remote_url = subprocess.check_output( + ["git", "ls-remote", "--get-url", "origin"], + cwd=db["git_repo"] + ).decode().strip() + match = re.match(r"github\.com/([^/]+)/", origin_remote_url) + if not match: + raise ReleaseException(f"Could not parse GitHub owner from 'origin' remote URL: {origin_remote_url}") + origin_remote_github_owner = match.group(1) + + # We ask for human verification at this point since this commit SHA is 'locked in' + print() + print(f"Go to https://github.com/{origin_remote_github_owner}/cpython/commit/{commit_sha}") + print("- Ensure that there is no warning that the commit does not belong to this repository.") + print("- Ensure that the commit diff does not contain any unexpected changes.") + print("- For the next step, ensure the commit SHA matches the one you verified on GitHub in this step.") + print() + if not ask_question( + "Have you verified the release commit hasn't been tampered with on GitHub?" + ): + raise ReleaseException("Commit must be visually reviewed before starting build") + + # After visually confirming the release manager can start the build process + # with the known good commit SHA. + print() + print("Go to https://github.com/python/release-tools/actions/workflows/source-and-docs-release.yml") + print("Select 'Run workflow' and enter the following values:") + print(f"- Git remote to checkout: {origin_remote_github_owner}") + print(f"- Git commit to target for the release: {commit_sha}") + print(f"- CPython release number: {db['release']}") + print() + + if not ask_question( + "Have you started the source and docs build?" + ): + raise ReleaseException("Source and docs build must be started") + + def send_email_to_platform_release_managers(db: DbfilenameShelf) -> None: if not ask_question( - "Have you notified the platform release managers about the availability of artifacts?" + "Have you notified the platform release managers about the availability of the commit SHA and tag?" ): raise ReleaseException("Platform release managers muy be notified") @@ -1008,7 +1076,6 @@ def _api_key(api_key): auth_key = args.auth_key or os.getenv("AUTH_INFO") tasks = [ Task(check_git, "Checking git is available"), - Task(check_latexmk, "Checking latexmk is available"), Task(check_make, "Checking make is available"), Task(check_blurb, "Checking blurb is available"), Task(check_docker, "Checking docker is available"), @@ -1027,17 +1094,18 @@ def _api_key(api_key): Task(bump_version, "Bump version"), Task(check_cpython_repo_is_clean, "Checking git repository is clean"), Task(create_tag, "Create tag"), - Task(build_release_artifacts, "Building release artifacts"), - Task(test_release_artifacts, "Test release artifacts"), - Task(upload_files_to_server, "Upload files to the PSF server"), - Task(place_files_in_download_folder, "Place files in the download folder"), - Task(upload_docs_to_the_docs_server, "Upload docs to the PSF docs server"), - Task(unpack_docs_in_the_docs_server, "Place docs files in the docs folder"), Task(push_to_local_fork, "Push new tags and branches to private fork"), + Task(start_build_of_source_and_docs, "Start the builds for source and docs artifacts"), Task( send_email_to_platform_release_managers, - "Platform release managers have been notified of the release artifacts", + "Platform release managers have been notified of the commit SHA", ), + Task(wait_for_source_and_docs_artifacts, "Wait for source and docs artifacts to build"), + Task(sign_source_artifacts, "Sign source artifacts"), + Task(upload_files_to_server, "Upload files to the PSF server"), + Task(place_files_in_download_folder, "Place files in the download folder"), + Task(upload_docs_to_the_docs_server, "Upload docs to the PSF docs server"), + Task(unpack_docs_in_the_docs_server, "Place docs files in the docs folder"), Task(wait_util_all_files_are_in_folder, "Wait until all files are ready"), Task(create_release_object_in_db, "The django release object has been created"), Task(post_release_merge, "Merge the tag into the release branch"), From 62bb1894f957ea6a329d9964441a4bb85a1eead1 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Mon, 4 Dec 2023 16:21:51 -0600 Subject: [PATCH 3/5] Use release_tag instead of db["release"] Co-authored-by: Ezio Melotti --- run_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_release.py b/run_release.py index e32d204b..cbd1e621 100755 --- a/run_release.py +++ b/run_release.py @@ -465,7 +465,7 @@ def wait_for_source_and_docs_artifacts(db: DbfilenameShelf) -> None: should_wait_for_docs = release_tag.is_final or release_tag.is_release_candiate # Create the directory so it's easier to place the artifacts there. - release_path = pathlib.Path(db["git_repo"] / str(db["release"])) + release_path = pathlib.Path(db["git_repo"] / str(release_tag)) release_path.mkdir(parents=True, exist_ok=True) # Build the list of filepaths we're expecting. From 7f6ef8d9e1a4aed36ddc5ec3e185c0bb33ee1fef Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Mon, 4 Dec 2023 16:22:01 -0600 Subject: [PATCH 4/5] Use docs_path prefix Co-authored-by: Ezio Melotti --- run_release.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/run_release.py b/run_release.py index cbd1e621..86604d6d 100755 --- a/run_release.py +++ b/run_release.py @@ -470,22 +470,23 @@ def wait_for_source_and_docs_artifacts(db: DbfilenameShelf) -> None: # Build the list of filepaths we're expecting. wait_for_paths = [ - release_path / "src" / f"Python-{db['release']}.tgz", - release_path / "src" / f"Python-{db['release']}.tar.xz" + release_path / "src" / f"Python-{release_tag}.tgz", + release_path / "src" / f"Python-{release_tag}.tar.xz" ] if should_wait_for_docs: + docs_path = release_path / "docs" wait_for_paths.extend([ - release_path / "docs" / f"python-{db['release']}-docs.epub", - release_path / "docs" / f"python-{db['release']}-docs-html.tar.bz2", - release_path / "docs" / f"python-{db['release']}-docs-html.zip", - release_path / "docs" / f"python-{db['release']}-docs-pdf-a4.tar.bz2", - release_path / "docs" / f"python-{db['release']}-docs-pdf-a4.zip", - release_path / "docs" / f"python-{db['release']}-docs-pdf-letter.tar.bz2", - release_path / "docs" / f"python-{db['release']}-docs-pdf-letter.zip", - release_path / "docs" / f"python-{db['release']}-docs-texinfo.tar.bz2", - release_path / "docs" / f"python-{db['release']}-docs-texinfo.zip", - release_path / "docs" / f"python-{db['release']}-docs-text.tar.bz2", - release_path / "docs" / f"python-{db['release']}-docs-text.zip", + docs_path / f"python-{release_tag}-docs.epub", + docs_path / f"python-{release_tag}-docs-html.tar.bz2", + docs_path / f"python-{release_tag}-docs-html.zip", + docs_path / f"python-{release_tag}-docs-pdf-a4.tar.bz2", + docs_path / f"python-{release_tag}-docs-pdf-a4.zip", + docs_path / f"python-{release_tag}-docs-pdf-letter.tar.bz2", + docs_path / f"python-{release_tag}-docs-pdf-letter.zip", + docs_path / f"python-{release_tag}-docs-texinfo.tar.bz2", + docs_path / f"python-{release_tag}-docs-texinfo.zip", + docs_path / f"python-{release_tag}-docs-text.tar.bz2", + docs_path / f"python-{release_tag}-docs-text.zip", ]) print(f"Waiting for source{' and docs' if should_wait_for_docs else ''} artifacts to be built") From ee1bbc0dac18f836ae533030346823f838b244fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 19 Mar 2024 22:34:58 +0100 Subject: [PATCH 5/5] Nit: formatting --- run_release.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run_release.py b/run_release.py index b2b8d1c8..f0d28038 100755 --- a/run_release.py +++ b/run_release.py @@ -1143,7 +1143,8 @@ def _api_key(api_key): Task(place_files_in_download_folder, "Place files in the download folder"), Task(upload_docs_to_the_docs_server, "Upload docs to the PSF docs server"), Task(unpack_docs_in_the_docs_server, "Place docs files in the docs folder"), - Task(wait_util_all_files_are_in_folder, "Wait until all files are ready"), Task(create_release_object_in_db, "The django release object has been created"), + Task(wait_util_all_files_are_in_folder, "Wait until all files are ready"), + Task(create_release_object_in_db, "The django release object has been created"), Task(post_release_merge, "Merge the tag into the release branch"), Task(branch_new_versions, "Branch out new versions and prepare main branch"), Task(post_release_tagging, "Final touches for the release"),