diff --git a/.github/workflows/source-and-docs-release.yml b/.github/workflows/source-and-docs-release.yml new file mode 100644 index 00000000..9e771ac3 --- /dev/null +++ b/.github/workflows/source-and-docs-release.yml @@ -0,0 +1,100 @@ +on: + workflow_dispatch: + inputs: + git_remote: + type: choice + description: "Git remote to checkout" + options: + - python + - Yhg1s + - pablogsal + - ambv + git_commit: + type: string + description: "Git commit to target for the release. Must use the full commit SHA, not the short ID" + cpython_release: + type: string + description: "CPython release number (ie '3.11.5', note without the 'v' prefix)" + +name: "Build Python source and docs artifacts" + +jobs: + source-and-docs: + runs-on: ubuntu-22.04 + steps: + - name: "Checkout python/release-tools" + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: "Checkout ${{ inputs.git_remote }}/cpython" + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: "${{ inputs.git_remote }}/cpython" + ref: "v${{ inputs.cpython_release }}" + path: "cpython" + + - name: "Verify CPython commit matches tag" + run: | + if [[ "${{ inputs.git_commit }}" != "$(cd cpython && git rev-parse HEAD)" ]]; then + echo "expected git commit ('${{ inputs.git_commit }}') didn't match tagged commit ('$(git rev-parse HEAD)')" + exit 1 + fi + + - name: "Setup Python" + uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 + with: + python-version: 3.11 + + - name: "Install source dependencies" + run: | + python -m pip install --no-deps \ + -r requirements.txt + + - name: "Install docs dependencies" + # Docs aren't built for alpha or beta releases. + if: ${{ !(contains(inputs.cpython_release, 'a') || contains(inputs.cpython_release, 'b')) }} + run: | + python -m pip install \ + -r cpython/Doc/requirements.txt + + sudo apt-get update + sudo apt-get install --yes --no-install-recommends \ + latexmk texlive-xetex xindy texinfo texlive-latex-base \ + texlive-fonts-recommended texlive-fonts-extra \ + texlive-full + + - name: "Build Python release artifacts" + run: | + cd cpython + python ../release.py --export ${{ inputs.cpython_release }} + + - name: "Test Python source tarballs" + run: | + mkdir -p ./tmp/installation/ + cp cpython/${{ inputs.cpython_release }}/src/Python-${{ inputs.cpython_release }}.tgz ./tmp/ + cd tmp/ + tar xvf Python-${{ inputs.cpython_release }}.tgz + cd Python-${{ inputs.cpython_release }} + + ./configure --prefix=$(realpath '../installation/') + make -j + make install -j + + cd ../installation + ./bin/python3 -m test -uall + + - name: "Upload the source artifacts" + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: source + path: | + cpython/${{ inputs.cpython_release }}/src + + - name: "Upload the docs artifacts" + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + # Conditionally run this step if there is a 'docs/' directory. + # Docs aren't built for alpha or beta releases. + if: ${{ hashFiles('cpython/${{ inputs.cpython_release }}/docs') != '' }} + with: + name: docs + path: | + cpython/${{ inputs.cpython_release }}/docs diff --git a/release.py b/release.py index 8e289f3e..ac78dc7c 100755 --- a/release.py +++ b/release.py @@ -277,19 +277,6 @@ def tarball(source, clamp_mtime): print(' %s %8s %s' % ( checksum_xz.hexdigest(), int(os.path.getsize(xz)), xz)) - print('Signing tarballs with GPG') - uid = os.environ.get("GPG_KEY_FOR_RELEASE") - if not uid: - print('List of available private keys:') - run_cmd(['gpg -K | grep -A 1 "^sec"'], shell=True) - uid = input('Please enter key ID to use for signing: ') - run_cmd(['gpg', '-bas', '-u', uid, tgz]) - run_cmd(['gpg', '-bas', '-u', uid, xz]) - - print('Signing tarballs with Sigstore') - run_cmd(['python3', '-m', 'sigstore', 'sign', - '--oidc-disable-ambient-providers', tgz, xz], shell=False) - def export(tag, silent=False): make_dist(tag.text) diff --git a/run_release.py b/run_release.py index c6cc3e5b..f0d28038 100755 --- a/run_release.py +++ b/run_release.py @@ -297,7 +297,6 @@ def check_tool(db: DbfilenameShelf, tool: str) -> None: check_git = functools.partial(check_tool, tool="git") -check_latexmk = functools.partial(check_tool, tool="latexmk") check_make = functools.partial(check_tool, tool="make") check_blurb = functools.partial(check_tool, tool="blurb") check_autoconf = functools.partial(check_tool, tool="autoconf") @@ -474,39 +473,63 @@ def create_tag(db: DbfilenameShelf) -> None: ) -def build_release_artifacts(db: DbfilenameShelf) -> None: - with cd(db["git_repo"]): - release_mod.export(db["release"]) - - -def test_release_artifacts(db: DbfilenameShelf) -> None: - with tempfile.TemporaryDirectory() as tmppath: - the_dir = pathlib.Path(tmppath) - the_dir.mkdir(exist_ok=True) - filename = f"Python-{db['release']}" - tarball = f"Python-{db['release']}.tgz" - shutil.copy2( - db["git_repo"] / str(db["release"]) / "src" / tarball, - the_dir / tarball, - ) - subprocess.check_call(["tar", "xvf", tarball], cwd=the_dir) - subprocess.check_call( - ["./configure", "--prefix", str(the_dir / "installation")], - cwd=the_dir / filename, - ) - subprocess.check_call(["make", "-j"], cwd=the_dir / filename) - subprocess.check_call(["make", "install", "-j"], cwd=the_dir / filename) - process = subprocess.run( - ["./bin/python3", "-m", "test", "-uall"], - cwd=str(the_dir / "installation"), - text=True, - ) +def wait_for_source_and_docs_artifacts(db: DbfilenameShelf) -> None: + # Determine if we need to wait for docs or only source artifacts. + release_tag = db["release"] + should_wait_for_docs = release_tag.is_final or release_tag.is_release_candiate - if process.returncode == 0: - return + # Create the directory so it's easier to place the artifacts there. + release_path = pathlib.Path(db["git_repo"] / str(release_tag)) + release_path.mkdir(parents=True, exist_ok=True) - if not ask_question("Some test_failed! Do you want to continue?"): - raise ReleaseException("Test failed!") + # Build the list of filepaths we're expecting. + wait_for_paths = [ + release_path / "src" / f"Python-{release_tag}.tgz", + release_path / "src" / f"Python-{release_tag}.tar.xz" + ] + if should_wait_for_docs: + docs_path = release_path / "docs" + wait_for_paths.extend([ + docs_path / f"python-{release_tag}-docs.epub", + docs_path / f"python-{release_tag}-docs-html.tar.bz2", + docs_path / f"python-{release_tag}-docs-html.zip", + docs_path / f"python-{release_tag}-docs-pdf-a4.tar.bz2", + docs_path / f"python-{release_tag}-docs-pdf-a4.zip", + docs_path / f"python-{release_tag}-docs-pdf-letter.tar.bz2", + docs_path / f"python-{release_tag}-docs-pdf-letter.zip", + docs_path / f"python-{release_tag}-docs-texinfo.tar.bz2", + docs_path / f"python-{release_tag}-docs-texinfo.zip", + docs_path / f"python-{release_tag}-docs-text.tar.bz2", + docs_path / f"python-{release_tag}-docs-text.zip", + ]) + + print(f"Waiting for source{' and docs' if should_wait_for_docs else ''} artifacts to be built") + print(f"Artifacts should be placed at '{release_path}':") + for path in wait_for_paths: + print(f"- '{os.path.relpath(path, release_path)}'") + + while not all(path.exists() for path in wait_for_paths): + time.sleep(1) + + +def sign_source_artifacts(db: DbfilenameShelf) -> None: + print('Signing tarballs with GPG') + uid = os.environ.get("GPG_KEY_FOR_RELEASE") + if not uid: + print('List of available private keys:') + subprocess.check_call('gpg -K | grep -A 1 "^sec"', shell=True) + uid = input('Please enter key ID to use for signing: ') + + tarballs_path = pathlib.Path(db["git_repo"] / str(db["release"]) / "src") + tgz = str(tarballs_path / ("Python-%s.tgz" % db["release"])) + xz = str(tarballs_path / ("Python-%s.tar.xz" % db["release"])) + + subprocess.check_call(['gpg', '-bas', '-u', uid, tgz]) + subprocess.check_call(['gpg', '-bas', '-u', uid, xz]) + + print('Signing tarballs with Sigstore') + subprocess.check_call(['python3', '-m', 'sigstore', 'sign', + '--oidc-disable-ambient-providers', tgz, xz]) def build_sbom_artifacts(db): @@ -698,9 +721,55 @@ def execute_command(command): execute_command(f"find {destination} -type f -exec chmod 664 {{}} \\;") +def start_build_of_source_and_docs(db: DbfilenameShelf) -> None: + # Get the git commit SHA for the tag + commit_sha = subprocess.check_output( + ["git", "rev-list", "-n", "1", db["release"].gitname], + cwd=db["git_repo"] + ).decode().strip() + + # Get the owner of the GitHub repo (first path segment in a 'github.com' remote URL) + # This works for both 'https' and 'ssh' style remote URLs. + origin_remote_url = subprocess.check_output( + ["git", "ls-remote", "--get-url", "origin"], + cwd=db["git_repo"] + ).decode().strip() + match = re.match(r"github\.com/([^/]+)/", origin_remote_url) + if not match: + raise ReleaseException(f"Could not parse GitHub owner from 'origin' remote URL: {origin_remote_url}") + origin_remote_github_owner = match.group(1) + + # We ask for human verification at this point since this commit SHA is 'locked in' + print() + print(f"Go to https://github.com/{origin_remote_github_owner}/cpython/commit/{commit_sha}") + print("- Ensure that there is no warning that the commit does not belong to this repository.") + print("- Ensure that the commit diff does not contain any unexpected changes.") + print("- For the next step, ensure the commit SHA matches the one you verified on GitHub in this step.") + print() + if not ask_question( + "Have you verified the release commit hasn't been tampered with on GitHub?" + ): + raise ReleaseException("Commit must be visually reviewed before starting build") + + # After visually confirming the release manager can start the build process + # with the known good commit SHA. + print() + print("Go to https://github.com/python/release-tools/actions/workflows/source-and-docs-release.yml") + print("Select 'Run workflow' and enter the following values:") + print(f"- Git remote to checkout: {origin_remote_github_owner}") + print(f"- Git commit to target for the release: {commit_sha}") + print(f"- CPython release number: {db['release']}") + print() + + if not ask_question( + "Have you started the source and docs build?" + ): + raise ReleaseException("Source and docs build must be started") + + def send_email_to_platform_release_managers(db: DbfilenameShelf) -> None: if not ask_question( - "Have you notified the platform release managers about the availability of artifacts?" + "Have you notified the platform release managers about the availability of the commit SHA and tag?" ): raise ReleaseException("Platform release managers muy be notified") @@ -1040,9 +1109,9 @@ def _api_key(api_key): ) args = parser.parse_args() auth_key = args.auth_key or os.getenv("AUTH_INFO") + assert isinstance(auth_key, str), "We need an AUTH_INFO env var or --auth-key" tasks = [ Task(check_git, "Checking git is available"), - Task(check_latexmk, "Checking latexmk is available"), Task(check_make, "Checking make is available"), Task(check_blurb, "Checking blurb is available"), Task(check_docker, "Checking docker is available"), @@ -1061,18 +1130,19 @@ def _api_key(api_key): Task(bump_version, "Bump version"), Task(check_cpython_repo_is_clean, "Checking git repository is clean"), Task(create_tag, "Create tag"), - Task(build_release_artifacts, "Building release artifacts"), - Task(test_release_artifacts, "Test release artifacts"), + Task(push_to_local_fork, "Push new tags and branches to private fork"), + Task(start_build_of_source_and_docs, "Start the builds for source and docs artifacts"), + Task( + send_email_to_platform_release_managers, + "Platform release managers have been notified of the commit SHA", + ), + Task(wait_for_source_and_docs_artifacts, "Wait for source and docs artifacts to build"), Task(build_sbom_artifacts, "Building SBOM artifacts"), + Task(sign_source_artifacts, "Sign source artifacts"), Task(upload_files_to_server, "Upload files to the PSF server"), Task(place_files_in_download_folder, "Place files in the download folder"), Task(upload_docs_to_the_docs_server, "Upload docs to the PSF docs server"), Task(unpack_docs_in_the_docs_server, "Place docs files in the docs folder"), - Task(push_to_local_fork, "Push new tags and branches to private fork"), - Task( - send_email_to_platform_release_managers, - "Platform release managers have been notified of the release artifacts", - ), Task(wait_util_all_files_are_in_folder, "Wait until all files are ready"), Task(create_release_object_in_db, "The django release object has been created"), Task(post_release_merge, "Merge the tag into the release branch"),