Skip to content

[release-2.43.0] Update staging of Python wheels #24124

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
AnandInguva wants to merge 8 commits into from
Closed

[release-2.43.0] Update staging of Python wheels #24124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
81d5e18
Add platform tag while downloading the wheel
AnandInguva Nov 11, 2022
b5fd0c3
Update pip version for apache_beam 2.43.0
AnandInguva Nov 11, 2022
1576d64
Update platform tag to stage new wheel format
AnandInguva Nov 11, 2022
ca3c338
Fix test
AnandInguva Nov 11, 2022
336f407
Change wheel name in boot.go
AnandInguva Nov 11, 2022
46e0e09
Regex to support 3.10
AnandInguva Nov 11, 2022
ec0472a
remove logging
AnandInguva Nov 11, 2022
37fb3d1
Update comment
AnandInguva Nov 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 14 additions & 6 deletions sdks/python/apache_beam/runners/portability/stager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -699,6 +699,8 @@ def _get_platform_for_default_sdk_container():
# Base image
pip_version = pkg_resources.get_distribution('pip').version
if parse_version(pip_version) >= parse_version('19.3'):
# pip can only recognize manylinux2014_x86_64 wheels
# from version 19.3.
return 'manylinux2014_x86_64'
else:
return 'manylinux2010_x86_64'
Expand Down Expand Up @@ -830,13 +832,15 @@ def _create_beam_sdk(sdk_remote_location, temp_dir):
try:
abi_suffix = 'm' if sys.version_info < (3, 8) else ''
# Stage binary distribution of the SDK, for now on a best-effort basis.
platform_tag = Stager._get_platform_for_default_sdk_container()
sdk_local_file = Stager._download_pypi_sdk_package(
temp_dir,
fetch_binary=True,
language_version_tag='%d%d' %
(sys.version_info[0], sys.version_info[1]),
abi_tag='cp%d%d%s' %
(sys.version_info[0], sys.version_info[1], abi_suffix))
(sys.version_info[0], sys.version_info[1], abi_suffix),
platform_tag=platform_tag)
sdk_binary_staged_name = Stager.\
_desired_sdk_filename_in_staging_location(sdk_local_file)
_LOGGER.info(
Expand Down Expand Up @@ -873,10 +877,10 @@ def _create_beam_sdk(sdk_remote_location, temp_dir):
def _download_pypi_sdk_package(
temp_dir,
fetch_binary=False,
language_version_tag='27',
language_version_tag='39',
language_implementation_tag='cp',
abi_tag='cp27mu',
platform_tag='manylinux1_x86_64'):
abi_tag='cp39',
platform_tag='manylinux2014_x86_64'):
"""Downloads SDK package from PyPI and returns path to local path."""
package_name = Stager.get_sdk_package_name()
try:
Expand Down Expand Up @@ -911,7 +915,10 @@ def _download_pypi_sdk_package(
'--platform',
platform_tag
])
# Example wheel: apache_beam-2.4.0-cp27-cp27mu-manylinux1_x86_64.whl
# Example wheel: with manylinux14 tag.
# apache_beam-2.43.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl # pylint: disable=line-too-long
if platform_tag == 'manylinux2014_x86_64':
platform_tag = 'manylinux_2_17_x86_64.' + platform_tag
expected_files = [
os.path.join(
temp_dir,
Expand All @@ -921,8 +928,9 @@ def _download_pypi_sdk_package(
language_implementation_tag,
language_version_tag,
abi_tag,
platform_tag))
platform_tag)),
]

else:
_LOGGER.info('Downloading source distribution of the SDK from PyPi')
cmd_args.extend(['--no-binary', ':all:'])
Expand Down
2 changes: 2 additions & 0 deletions sdks/python/apache_beam/runners/portability/stager_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,8 @@ def pip_fake(args):
# Per PEP-0427 in wheel filenames non-alphanumeric characters
# in distribution name are replaced with underscore.
distribution_name = distribution_name.replace('-', '_')
if args[17] == 'manylinux2014_x86_64':
args[17] = 'manylinux_2_17_x86_64.' + args[17]
package_file = '%s-%s-%s%s-%s-%s.whl' % (
distribution_name,
distribution_version,
Expand Down
6 changes: 3 additions & 3 deletions sdks/python/container/boot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ func setupAcceptableWheelSpecs() error {
if err != nil {
return err
}
re := regexp.MustCompile(`Python (\d)\.(\d).*`)
re := regexp.MustCompile(`Python (\d)\.(\d+).*`)
pyVersions := re.FindStringSubmatch(string(stdoutStderr[:]))
if len(pyVersions) != 3 {
return fmt.Errorf("cannot get parse Python version from %s", stdoutStderr)
Expand All @@ -233,9 +233,9 @@ func setupAcceptableWheelSpecs() error {
var wheelName string
switch pyVersion {
case "36", "37":
wheelName = fmt.Sprintf("cp%s-cp%sm-manylinux1_x86_64.whl", pyVersion, pyVersion)
wheelName = fmt.Sprintf("cp%s-cp%sm-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", pyVersion, pyVersion)
default:
wheelName = fmt.Sprintf("cp%s-cp%s-manylinux1_x86_64.whl", pyVersion, pyVersion)
wheelName = fmt.Sprintf("cp%s-cp%s-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", pyVersion, pyVersion)
}
acceptableWhlSpecs = append(acceptableWhlSpecs, wheelName)
return nil
Expand Down
2 changes: 1 addition & 1 deletion sdks/python/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ def get_version():
different technologies and user communities.
'''

RECOMMENDED_MIN_PIP_VERSION = '7.0.0'
RECOMMENDED_MIN_PIP_VERSION = '19.3.0'
try:
_PIP_VERSION = get_distribution('pip').version
if parse_version(_PIP_VERSION) < parse_version(RECOMMENDED_MIN_PIP_VERSION):
Expand Down