From 81d5e182592a95222a17f499bdb02a4cee7c7667 Mon Sep 17 00:00:00 2001 From: Anand Inguva Date: Thu, 10 Nov 2022 23:34:30 -0500 Subject: [PATCH 1/8] Add platform tag while downloading the wheel --- .../python/apache_beam/runners/portability/stager.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/sdks/python/apache_beam/runners/portability/stager.py b/sdks/python/apache_beam/runners/portability/stager.py index e06c71c917d2..f4ec5c32defb 100644 --- a/sdks/python/apache_beam/runners/portability/stager.py +++ b/sdks/python/apache_beam/runners/portability/stager.py @@ -699,6 +699,8 @@ def _get_platform_for_default_sdk_container(): # Base image pip_version = pkg_resources.get_distribution('pip').version if parse_version(pip_version) >= parse_version('19.3'): + # pip can only recognize manylinux2014_x86_64 wheels + # from version 19.3. return 'manylinux2014_x86_64' else: return 'manylinux2010_x86_64' @@ -830,13 +832,15 @@ def _create_beam_sdk(sdk_remote_location, temp_dir): try: abi_suffix = 'm' if sys.version_info < (3, 8) else '' # Stage binary distribution of the SDK, for now on a best-effort basis. + platform_tag = Stager._get_platform_for_default_sdk_container() sdk_local_file = Stager._download_pypi_sdk_package( temp_dir, fetch_binary=True, language_version_tag='%d%d' % (sys.version_info[0], sys.version_info[1]), abi_tag='cp%d%d%s' % - (sys.version_info[0], sys.version_info[1], abi_suffix)) + (sys.version_info[0], sys.version_info[1], abi_suffix), + platform_tag=platform_tag) sdk_binary_staged_name = Stager.\ _desired_sdk_filename_in_staging_location(sdk_local_file) _LOGGER.info( @@ -873,10 +877,10 @@ def _create_beam_sdk(sdk_remote_location, temp_dir): def _download_pypi_sdk_package( temp_dir, fetch_binary=False, - language_version_tag='27', + language_version_tag='39', language_implementation_tag='cp', - abi_tag='cp27mu', - platform_tag='manylinux1_x86_64'): + abi_tag='cp39', + platform_tag='manylinux2014_x86_64'): """Downloads SDK package from PyPI and returns path to local path.""" package_name = Stager.get_sdk_package_name() try: From b5fd0c3e530a689099ee8f73bf7d765c94ad42f1 Mon Sep 17 00:00:00 2001 From: Anand Inguva Date: Thu, 10 Nov 2022 23:35:20 -0500 Subject: [PATCH 2/8] Update pip version for apache_beam 2.43.0 --- sdks/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index c91fb2e71a85..c4c7f106c39f 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -95,7 +95,7 @@ def get_version(): different technologies and user communities. ''' -RECOMMENDED_MIN_PIP_VERSION = '7.0.0' +RECOMMENDED_MIN_PIP_VERSION = '19.3.0' try: _PIP_VERSION = get_distribution('pip').version if parse_version(_PIP_VERSION) < parse_version(RECOMMENDED_MIN_PIP_VERSION): From 1576d64828f8e11ba992f44a047eda0e3f164287 Mon Sep 17 00:00:00 2001 From: Anand Inguva Date: Fri, 11 Nov 2022 10:19:50 -0500 Subject: [PATCH 3/8] Update platform tag to stage new wheel format --- sdks/python/apache_beam/runners/portability/stager.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/runners/portability/stager.py b/sdks/python/apache_beam/runners/portability/stager.py index f4ec5c32defb..f436f6d0a913 100644 --- a/sdks/python/apache_beam/runners/portability/stager.py +++ b/sdks/python/apache_beam/runners/portability/stager.py @@ -915,7 +915,12 @@ def _download_pypi_sdk_package( '--platform', platform_tag ]) - # Example wheel: apache_beam-2.4.0-cp27-cp27mu-manylinux1_x86_64.whl + # Example wheel: apache_beam-2.42.0-cp39-cp39-manylinux1_x86_64.whl + # Example wheel: with manylinux14 tag. + # apache_beam-2.43.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_ + # x86_64.whl + if platform_tag == 'manylinux2014_x86_64': + platform_tag = 'manylinux_2_17_x86_64.' + platform_tag expected_files = [ os.path.join( temp_dir, @@ -925,8 +930,9 @@ def _download_pypi_sdk_package( language_implementation_tag, language_version_tag, abi_tag, - platform_tag)) + platform_tag)), ] + else: _LOGGER.info('Downloading source distribution of the SDK from PyPi') cmd_args.extend(['--no-binary', ':all:']) @@ -942,6 +948,7 @@ def _download_pypi_sdk_package( raise RuntimeError(repr(e)) for sdk_file in expected_files: + logging.info('Looking for %s' % sdk_file) if os.path.exists(sdk_file): return sdk_file From ca3c338f7828a2981a8bfac239d901b221297978 Mon Sep 17 00:00:00 2001 From: Anand Inguva Date: Fri, 11 Nov 2022 11:16:04 -0500 Subject: [PATCH 4/8] Fix test --- sdks/python/apache_beam/runners/portability/stager_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/python/apache_beam/runners/portability/stager_test.py b/sdks/python/apache_beam/runners/portability/stager_test.py index b221bb1ec6f6..84b6c5ddbb6f 100644 --- a/sdks/python/apache_beam/runners/portability/stager_test.py +++ b/sdks/python/apache_beam/runners/portability/stager_test.py @@ -137,6 +137,8 @@ def pip_fake(args): # Per PEP-0427 in wheel filenames non-alphanumeric characters # in distribution name are replaced with underscore. distribution_name = distribution_name.replace('-', '_') + if args[17] == 'manylinux2014_x86_64': + args[17] = 'manylinux_2_17_x86_64.' + args[17] package_file = '%s-%s-%s%s-%s-%s.whl' % ( distribution_name, distribution_version, From 336f407f4083fbb77b55b08d3f434544b1592c54 Mon Sep 17 00:00:00 2001 From: Anand Inguva Date: Fri, 11 Nov 2022 12:07:15 -0500 Subject: [PATCH 5/8] Change wheel name in boot.go --- sdks/python/container/boot.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/container/boot.go b/sdks/python/container/boot.go index 8e4cf772c0e0..8c27f3e5aaa9 100644 --- a/sdks/python/container/boot.go +++ b/sdks/python/container/boot.go @@ -233,9 +233,9 @@ func setupAcceptableWheelSpecs() error { var wheelName string switch pyVersion { case "36", "37": - wheelName = fmt.Sprintf("cp%s-cp%sm-manylinux1_x86_64.whl", pyVersion, pyVersion) + wheelName = fmt.Sprintf("cp%s-cp%sm-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", pyVersion, pyVersion) default: - wheelName = fmt.Sprintf("cp%s-cp%s-manylinux1_x86_64.whl", pyVersion, pyVersion) + wheelName = fmt.Sprintf("cp%s-cp%s-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", pyVersion, pyVersion) } acceptableWhlSpecs = append(acceptableWhlSpecs, wheelName) return nil From 46e0e093586f68693386fec932edb9a58b50f1c6 Mon Sep 17 00:00:00 2001 From: Anand Inguva Date: Fri, 11 Nov 2022 13:05:42 -0500 Subject: [PATCH 6/8] Regex to support 3.10 --- sdks/python/container/boot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/container/boot.go b/sdks/python/container/boot.go index 8c27f3e5aaa9..47b9ea186c93 100644 --- a/sdks/python/container/boot.go +++ b/sdks/python/container/boot.go @@ -224,7 +224,7 @@ func setupAcceptableWheelSpecs() error { if err != nil { return err } - re := regexp.MustCompile(`Python (\d)\.(\d).*`) + re := regexp.MustCompile(`Python (\d)\.(\d+).*`) pyVersions := re.FindStringSubmatch(string(stdoutStderr[:])) if len(pyVersions) != 3 { return fmt.Errorf("cannot get parse Python version from %s", stdoutStderr) From ec0472a59720e02a20272468429c1d2f654d8e02 Mon Sep 17 00:00:00 2001 From: Anand Inguva Date: Fri, 11 Nov 2022 13:06:58 -0500 Subject: [PATCH 7/8] remove logging --- sdks/python/apache_beam/runners/portability/stager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/apache_beam/runners/portability/stager.py b/sdks/python/apache_beam/runners/portability/stager.py index f436f6d0a913..cc9033dba98a 100644 --- a/sdks/python/apache_beam/runners/portability/stager.py +++ b/sdks/python/apache_beam/runners/portability/stager.py @@ -948,7 +948,6 @@ def _download_pypi_sdk_package( raise RuntimeError(repr(e)) for sdk_file in expected_files: - logging.info('Looking for %s' % sdk_file) if os.path.exists(sdk_file): return sdk_file From 37fb3d166fa933449d9bbde65a2cd0b9d1f7ae66 Mon Sep 17 00:00:00 2001 From: Anand Inguva Date: Fri, 11 Nov 2022 14:03:39 -0500 Subject: [PATCH 8/8] Update comment --- sdks/python/apache_beam/runners/portability/stager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/runners/portability/stager.py b/sdks/python/apache_beam/runners/portability/stager.py index cc9033dba98a..fbcf4d5a2e59 100644 --- a/sdks/python/apache_beam/runners/portability/stager.py +++ b/sdks/python/apache_beam/runners/portability/stager.py @@ -915,10 +915,8 @@ def _download_pypi_sdk_package( '--platform', platform_tag ]) - # Example wheel: apache_beam-2.42.0-cp39-cp39-manylinux1_x86_64.whl # Example wheel: with manylinux14 tag. - # apache_beam-2.43.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_ - # x86_64.whl + # apache_beam-2.43.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl # pylint: disable=line-too-long if platform_tag == 'manylinux2014_x86_64': platform_tag = 'manylinux_2_17_x86_64.' + platform_tag expected_files = [