From bb5a7c4c3ece0ae7788d2f7944ec6d0b7aaaf4a2 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Sun, 29 May 2022 00:11:32 +1000 Subject: [PATCH 1/3] Replace manual unzip with installer. --- examples/pip_install/pip_install_test.py | 24 +-- examples/pip_parse/pip_parse_test.py | 22 +- examples/pip_repository_annotations/WORKSPACE | 2 +- .../pip_repository_annotations_test.py | 2 +- python/pip_install/extract_wheels/lib/BUILD | 14 +- .../pip_install/extract_wheels/lib/bazel.py | 65 +++--- .../pip_install/extract_wheels/lib/purelib.py | 67 ------- .../extract_wheels/lib/purelib_test.py | 40 ---- .../pip_install/extract_wheels/lib/wheel.py | 189 +++++------------- python/pip_install/private/srcs.bzl | 1 - python/pip_install/repositories.bzl | 10 +- python/repositories.bzl | 1 + python/versions.bzl | 8 +- 13 files changed, 111 insertions(+), 334 deletions(-) delete mode 100644 python/pip_install/extract_wheels/lib/purelib.py delete mode 100644 python/pip_install/extract_wheels/lib/purelib_test.py diff --git a/examples/pip_install/pip_install_test.py b/examples/pip_install/pip_install_test.py index f9a62ca6e8..eb4d7d8e95 100644 --- a/examples/pip_install/pip_install_test.py +++ b/examples/pip_install/pip_install_test.py @@ -37,12 +37,11 @@ def test_data(self): self.assertListEqual( env.split(" "), [ - "external/pip/pypi__s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/INSTALL.md", - "external/pip/pypi__s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/LICENSE", - "external/pip/pypi__s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/NEWS", - "external/pip/pypi__s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/README.md", - "external/pip/pypi__s3cmd/s3cmd-2.1.0.data/data/share/man/man1/s3cmd.1", - "external/pip/pypi__s3cmd/s3cmd-2.1.0.data/scripts/s3cmd", + "external/pip/pypi__s3cmd/data/share/doc/packages/s3cmd/INSTALL.md", + "external/pip/pypi__s3cmd/data/share/doc/packages/s3cmd/LICENSE", + "external/pip/pypi__s3cmd/data/share/doc/packages/s3cmd/NEWS", + "external/pip/pypi__s3cmd/data/share/doc/packages/s3cmd/README.md", + "external/pip/pypi__s3cmd/data/share/man/man1/s3cmd.1", ], ) @@ -52,12 +51,13 @@ def test_dist_info(self): self.assertListEqual( env.split(" "), [ - "external/pip/pypi__boto3/boto3-1.14.51.dist-info/DESCRIPTION.rst", - "external/pip/pypi__boto3/boto3-1.14.51.dist-info/METADATA", - "external/pip/pypi__boto3/boto3-1.14.51.dist-info/RECORD", - "external/pip/pypi__boto3/boto3-1.14.51.dist-info/WHEEL", - "external/pip/pypi__boto3/boto3-1.14.51.dist-info/metadata.json", - "external/pip/pypi__boto3/boto3-1.14.51.dist-info/top_level.txt", + "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/DESCRIPTION.rst", + 'external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/INSTALLER', + "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/METADATA", + "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/RECORD", + "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/WHEEL", + "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/metadata.json", + "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/top_level.txt", ], ) diff --git a/examples/pip_parse/pip_parse_test.py b/examples/pip_parse/pip_parse_test.py index ef684c4294..030e38c077 100644 --- a/examples/pip_parse/pip_parse_test.py +++ b/examples/pip_parse/pip_parse_test.py @@ -35,12 +35,11 @@ def test_data(self): self.assertListEqual( env.split(" "), [ - "external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/INSTALL.md", - "external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/LICENSE", - "external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/NEWS", - "external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/README.md", - "external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/man/man1/s3cmd.1", - "external/pypi_s3cmd/s3cmd-2.1.0.data/scripts/s3cmd", + "external/pypi_s3cmd/data/share/doc/packages/s3cmd/INSTALL.md", + "external/pypi_s3cmd/data/share/doc/packages/s3cmd/LICENSE", + "external/pypi_s3cmd/data/share/doc/packages/s3cmd/NEWS", + "external/pypi_s3cmd/data/share/doc/packages/s3cmd/README.md", + "external/pypi_s3cmd/data/share/man/man1/s3cmd.1", ], ) @@ -50,11 +49,12 @@ def test_dist_info(self): self.assertListEqual( env.split(" "), [ - "external/pypi_requests/requests-2.25.1.dist-info/LICENSE", - "external/pypi_requests/requests-2.25.1.dist-info/METADATA", - "external/pypi_requests/requests-2.25.1.dist-info/RECORD", - "external/pypi_requests/requests-2.25.1.dist-info/WHEEL", - "external/pypi_requests/requests-2.25.1.dist-info/top_level.txt", + 'external/pypi_requests/site-packages/requests-2.25.1.dist-info/INSTALLER', + "external/pypi_requests/site-packages/requests-2.25.1.dist-info/LICENSE", + "external/pypi_requests/site-packages/requests-2.25.1.dist-info/METADATA", + "external/pypi_requests/site-packages/requests-2.25.1.dist-info/RECORD", + "external/pypi_requests/site-packages/requests-2.25.1.dist-info/WHEEL", + "external/pypi_requests/site-packages/requests-2.25.1.dist-info/top_level.txt", ], ) diff --git a/examples/pip_repository_annotations/WORKSPACE b/examples/pip_repository_annotations/WORKSPACE index eb712cfc6b..8ee885d468 100644 --- a/examples/pip_repository_annotations/WORKSPACE +++ b/examples/pip_repository_annotations/WORKSPACE @@ -42,7 +42,7 @@ write_file( copy_executables = {"@pip_repository_annotations_example//:data/copy_executable.py": "copied_content/executable.py"}, copy_files = {"@pip_repository_annotations_example//:data/copy_file.txt": "copied_content/file.txt"}, data = [":generated_file"], - data_exclude_glob = ["*.dist-info/WHEEL"], + data_exclude_glob = ["site-packages/*.dist-info/WHEEL"], ), } diff --git a/examples/pip_repository_annotations/pip_repository_annotations_test.py b/examples/pip_repository_annotations/pip_repository_annotations_test.py index 79c354d105..468788f50b 100644 --- a/examples/pip_repository_annotations/pip_repository_annotations_test.py +++ b/examples/pip_repository_annotations/pip_repository_annotations_test.py @@ -69,7 +69,7 @@ def test_data_exclude_glob(self): r = runfiles.Create() dist_info_dir = ( - "pip_repository_annotations_example/external/{}/wheel-{}.dist-info".format( + "pip_repository_annotations_example/external/{}/site-packages/wheel-{}.dist-info".format( self.wheel_pkg_dir(), current_wheel_version, ) diff --git a/python/pip_install/extract_wheels/lib/BUILD b/python/pip_install/extract_wheels/lib/BUILD index 3e2f307a73..31d6bb8918 100644 --- a/python/pip_install/extract_wheels/lib/BUILD +++ b/python/pip_install/extract_wheels/lib/BUILD @@ -9,7 +9,6 @@ py_library( "arguments.py", "bazel.py", "namespace_pkgs.py", - "purelib.py", "requirements.py", "wheel.py", ], @@ -18,7 +17,7 @@ py_library( "//python/pip_install/parse_requirements_to_bzl:__subpackages__", ], deps = [ - requirement("pkginfo"), + requirement("installer"), requirement("setuptools"), ], ) @@ -135,17 +134,6 @@ py_test( ], ) -py_test( - name = "purelib_test", - size = "small", - srcs = [ - "purelib_test.py", - ], - deps = [ - ":lib", - ], -) - filegroup( name = "distribution", srcs = glob( diff --git a/python/pip_install/extract_wheels/lib/bazel.py b/python/pip_install/extract_wheels/lib/bazel.py index ecb91fc828..7bff17e95f 100644 --- a/python/pip_install/extract_wheels/lib/bazel.py +++ b/python/pip_install/extract_wheels/lib/bazel.py @@ -9,7 +9,6 @@ from python.pip_install.extract_wheels.lib import ( annotation, namespace_pkgs, - purelib, wheel, ) @@ -21,34 +20,33 @@ def generate_entry_point_contents( - entry_point: str, shebang: str = "#!/usr/bin/env python3" + module: str, attribute: str, shebang: str = "#!/usr/bin/env python3" ) -> str: """Generate the contents of an entry point script. Args: - entry_point (str): The name of the entry point as show in the - `console_scripts` section of `entry_point.txt`. + module (str): The name of the module to use. + attribute (str): The name of the attribute to call. shebang (str, optional): The shebang to use for the entry point python file. Returns: str: A string of python code. """ - module, method = entry_point.split(":", 1) return textwrap.dedent( """\ {shebang} import sys - from {module} import {method} + from {module} import {attribute} if __name__ == "__main__": - sys.exit({method}()) + sys.exit({attribute}()) """.format( - shebang=shebang, module=module, method=method + shebang=shebang, module=module, attribute=attribute ) ) -def generate_entry_point_rule(script: str, pkg: str) -> str: +def generate_entry_point_rule(name: str, script: str, pkg: str) -> str: """Generate a Bazel `py_binary` rule for an entry point script. Note that the script is used to determine the name of the target. The name of @@ -56,6 +54,7 @@ def generate_entry_point_rule(script: str, pkg: str) -> str: directories within a wheel. Args: + name (str): The name of the generated py_binary. script (str): The path to the entry point's python file. pkg (str): The package owning the entry point. This is expected to match up with the `py_library` defined for each repository. @@ -64,7 +63,6 @@ def generate_entry_point_rule(script: str, pkg: str) -> str: Returns: str: A `py_binary` instantiation. """ - name = os.path.splitext(script)[0] return textwrap.dedent( """\ py_binary( @@ -138,27 +136,18 @@ def generate_build_file_contents( there may be no Python sources whatsoever (e.g. packages written in Cython: like `pymssql`). """ - dist_info_ignores = [ - # RECORD is known to contain sha256 checksums of files which might include the checksums - # of generated files produced when wheels are installed. The file is ignored to avoid - # Bazel caching issues. - "**/*.dist-info/RECORD", - ] - data_exclude = list( set( [ - "*.whl", - "**/__pycache__/**", "**/* *", "**/*.py", "**/*.pyc", - "BUILD.bazel", - "WORKSPACE", - f"{WHEEL_ENTRY_POINT_PREFIX}*.py", + # RECORD is known to contain sha256 checksums of files which might include the checksums + # of generated files produced when wheels are installed. The file is ignored to avoid + # Bazel caching issues. + "**/*.dist-info/RECORD", ] + data_exclude - + dist_info_ignores ) ) @@ -173,12 +162,12 @@ def generate_build_file_contents( filegroup( name = "{dist_info_label}", - srcs = glob(["*.dist-info/**"], allow_empty = True), + srcs = glob(["site-packages/*.dist-info/**"], allow_empty = True), ) filegroup( name = "{data_label}", - srcs = glob(["*.data/**"], allow_empty = True), + srcs = glob(["data/**"], allow_empty = True), ) filegroup( @@ -189,11 +178,11 @@ def generate_build_file_contents( py_library( name = "{name}", - srcs = glob(["**/*.py"], exclude={srcs_exclude}, allow_empty = True), - data = {data} + glob(["**/*"], exclude={data_exclude}), + srcs = glob(["site-packages/**/*.py"], exclude={srcs_exclude}, allow_empty = True), + data = {data} + glob(["site-packages/**/*"], exclude={data_exclude}), # This makes this directory a top-level in the python import # search path for anything that depends on this. - imports = ["."], + imports = ["site-packages"], deps = [{dependencies}], tags = [{tags}], ) @@ -378,9 +367,6 @@ def extract_wheel( shutil.copy(whl.path, directory) whl.unzip(directory) - # Note: Order of operations matters here - purelib.spread_purelib_into_root(directory) - if not enable_implicit_namespace_pkgs: setup_namespace_pkg_compatibility(directory) @@ -408,14 +394,19 @@ def extract_wheel( directory_path = Path(directory) entry_points = [] - for name, entry_point in sorted(whl.entry_points().items()): - entry_point_script = f"{WHEEL_ENTRY_POINT_PREFIX}_{name}.py" - (directory_path / entry_point_script).write_text( - generate_entry_point_contents(entry_point) + for name, (module, attribute) in sorted(whl.entry_points().items()): + # There is an extreme edge-case with entry_points that end with `.py` + # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 + entry_point_without_py = name[:-3] if name.endswith(".py") else name + entry_point_target_name = f"{WHEEL_ENTRY_POINT_PREFIX}_{entry_point_without_py}" + entry_point_script_name = f"{entry_point_target_name}.py" + (directory_path / entry_point_script_name).write_text( + generate_entry_point_contents(module, attribute) ) entry_points.append( generate_entry_point_rule( - entry_point_script, + entry_point_target_name, + entry_point_script_name, library_name, ) ) @@ -449,7 +440,7 @@ def extract_wheel( data_exclude=data_exclude, data=data, srcs_exclude=srcs_exclude, - tags=["pypi_name=" + whl.name, "pypi_version=" + whl.metadata.version], + tags=["pypi_name=" + whl.name, "pypi_version=" + whl.version], additional_content=additional_content, ) build_file.write(contents) diff --git a/python/pip_install/extract_wheels/lib/purelib.py b/python/pip_install/extract_wheels/lib/purelib.py deleted file mode 100644 index 978e0f18f5..0000000000 --- a/python/pip_install/extract_wheels/lib/purelib.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Functions to make purelibs Bazel compatible""" -import os -import pathlib -import shutil - -from python.pip_install.extract_wheels.lib import wheel - - -def spread_purelib_into_root(wheel_dir: str) -> None: - """Unpacks purelib directories into the root. - - Args: - wheel_dir: The root of the extracted wheel directory. - """ - dist_info = wheel.get_dist_info(wheel_dir) - wheel_metadata_file_path = pathlib.Path(dist_info, "WHEEL") - wheel_metadata_dict = wheel.parse_wheel_meta_file(str(wheel_metadata_file_path)) - - # It is not guaranteed that a WHEEL file author populates 'Root-Is-Purelib'. - # See: https://github.com/bazelbuild/rules_python/issues/435 - root_is_purelib: str = wheel_metadata_dict.get("Root-Is-Purelib", "") - if root_is_purelib.lower() == "true": - # The Python package code is in the root of the Wheel, so no need to 'spread' anything. - return - - dot_data_dir = wheel.get_dot_data_directory(wheel_dir) - # 'Root-Is-Purelib: false' is no guarantee a .data directory exists with - # package code in it. eg. the 'markupsafe' package. - if not dot_data_dir: - return - - for child in pathlib.Path(dot_data_dir).iterdir(): - # TODO(Jonathon): Should all other potential folders get ignored? eg. 'platlib' - if str(child).endswith("purelib"): - _spread_purelib(child, wheel_dir) - - -def backport_copytree(src: pathlib.Path, dst: pathlib.Path): - """Implementation similar to shutil.copytree. - - shutil.copytree before python3.8 does not allow merging one tree with - an existing one. This function does that, while ignoring complications around symlinks, which - can't exist is wheels (See https://bugs.python.org/issue27318). - """ - os.makedirs(dst, exist_ok=True) - for path in src.iterdir(): - if path.is_dir(): - backport_copytree(path, pathlib.Path(dst, path.name)) - elif not pathlib.Path(dst, path.name).exists(): - shutil.copy(path, dst) - - -def _spread_purelib(purelib_dir: pathlib.Path, root_dir: str) -> None: - """Recursively moves all sibling directories of the purelib to the root. - - Args: - purelib_dir: The directory of the purelib. - root_dir: The directory to move files into. - """ - for child in purelib_dir.iterdir(): - if child.is_dir(): - backport_copytree(src=child, dst=pathlib.Path(root_dir, child.name)) - elif not pathlib.Path(root_dir, child.name).exists(): - shutil.copy( - src=str(child), - dst=root_dir, - ) diff --git a/python/pip_install/extract_wheels/lib/purelib_test.py b/python/pip_install/extract_wheels/lib/purelib_test.py deleted file mode 100644 index 02fd9220c5..0000000000 --- a/python/pip_install/extract_wheels/lib/purelib_test.py +++ /dev/null @@ -1,40 +0,0 @@ -import os -import unittest -from contextlib import contextmanager -from pathlib import Path -from tempfile import TemporaryDirectory - -from python.pip_install.extract_wheels.lib import purelib - - -class TestPurelibTestCase(unittest.TestCase): - @contextmanager - def setup_faux_unzipped_wheel(self): - files = [ - ("faux_wheel.data/purelib/toplevel/foo.py", "# foo"), - ("faux_wheel.data/purelib/toplevel/dont_overwrite.py", "overwritten"), - ("faux_wheel.data/purelib/toplevel/subdir/baz.py", "overwritten"), - ("toplevel/bar.py", "# bar"), - ("toplevel/dont_overwrite.py", "original"), - ] - with TemporaryDirectory() as td: - self.td_path = Path(td) - self.purelib_path = self.td_path / Path("faux_wheel.data/purelib") - for file_, content in files: - path = self.td_path / Path(file_) - path.parent.mkdir(parents=True, exist_ok=True) - with open(str(path), "w") as f: - f.write(content) - yield - - def test_spread_purelib_(self): - with self.setup_faux_unzipped_wheel(): - purelib._spread_purelib(self.purelib_path, self.td_path) - self.assertTrue(Path(self.td_path, "toplevel/foo.py").exists()) - self.assertTrue(Path(self.td_path, "toplevel/subdir/baz.py").exists()) - with open(Path(self.td_path, "toplevel/dont_overwrite.py")) as original: - self.assertEqual(original.read().strip(), "original") - - -if __name__ == "__main__": - unittest.main() diff --git a/python/pip_install/extract_wheels/lib/wheel.py b/python/pip_install/extract_wheels/lib/wheel.py index 85bc95830d..afd4329310 100644 --- a/python/pip_install/extract_wheels/lib/wheel.py +++ b/python/pip_install/extract_wheels/lib/wheel.py @@ -1,28 +1,9 @@ """Utility class to inspect an extracted wheel directory""" -import configparser -import glob -import os -import stat -import zipfile +import email from typing import Dict, Optional, Set +import installer import pkg_resources -import pkginfo - - -def current_umask() -> int: - """Get the current umask which involves having to set it temporarily.""" - mask = os.umask(0) - os.umask(mask) - return mask - - -def set_extracted_file_to_default_mode_plus_executable(path: str) -> None: - """ - Make file present at path have execute for user/group/world - (chmod +x) is no-op on windows per python docs - """ - os.chmod(path, (0o777 & ~current_umask() | 0o111)) class Wheel: @@ -37,57 +18,46 @@ def path(self) -> str: @property def name(self) -> str: - return str(self.metadata.name) + # TODO Also available as installer.sources.WheelSource.distribution + return str(self.metadata['Name']) @property - def metadata(self) -> pkginfo.Wheel: - return pkginfo.get_metadata(self.path) + def metadata(self) -> email.message.Message: + with installer.sources.WheelFile.open(self.path) as wheel_source: + metadata_contents = wheel_source.read_dist_info("METADATA") + metadata = installer.utils.parse_metadata_file(metadata_contents) + return metadata - def entry_points(self) -> Dict[str, str]: + @property + def version(self) -> str: + # TODO Also available as installer.sources.WheelSource.version + return str(self.metadata["Version"]) + + def entry_points(self) -> Dict[str, tuple[str, str]]: """Returns the entrypoints defined in the current wheel See https://packaging.python.org/specifications/entry-points/ for more info Returns: - Dict[str, str]: A mappying of the entry point's name to it's method + Dict[str, Tuple[str, str]]: A mapping of the entry point's name to it's module and attribute """ - with zipfile.ZipFile(self.path, "r") as whl: - # Calculate the location of the entry_points.txt file - metadata = self.metadata - name = "{}-{}".format(metadata.name.replace("-", "_"), metadata.version) - - # Note that the zipfile module always uses the forward slash as - # directory separator, even on Windows, so don't use os.path.join - # here. Reference for Python 3.10: - # https://github.com/python/cpython/blob/3.10/Lib/zipfile.py#L355. - # TODO: use zipfile.Path once 3.8 is our minimum supported version - entry_points_path = "{}.dist-info/entry_points.txt".format(name) - - # If this file does not exist in the wheel, there are no entry points - if entry_points_path not in whl.namelist(): + with installer.sources.WheelFile.open(self.path) as wheel_source: + if "entry_points.txt" not in wheel_source.dist_info_filenames: return dict() - # Parse the avaialble entry points - config = configparser.ConfigParser() - try: - config.read_string(whl.read(entry_points_path).decode("utf-8")) - if "console_scripts" in config.sections(): - return dict(config["console_scripts"]) - - # TODO: It's unclear what to do in a situation with duplicate sections or options. - # For now, we treat the config file as though it contains no scripts. For more - # details on the config parser, see: - # https://docs.python.org/3.7/library/configparser.html#configparser.ConfigParser - # https://docs.python.org/3.7/library/configparser.html#configparser.Error - except configparser.Error: - pass + entry_points_mapping = dict() + entry_points_contents = wheel_source.read_dist_info("entry_points.txt") + entry_points = installer.utils.parse_entrypoints(entry_points_contents) + for script, module, attribute, script_section in entry_points: + if script_section == "console": + entry_points_mapping[script] = (module, attribute) - return dict() + return entry_points_mapping def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]: dependency_set = set() - for wheel_req in self.metadata.requires_dist: + for wheel_req in self.metadata.get_all('Requires-Dist', []): req = pkg_resources.Requirement(wheel_req) # type: ignore if req.marker is None or any( @@ -99,91 +69,26 @@ def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]: return dependency_set def unzip(self, directory: str) -> None: - with zipfile.ZipFile(self.path, "r") as whl: - whl.extractall(directory) - # The following logic is borrowed from Pip: - # https://github.com/pypa/pip/blob/cc48c07b64f338ac5e347d90f6cb4efc22ed0d0b/src/pip/_internal/utils/unpacking.py#L240 - for info in whl.infolist(): - name = info.filename - # Do not attempt to modify directories. - if name.endswith("/") or name.endswith("\\"): - continue - mode = info.external_attr >> 16 - # if mode and regular file and any execute permissions for - # user/group/world? - if mode and stat.S_ISREG(mode) and mode & 0o111: - name = os.path.join(directory, name) - set_extracted_file_to_default_mode_plus_executable(name) - - -def get_dist_info(wheel_dir: str) -> str: - """ "Returns the relative path to the dist-info directory if it exists. - - Args: - wheel_dir: The root of the extracted wheel directory. - - Returns: - Relative path to the dist-info directory if it exists, else, None. - """ - dist_info_dirs = glob.glob(os.path.join(wheel_dir, "*.dist-info")) - if not dist_info_dirs: - raise ValueError( - "No *.dist-info directory found. %s is not a valid Wheel." % wheel_dir - ) - - if len(dist_info_dirs) > 1: - raise ValueError( - "Found more than 1 *.dist-info directory. %s is not a valid Wheel." - % wheel_dir - ) - - return dist_info_dirs[0] - - -def get_dot_data_directory(wheel_dir: str) -> Optional[str]: - """Returns the relative path to the data directory if it exists. - - See: https://www.python.org/dev/peps/pep-0491/#the-data-directory - - Args: - wheel_dir: The root of the extracted wheel directory. - - Returns: - Relative path to the data directory if it exists, else, None. - """ - - dot_data_dirs = glob.glob(os.path.join(wheel_dir, "*.data")) - if not dot_data_dirs: - return None - - if len(dot_data_dirs) > 1: - raise ValueError( - "Found more than 1 *.data directory. %s is not a valid Wheel." % wheel_dir + installation_schemes = { + "purelib": f"/site-packages", + "platlib": f"/site-packages", + "headers": f"/include", + "scripts": f"/bin", + "data": f"/data", + } + destination = installer.destinations.SchemeDictionaryDestination( + installation_schemes, + # TODO Should entry_point scripts also be handled by installer rather than custom code? + interpreter="/dev/null", + script_kind="posix", + destdir=directory, ) - return dot_data_dirs[0] - - -def parse_wheel_meta_file(wheel_dir: str) -> Dict[str, str]: - """Parses the given WHEEL file into a dictionary. - - Args: - wheel_dir: The file path of the WHEEL metadata file in dist-info. - - Returns: - The WHEEL file mapped into a dictionary. - """ - contents = {} - with open(wheel_dir, "r") as wheel_file: - for line in wheel_file: - cleaned = line.strip() - if not cleaned: - continue - try: - key, value = cleaned.split(":", maxsplit=1) - contents[key] = value.strip() - except ValueError: - raise RuntimeError( - "Encounted invalid line in WHEEL file: '%s'" % cleaned - ) - return contents + with installer.sources.WheelFile.open(self.path) as wheel_source: + installer.install( + source=wheel_source, + destination=destination, + additional_metadata={ + "INSTALLER": b"https://github.com/bazelbuild/rules_python", + }, + ) diff --git a/python/pip_install/private/srcs.bzl b/python/pip_install/private/srcs.bzl index 3f20c4596f..a253b66bbb 100644 --- a/python/pip_install/private/srcs.bzl +++ b/python/pip_install/private/srcs.bzl @@ -14,7 +14,6 @@ PIP_INSTALL_PY_SRCS = [ "@rules_python//python/pip_install/extract_wheels/lib:arguments.py", "@rules_python//python/pip_install/extract_wheels/lib:bazel.py", "@rules_python//python/pip_install/extract_wheels/lib:namespace_pkgs.py", - "@rules_python//python/pip_install/extract_wheels/lib:purelib.py", "@rules_python//python/pip_install/extract_wheels/lib:requirements.py", "@rules_python//python/pip_install/extract_wheels/lib:wheel.py", "@rules_python//python/pip_install/parse_requirements_to_bzl:__init__.py", diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index 352f66341c..a57ff9deaa 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -17,6 +17,11 @@ _RULE_DEPS = [ "https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl", "9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2", ), + ( + "pypi__installer", + "https://files.pythonhosted.org/packages/1b/21/3e6ebd12d8dccc55bcb7338db462c75ac86dbd0ac7439ac114616b21667b/installer-0.5.1-py3-none-any.whl", + "1d6c8d916ed82771945b9c813699e6f57424ded970c9d8bf16bbc23e1e826ed3", + ), ( "pypi__pip", "https://files.pythonhosted.org/packages/4d/16/0a14ca596f30316efd412a60bdfac02a7259bf8673d4d917dc60b9a21812/pip-22.0.4-py3-none-any.whl", @@ -27,11 +32,6 @@ _RULE_DEPS = [ "https://files.pythonhosted.org/packages/6d/16/75d65bdccd48bb59a08e2bf167b01d8532f65604270d0a292f0f16b7b022/pip_tools-5.5.0-py2.py3-none-any.whl", "10841c1e56c234d610d0466447685b9ea4ee4a2c274f858c0ef3c33d9bd0d985", ), - ( - "pypi__pkginfo", - "https://files.pythonhosted.org/packages/cd/00/49f59cdd2c6a52e6665fda4de671dac5614366dc827e050c55428241b929/pkginfo-1.8.2-py2.py3-none-any.whl", - "c24c487c6a7f72c66e816ab1796b96ac6c3d14d49338293d2141664330b55ffc", - ), ( "pypi__setuptools", "https://files.pythonhosted.org/packages/7c/5b/3d92b9f0f7ca1645cba48c080b54fe7d8b1033a4e5720091d1631c4266db/setuptools-60.10.0-py3-none-any.whl", diff --git a/python/repositories.bzl b/python/repositories.bzl index bd08ec466c..687388c62e 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -117,6 +117,7 @@ filegroup( srcs = glob( include = [ "*.exe", + "*.dll", "bin/**", "DLLs/**", "extensions/**", diff --git a/python/versions.bzl b/python/versions.bzl index b263abf000..1f18528336 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -42,7 +42,7 @@ TOOL_VERSIONS = { "aarch64-apple-darwin": "f9a3cbb81e0463d6615125964762d133387d561b226a30199f5b039b20f1d944", # no aarch64-unknown-linux-gnu build available for 3.8.12 "x86_64-apple-darwin": "f323fbc558035c13a85ce2267d0fad9e89282268ecb810e364fff1d0a079d525", - "x86_64-pc-windows-msvc": "924f9fd51ff6ccc533ed8e96c5461768da5781eb3dfc11d846f9e300fab44eda", + "x86_64-pc-windows-msvc": "4658e08a00d60b1e01559b74d58ff4dd04da6df935d55f6268a15d6d0a679d74", "x86_64-unknown-linux-gnu": "5be9c6d61e238b90dfd94755051c0d3a2d8023ebffdb4b0fa4e8fedd09a6cab6", }, "strip_prefix": "python", @@ -53,7 +53,7 @@ TOOL_VERSIONS = { "aarch64-apple-darwin": "ad66c2a3e7263147e046a32694de7b897a46fb0124409d29d3a93ede631c8aee", "aarch64-unknown-linux-gnu": "12dd1f125762f47975990ec744532a1cf3db74ad60f4dfb476ca42deb7f78ca4", "x86_64-apple-darwin": "fdaf594142446029e314a9beb91f1ac75af866320b50b8b968181e592550cd68", - "x86_64-pc-windows-msvc": "5bc65ce023614bf496a6748e41dca934b70fc5fac6dfacc46aa8dbcad772afc2", + "x86_64-pc-windows-msvc": "c145d9d8143ce163670af124b623d7a2405143a3708b033b4d33eed355e61b24", "x86_64-unknown-linux-gnu": "455089cc576bd9a58db45e919d1fc867ecdbb0208067dffc845cc9bbf0701b70", }, "strip_prefix": "python", @@ -64,7 +64,7 @@ TOOL_VERSIONS = { "aarch64-apple-darwin": "1409acd9a506e2d1d3b65c1488db4e40d8f19d09a7df099667c87a506f71c0ef", "aarch64-unknown-linux-gnu": "8f351a8cc348bb45c0f95b8634c8345ec6e749e483384188ad865b7428342703", "x86_64-apple-darwin": "8146ad4390710ec69b316a5649912df0247d35f4a42e2aa9615bffd87b3e235a", - "x86_64-pc-windows-msvc": "a293c5838dd9c8438a84372fb95dda9752df63928a8a2ae516438f187f89567d", + "x86_64-pc-windows-msvc": "a1d9a594cd3103baa24937ad9150c1a389544b4350e859200b3e5c036ac352bd", "x86_64-unknown-linux-gnu": "9b64eca2a94f7aff9409ad70bdaa7fbbf8148692662e764401883957943620dd", }, "strip_prefix": "python", @@ -151,7 +151,7 @@ def get_release_url(platform, python_version, base_url = DEFAULT_RELEASE_BASE_UR release_filename = url.format( platform = platform, python_version = python_version, - build = "static-install_only" if (WINDOWS_NAME in platform) else "install_only", + build = "shared-install_only" if (WINDOWS_NAME in platform) else "install_only", ) url = "/".join([base_url, release_filename]) return (release_filename, url, strip_prefix) From 5861008ca6476e836e8202adfdcf34e1728e7a10 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Tue, 14 Jun 2022 19:27:15 +1000 Subject: [PATCH 2/3] Remove f-strings --- .idea/.gitignore | 8 ++++++++ .idea/codeStyles/Project.xml | 13 +++++++++++++ .idea/codeStyles/codeStyleConfig.xml | 5 +++++ .idea/modules.xml | 8 ++++++++ .idea/rules_python.iml | 9 +++++++++ .idea/runConfigurations.xml | 10 ++++++++++ .idea/vcs.xml | 6 ++++++ python/pip_install/extract_wheels/lib/wheel.py | 10 +++++----- 8 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/codeStyles/Project.xml create mode 100644 .idea/codeStyles/codeStyleConfig.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/rules_python.iml create mode 100644 .idea/runConfigurations.xml create mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000000..13566b81b0 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 0000000000..5fac255b10 --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,13 @@ + + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000000..a55e7a179b --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000000..1a62816ff0 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/rules_python.iml b/.idea/rules_python.iml new file mode 100644 index 0000000000..d6ebd48059 --- /dev/null +++ b/.idea/rules_python.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations.xml b/.idea/runConfigurations.xml new file mode 100644 index 0000000000..797acea53e --- /dev/null +++ b/.idea/runConfigurations.xml @@ -0,0 +1,10 @@ + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000000..35eb1ddfbb --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/python/pip_install/extract_wheels/lib/wheel.py b/python/pip_install/extract_wheels/lib/wheel.py index ac6d6779ab..f6eb3449a4 100644 --- a/python/pip_install/extract_wheels/lib/wheel.py +++ b/python/pip_install/extract_wheels/lib/wheel.py @@ -70,11 +70,11 @@ def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]: def unzip(self, directory: str) -> None: installation_schemes = { - "purelib": f"/site-packages", - "platlib": f"/site-packages", - "headers": f"/include", - "scripts": f"/bin", - "data": f"/data", + "purelib": "/site-packages", + "platlib": "/site-packages", + "headers": "/include", + "scripts": "/bin", + "data": "/data", } destination = installer.destinations.SchemeDictionaryDestination( installation_schemes, From 60e63b4ba3859a631599c537ca5bcc54ca6d837a Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Tue, 14 Jun 2022 19:27:55 +1000 Subject: [PATCH 3/3] Remove .idea --- .idea/.gitignore | 8 -------- .idea/codeStyles/Project.xml | 13 ------------- .idea/codeStyles/codeStyleConfig.xml | 5 ----- .idea/modules.xml | 8 -------- .idea/rules_python.iml | 9 --------- .idea/runConfigurations.xml | 10 ---------- .idea/vcs.xml | 6 ------ 7 files changed, 59 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/codeStyles/Project.xml delete mode 100644 .idea/codeStyles/codeStyleConfig.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/rules_python.iml delete mode 100644 .idea/runConfigurations.xml delete mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b81b0..0000000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml deleted file mode 100644 index 5fac255b10..0000000000 --- a/.idea/codeStyles/Project.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml deleted file mode 100644 index a55e7a179b..0000000000 --- a/.idea/codeStyles/codeStyleConfig.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 1a62816ff0..0000000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/rules_python.iml b/.idea/rules_python.iml deleted file mode 100644 index d6ebd48059..0000000000 --- a/.idea/rules_python.iml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations.xml b/.idea/runConfigurations.xml deleted file mode 100644 index 797acea53e..0000000000 --- a/.idea/runConfigurations.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1ddfbb..0000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file