Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions examples/build_file_generation/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import requests


def main(url):
r = requests.get(url)
print(r.text)
1 change: 0 additions & 1 deletion examples/build_file_generation/__main__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
from __init__ import main


if __name__ == "__main__":
main("https://example.com")
1 change: 1 addition & 0 deletions examples/pip_install/main.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import boto3


def the_dir():
return dir(boto3)

Expand Down
2 changes: 1 addition & 1 deletion examples/pip_install/pip_install_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def test_dist_info(self):
env.split(" "),
[
"external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/DESCRIPTION.rst",
'external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/INSTALLER',
"external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/INSTALLER",
"external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/METADATA",
"external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/RECORD",
"external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/WHEEL",
Expand Down
2 changes: 1 addition & 1 deletion examples/pip_parse/pip_parse_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def test_dist_info(self):
self.assertListEqual(
env.split(" "),
[
'external/pypi_requests/site-packages/requests-2.25.1.dist-info/INSTALLER',
"external/pypi_requests/site-packages/requests-2.25.1.dist-info/INSTALLER",
"external/pypi_requests/site-packages/requests-2.25.1.dist-info/LICENSE",
"external/pypi_requests/site-packages/requests-2.25.1.dist-info/METADATA",
"external/pypi_requests/site-packages/requests-2.25.1.dist-info/RECORD",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,9 @@ def test_data_exclude_glob(self):
current_wheel_version = "0.37.1"

r = runfiles.Create()
dist_info_dir = (
"pip_repository_annotations_example/external/{}/site-packages/wheel-{}.dist-info".format(
self.wheel_pkg_dir(),
current_wheel_version,
)
dist_info_dir = "pip_repository_annotations_example/external/{}/site-packages/wheel-{}.dist-info".format(
self.wheel_pkg_dir(),
current_wheel_version,
)

# Note: `METADATA` is important as it's consumed by https://docs.python.org/3/library/importlib.metadata.html
Expand Down
7 changes: 2 additions & 5 deletions gazelle/modules_mapping/generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,7 @@ def is_metadata(path):
# Ref: https://peps.python.org/pep-0427/#what-s-the-deal-with-purelib-vs-platlib
def data_has_purelib_or_platlib(path):
maybe_lib = path.split("/")[1].lower()
return is_metadata(path) and (
maybe_lib == "purelib" or maybe_lib == "platlib"
)

return is_metadata(path) and (maybe_lib == "purelib" or maybe_lib == "platlib")


def module_for_path(path, whl, mapping):
Expand All @@ -95,7 +92,7 @@ def module_for_path(path, whl, mapping):
if ext == ".so":
# Also remove extra metadata that is embeded as part of
# the file name as an extra extension.
ext = ''.join(pathlib.Path(root).suffixes)
ext = "".join(pathlib.Path(root).suffixes)
module = root[: -len(ext)].replace("/", ".")
mapping[module] = wheel_name

Expand Down
16 changes: 8 additions & 8 deletions gazelle/parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def parse_import_statements(content, filepath):
"name": subnode.name,
"lineno": node.lineno,
"filepath": filepath,
"from": ""
"from": "",
}
modules.append(module)
elif isinstance(node, ast.ImportFrom) and node.level == 0:
Expand All @@ -30,7 +30,7 @@ def parse_import_statements(content, filepath):
"name": f"{node.module}.{subnode.name}",
"lineno": node.lineno,
"filepath": filepath,
"from": node.module
"from": node.module,
}
modules.append(module)
return modules
Expand All @@ -52,8 +52,9 @@ def parse(repo_root, rel_package_path, filename):
content = file.read()
# From simple benchmarks, 2 workers gave the best performance here.
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
modules_future = executor.submit(parse_import_statements, content,
rel_filepath)
modules_future = executor.submit(
parse_import_statements, content, rel_filepath
)
comments_future = executor.submit(parse_comments, content)
modules = modules_future.result()
comments = comments_future.result()
Expand All @@ -73,12 +74,11 @@ def main(stdin, stdout):
filenames = parse_request["filenames"]
outputs = list()
if len(filenames) == 1:
outputs.append(parse(repo_root, rel_package_path,
filenames[0]))
outputs.append(parse(repo_root, rel_package_path, filenames[0]))
else:
futures = [
executor.submit(parse, repo_root, rel_package_path,
filename) for filename in filenames
executor.submit(parse, repo_root, rel_package_path, filename)
for filename in filenames
if filename != ""
]
for future in concurrent.futures.as_completed(futures):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from google.cloud import aiplatform

from bar import main
from google.cloud import aiplatform

if __name__ == "__main__":
print(aiplatform)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from google.cloud import aiplatform, storage


def main():
a = dir(aiplatform)
b = dir(storage)
6 changes: 1 addition & 5 deletions python/pip_install/extract_wheels/bazel.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,7 @@
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Set

from python.pip_install.extract_wheels import (
annotation,
namespace_pkgs,
wheel,
)
from python.pip_install.extract_wheels import annotation, namespace_pkgs, wheel

WHEEL_FILE_LABEL = "whl"
PY_LIBRARY_LABEL = "pkg"
Expand Down
4 changes: 3 additions & 1 deletion python/pip_install/extract_wheels/extract_single_wheel.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@
import sys
from tempfile import NamedTemporaryFile

from python.pip_install.extract_wheels.extract_wheels import configure_reproducible_wheels
from python.pip_install.extract_wheels import arguments, bazel, requirements
from python.pip_install.extract_wheels.annotation import annotation_from_str_path
from python.pip_install.extract_wheels.extract_wheels import (
configure_reproducible_wheels,
)


def main() -> None:
Expand Down
25 changes: 15 additions & 10 deletions python/pip_install/extract_wheels/parse_requirements_to_bzl.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,25 +40,25 @@ def parse_install_requirements(
if (
# PEP-440 direct references are considered pinned
# See: https://peps.python.org/pep-0440/#direct-references and https://peps.python.org/pep-0508/
not install_req.link and
not install_req.is_pinned
not install_req.link
and not install_req.is_pinned
):
unpinned_reqs.append(str(install_req))
install_req_and_lines.append(
(install_req, line)
)
install_req_and_lines.append((install_req, line))

else:
extra_pip_args.extend(shlex.split(line))

if len(unpinned_reqs) > 0:
unpinned_reqs_str = "\n".join(unpinned_reqs)
raise RuntimeError(f"""\
raise RuntimeError(
f"""\
The `requirements_lock` file must be fully pinned. See `compile_pip_requirements`.
Alternatively, use `pip-tools` or a similar mechanism to produce a pinned lockfile.

The following requirements were not pinned:
{unpinned_reqs_str}""")
{unpinned_reqs_str}"""
)

return install_req_and_lines

Expand Down Expand Up @@ -259,14 +259,19 @@ def main(output: TextIO) -> None:
}
)

output.write(textwrap.dedent(
"""\
output.write(
textwrap.dedent(
"""\
\"\"\"Starlark representation of locked requirements.

@generated by rules_python pip_parse repository rule
from {}
\"\"\"
""".format(args.requirements_lock_label)))
""".format(
args.requirements_lock_label
)
)
)
output.write(
generate_parsed_requirements_contents(
requirements_lock=args.requirements_lock,
Expand Down
4 changes: 2 additions & 2 deletions python/pip_install/extract_wheels/wheel.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def path(self) -> str:
@property
def name(self) -> str:
# TODO Also available as installer.sources.WheelSource.distribution
name = str(self.metadata['Name'])
name = str(self.metadata["Name"])
return canonicalize_name(name)

@property
Expand Down Expand Up @@ -59,7 +59,7 @@ def entry_points(self) -> Dict[str, Tuple[str, str]]:
def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]:
dependency_set = set()

for wheel_req in self.metadata.get_all('Requires-Dist', []):
for wheel_req in self.metadata.get_all("Requires-Dist", []):
req = pkg_resources.Requirement(wheel_req) # type: ignore

if req.marker is None or any(
Expand Down
10 changes: 8 additions & 2 deletions python/pip_install/pip_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,11 @@ def _select_golden_requirements_file(
parts = requirements_in.split(os.path.sep, 2)
if parts[0] == "external":
requirements_in = parts[2]
requirements_txt = requirements_txt if "BUILD_WORKSPACE_DIRECTORY" in os.environ else os.path.join("..", "..", requirements_txt)
requirements_txt = (
requirements_txt
if "BUILD_WORKSPACE_DIRECTORY" in os.environ
else os.path.join("..", "..", requirements_txt)
)
os.chdir(os.path.join(parts[0], parts[1]))
update_target_label = sys.argv.pop(1)

Expand Down Expand Up @@ -79,7 +83,9 @@ def _select_golden_requirements_file(
#
# Changing to the WORKSPACE root avoids 'file not found' errors when the `.update` target is run
# from different directories within the WORKSPACE.
requirements_txt = os.path.join(os.environ["BUILD_WORKSPACE_DIRECTORY"], requirements_txt)
requirements_txt = os.path.join(
os.environ["BUILD_WORKSPACE_DIRECTORY"], requirements_txt
)
else:
err_msg = (
"Expected to find BUILD_WORKSPACE_DIRECTORY (running under `bazel run`) or "
Expand Down
2 changes: 1 addition & 1 deletion tools/wheelmaker.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ def arcname_from(name):
size = 0
with open(real_filename, "rb") as f:
while True:
block = f.read(2 ** 20)
block = f.read(2**20)
if not block:
break
hash.update(block)
Expand Down