From 074b5efc7cfd99ec2940190e0dd7197b78bd85db Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Tue, 15 Mar 2022 14:22:01 -0700 Subject: [PATCH 1/2] Document how to vendor a pip_parse requirements.bzl file fixes #608 --- docs/pip.md | 26 +++++++++++++++++++ python/pip.bzl | 26 +++++++++++++++++++ .../parse_requirements_to_bzl/__init__.py | 19 +++++++------- .../parse_requirements_to_bzl/__main__.py | 13 +++++++++- python/pip_install/pip_repository.bzl | 13 +++++++--- 5 files changed, 82 insertions(+), 15 deletions(-) diff --git a/docs/pip.md b/docs/pip.md index 0d657b4a42..9620a481b5 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -156,6 +156,7 @@ pip_parse(requirements_lock, requirements.bzl +``` + +Then load the requirements.bzl file directly, without using `pip_parse` in the WORKSPACE. ``` diff --git a/python/pip.bzl b/python/pip.bzl index 0530a2c9f1..32e46eee5f 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -105,6 +105,7 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs): """Accepts a locked/compiled requirements file and installs the dependencies listed within. Those dependencies become available in a generated `requirements.bzl` file. + You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below. This macro runs a repository rule that invokes `pip`. In your WORKSPACE file: @@ -165,6 +166,31 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs): name = "flake8", actual = entry_point("flake8"), ) + + ## Vendoring the requirements.bzl file + + In some cases you may not want to generate the requirements.bzl file as a repository rule + while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module + such as a ruleset, you may want to include the requirements.bzl file rather than make your users + install the WORKSPACE setup to generate it. + See https://github.com/bazelbuild/rules_python/issues/608 + + This is the same workflow as Gazelle, which creates `go_repository` rules with + [`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) + + Simply run the same tool that the `pip_parse` repository rule calls. + You can find the arguments in the generated BUILD file in the pip_parse repo, + for example in `$(bazel info output_base)/external/pypi/BUILD.bazel` for a repo + named `pypi`. + + The command will look like this: + ```shell + bazel run -- @rules_python//python/pip_install/parse_requirements_to_bzl \\ + --requirements_lock ./requirements_lock.txt \\ + --quiet False --timeout 120 --repo pypi --repo-prefix pypi_ > requirements.bzl + ``` + + Then load the requirements.bzl file directly, without using `pip_parse` in the WORKSPACE. ``` Args: diff --git a/python/pip_install/parse_requirements_to_bzl/__init__.py b/python/pip_install/parse_requirements_to_bzl/__init__.py index 3fc22898cb..a839bc1b82 100644 --- a/python/pip_install/parse_requirements_to_bzl/__init__.py +++ b/python/pip_install/parse_requirements_to_bzl/__init__.py @@ -166,7 +166,7 @@ def coerce_to_bool(option): return str(option).lower() == "true" -def main() -> None: +def main(output) -> None: parser = argparse.ArgumentParser( description="Create rules to incrementally fetch needed \ dependencies from a fully resolved requirements lock file." @@ -216,7 +216,7 @@ def main() -> None: args.requirements_lock, whl_library_args["extra_pip_args"] ) req_names = sorted([req.name for req, _ in install_requirements]) - annotations = args.annotations.collect(req_names) + annotations = args.annotations.collect(req_names) if args.annotations else {} # Write all rendered annotation files and generate a list of the labels to write to the requirements file annotated_requirements = dict() @@ -231,12 +231,11 @@ def main() -> None: } ) - with open("requirements.bzl", "w") as requirement_file: - requirement_file.write( - generate_parsed_requirements_contents( - requirements_lock=args.requirements_lock, - repo_prefix=args.repo_prefix, - whl_library_args=whl_library_args, - annotations=annotated_requirements, - ) + output.write( + generate_parsed_requirements_contents( + requirements_lock=args.requirements_lock, + repo_prefix=args.repo_prefix, + whl_library_args=whl_library_args, + annotations=annotated_requirements, ) + ) diff --git a/python/pip_install/parse_requirements_to_bzl/__main__.py b/python/pip_install/parse_requirements_to_bzl/__main__.py index 89199612b5..4ca73867a7 100644 --- a/python/pip_install/parse_requirements_to_bzl/__main__.py +++ b/python/pip_install/parse_requirements_to_bzl/__main__.py @@ -1,5 +1,16 @@ """Main entry point.""" +import os +import sys + from python.pip_install.parse_requirements_to_bzl import main if __name__ == "__main__": - main() + # Under `bazel run`, just print the generated starlark code. + # This allows users to check that into their repository rather than + # call pip_parse to generate as a repository rule. + if "BUILD_WORKING_DIRECTORY" in os.environ: + os.chdir(os.environ["BUILD_WORKING_DIRECTORY"]) + main(sys.stdout) + else: + with open("requirements.bzl", "w") as requirement_file: + main(requirement_file) diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 51d4fb558d..59845d373c 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -116,6 +116,8 @@ _BUILD_FILE_CONTENTS = """\ package(default_visibility = ["//visibility:public"]) # Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it +# This file was generated by running: +# {0} exports_files(["requirements.bzl"]) """ @@ -125,9 +127,6 @@ def _pip_repository_impl(rctx): if rctx.attr.incremental and not rctx.attr.requirements_lock: fail("Incremental mode requires a requirements_lock attribute be specified.") - # We need a BUILD file to load the generated requirements.bzl - rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) - # Write the annotations file to pass to the wheel maker annotations = {package: json.decode(data) for (package, data) in rctx.attr.annotations.items()} annotations_file = rctx.path("annotations.json") @@ -152,7 +151,7 @@ def _pip_repository_impl(rctx): args += ["--python_interpreter", _get_python_interpreter_attr(rctx)] if rctx.attr.python_interpreter_target: args += ["--python_interpreter_target", str(rctx.attr.python_interpreter_target)] - + progress_message = "Parsing requirements to starlark" else: args = [ python_interpreter, @@ -163,10 +162,13 @@ def _pip_repository_impl(rctx): "--annotations", annotations_file, ] + progress_message = "Extracting wheels" args += ["--repo", rctx.attr.name, "--repo-prefix", rctx.attr.repo_prefix] args = _parse_optional_attrs(rctx, args) + rctx.report_progress(progress_message) + result = rctx.execute( args, # Manually construct the PYTHONPATH since we cannot use the toolchain here @@ -178,6 +180,9 @@ def _pip_repository_impl(rctx): if result.return_code: fail("rules_python failed: %s (%s)" % (result.stdout, result.stderr)) + # We need a BUILD file to load the generated requirements.bzl + rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS.format(" ".join([str(a) for a in args]))) + return common_env = [ From f5be3ab8d09f43182e2d48fa61e151e7fa6afc91 Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Tue, 15 Mar 2022 20:14:55 -0700 Subject: [PATCH 2/2] code review feedback --- docs/pip.md | 2 +- python/pip.bzl | 2 +- python/pip_install/parse_requirements_to_bzl/__init__.py | 8 ++++++-- python/pip_install/pip_repository.bzl | 4 +--- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/pip.md b/docs/pip.md index 9620a481b5..74248cef7c 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -217,6 +217,7 @@ alias( name = "flake8", actual = entry_point("flake8"), ) +``` ## Vendoring the requirements.bzl file @@ -242,7 +243,6 @@ bazel run -- @rules_python//python/pip_install/parse_requirements_to_bzl \ ``` Then load the requirements.bzl file directly, without using `pip_parse` in the WORKSPACE. -``` **PARAMETERS** diff --git a/python/pip.bzl b/python/pip.bzl index 32e46eee5f..b0e13194b0 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -166,6 +166,7 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs): name = "flake8", actual = entry_point("flake8"), ) + ``` ## Vendoring the requirements.bzl file @@ -191,7 +192,6 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs): ``` Then load the requirements.bzl file directly, without using `pip_parse` in the WORKSPACE. - ``` Args: requirements_lock (Label): A fully resolved 'requirements.txt' pip requirement file diff --git a/python/pip_install/parse_requirements_to_bzl/__init__.py b/python/pip_install/parse_requirements_to_bzl/__init__.py index a839bc1b82..252cd26e00 100644 --- a/python/pip_install/parse_requirements_to_bzl/__init__.py +++ b/python/pip_install/parse_requirements_to_bzl/__init__.py @@ -4,7 +4,7 @@ import sys import textwrap from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, TextIO, Tuple from pip._internal.network.session import PipSession from pip._internal.req import constructors @@ -166,7 +166,11 @@ def coerce_to_bool(option): return str(option).lower() == "true" -def main(output) -> None: +def main(output: TextIO) -> None: + """Args: + + output: where to write the resulting starlark, such as sys.stdout or an open file + """ parser = argparse.ArgumentParser( description="Create rules to incrementally fetch needed \ dependencies from a fully resolved requirements lock file." diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 59845d373c..e587f34c73 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -116,8 +116,6 @@ _BUILD_FILE_CONTENTS = """\ package(default_visibility = ["//visibility:public"]) # Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it -# This file was generated by running: -# {0} exports_files(["requirements.bzl"]) """ @@ -181,7 +179,7 @@ def _pip_repository_impl(rctx): fail("rules_python failed: %s (%s)" % (result.stdout, result.stderr)) # We need a BUILD file to load the generated requirements.bzl - rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS.format(" ".join([str(a) for a in args]))) + rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS + "\n# The requirements.bzl file was generated by running:\n# " + " ".join([str(a) for a in args])) return