diff --git a/docs/pip.md b/docs/pip.md index 0d657b4a42..74248cef7c 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -156,6 +156,7 @@ pip_parse(requirements_lock, requirements.bzl +``` + +Then load the requirements.bzl file directly, without using `pip_parse` in the WORKSPACE. + **PARAMETERS** diff --git a/python/pip.bzl b/python/pip.bzl index 0530a2c9f1..b0e13194b0 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -105,6 +105,7 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs): """Accepts a locked/compiled requirements file and installs the dependencies listed within. Those dependencies become available in a generated `requirements.bzl` file. + You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below. This macro runs a repository rule that invokes `pip`. In your WORKSPACE file: @@ -167,6 +168,31 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs): ) ``` + ## Vendoring the requirements.bzl file + + In some cases you may not want to generate the requirements.bzl file as a repository rule + while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module + such as a ruleset, you may want to include the requirements.bzl file rather than make your users + install the WORKSPACE setup to generate it. + See https://github.com/bazelbuild/rules_python/issues/608 + + This is the same workflow as Gazelle, which creates `go_repository` rules with + [`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) + + Simply run the same tool that the `pip_parse` repository rule calls. + You can find the arguments in the generated BUILD file in the pip_parse repo, + for example in `$(bazel info output_base)/external/pypi/BUILD.bazel` for a repo + named `pypi`. + + The command will look like this: + ```shell + bazel run -- @rules_python//python/pip_install/parse_requirements_to_bzl \\ + --requirements_lock ./requirements_lock.txt \\ + --quiet False --timeout 120 --repo pypi --repo-prefix pypi_ > requirements.bzl + ``` + + Then load the requirements.bzl file directly, without using `pip_parse` in the WORKSPACE. + Args: requirements_lock (Label): A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead diff --git a/python/pip_install/parse_requirements_to_bzl/__init__.py b/python/pip_install/parse_requirements_to_bzl/__init__.py index 3fc22898cb..252cd26e00 100644 --- a/python/pip_install/parse_requirements_to_bzl/__init__.py +++ b/python/pip_install/parse_requirements_to_bzl/__init__.py @@ -4,7 +4,7 @@ import sys import textwrap from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, TextIO, Tuple from pip._internal.network.session import PipSession from pip._internal.req import constructors @@ -166,7 +166,11 @@ def coerce_to_bool(option): return str(option).lower() == "true" -def main() -> None: +def main(output: TextIO) -> None: + """Args: + + output: where to write the resulting starlark, such as sys.stdout or an open file + """ parser = argparse.ArgumentParser( description="Create rules to incrementally fetch needed \ dependencies from a fully resolved requirements lock file." @@ -216,7 +220,7 @@ def main() -> None: args.requirements_lock, whl_library_args["extra_pip_args"] ) req_names = sorted([req.name for req, _ in install_requirements]) - annotations = args.annotations.collect(req_names) + annotations = args.annotations.collect(req_names) if args.annotations else {} # Write all rendered annotation files and generate a list of the labels to write to the requirements file annotated_requirements = dict() @@ -231,12 +235,11 @@ def main() -> None: } ) - with open("requirements.bzl", "w") as requirement_file: - requirement_file.write( - generate_parsed_requirements_contents( - requirements_lock=args.requirements_lock, - repo_prefix=args.repo_prefix, - whl_library_args=whl_library_args, - annotations=annotated_requirements, - ) + output.write( + generate_parsed_requirements_contents( + requirements_lock=args.requirements_lock, + repo_prefix=args.repo_prefix, + whl_library_args=whl_library_args, + annotations=annotated_requirements, ) + ) diff --git a/python/pip_install/parse_requirements_to_bzl/__main__.py b/python/pip_install/parse_requirements_to_bzl/__main__.py index 89199612b5..4ca73867a7 100644 --- a/python/pip_install/parse_requirements_to_bzl/__main__.py +++ b/python/pip_install/parse_requirements_to_bzl/__main__.py @@ -1,5 +1,16 @@ """Main entry point.""" +import os +import sys + from python.pip_install.parse_requirements_to_bzl import main if __name__ == "__main__": - main() + # Under `bazel run`, just print the generated starlark code. + # This allows users to check that into their repository rather than + # call pip_parse to generate as a repository rule. + if "BUILD_WORKING_DIRECTORY" in os.environ: + os.chdir(os.environ["BUILD_WORKING_DIRECTORY"]) + main(sys.stdout) + else: + with open("requirements.bzl", "w") as requirement_file: + main(requirement_file) diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 51d4fb558d..e587f34c73 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -125,9 +125,6 @@ def _pip_repository_impl(rctx): if rctx.attr.incremental and not rctx.attr.requirements_lock: fail("Incremental mode requires a requirements_lock attribute be specified.") - # We need a BUILD file to load the generated requirements.bzl - rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) - # Write the annotations file to pass to the wheel maker annotations = {package: json.decode(data) for (package, data) in rctx.attr.annotations.items()} annotations_file = rctx.path("annotations.json") @@ -152,7 +149,7 @@ def _pip_repository_impl(rctx): args += ["--python_interpreter", _get_python_interpreter_attr(rctx)] if rctx.attr.python_interpreter_target: args += ["--python_interpreter_target", str(rctx.attr.python_interpreter_target)] - + progress_message = "Parsing requirements to starlark" else: args = [ python_interpreter, @@ -163,10 +160,13 @@ def _pip_repository_impl(rctx): "--annotations", annotations_file, ] + progress_message = "Extracting wheels" args += ["--repo", rctx.attr.name, "--repo-prefix", rctx.attr.repo_prefix] args = _parse_optional_attrs(rctx, args) + rctx.report_progress(progress_message) + result = rctx.execute( args, # Manually construct the PYTHONPATH since we cannot use the toolchain here @@ -178,6 +178,9 @@ def _pip_repository_impl(rctx): if result.return_code: fail("rules_python failed: %s (%s)" % (result.stdout, result.stderr)) + # We need a BUILD file to load the generated requirements.bzl + rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS + "\n# The requirements.bzl file was generated by running:\n# " + " ".join([str(a) for a in args])) + return common_env = [