diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 938ba85dd5..04f51ed04d 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -22,6 +22,7 @@ package(default_visibility = ["//visibility:public"]) licenses(["notice"]) # Apache 2.0 _DOCS = { + "extensions": "//docs:extensions-docs", "packaging": "//docs:packaging-docs", "pip": "//docs:pip-docs", "pip_repository": "//docs:pip-repository", @@ -101,6 +102,17 @@ stardoc( deps = [":defs"], ) +stardoc( + name = "extensions-docs", + out = "extensions.md_", + input = "//python:extensions.bzl", + target_compatible_with = _NOT_WINDOWS, + deps = [ + ":pip_install_bzl", + "@bazel_skylib//lib:versions", + ], +) + stardoc( name = "pip-docs", out = "pip.md_", diff --git a/docs/extensions.md b/docs/extensions.md new file mode 100644 index 0000000000..d7dd43c481 --- /dev/null +++ b/docs/extensions.md @@ -0,0 +1,40 @@ + + +Module extensions for use with bzlmod. + +# python + +## python.toolchain + +TODO + +# pip + +## pip.parse + +You can use the `pip_parse` to access the generate entry_point targets as follows. +First, ensure you use the `incompatible_generate_aliases=True` feature to re-export the +external spoke repository contents in distinct folders in the hub repo: +```starlark +pip = use_extension("@rules_python//python:extensions.bzl", "pip") +pip.parse( + name = "pypi", + # Generate aliases for more ergonomic consumption of dependencies from + # the `pypi` external repo. + incompatible_generate_aliases = True, + requirements_lock = "//:requirements_lock.txt", + requirements_windows = "//:requirements_windows.txt", +) +use_repo(pip, "pip") +``` + +Then, similarly to the legacy usage, you can create an alias for the `flake8` entry_point: +```starlark +load("@pypi//flake8:bin.bzl", "bin") + +alias( + name = "flake8", + actual = bin.flake8, +) +``` + diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel index 7ecc035853..6952e9b4c0 100644 --- a/examples/bzlmod/BUILD.bazel +++ b/examples/bzlmod/BUILD.bazel @@ -1,4 +1,5 @@ load("@pip//:requirements.bzl", "requirement") +load("@pip//yamllint:bin.bzl", yamllint_bin = "bin") load("@python3_9//:defs.bzl", py_test_with_transition = "py_test") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") @@ -43,3 +44,11 @@ py_test_with_transition( main = "test.py", deps = [":lib"], ) + +alias( + name = "yamllint", + # This is using the struct defined by the spoke repo 'yamllint' and it is + # re-exported by the hub repo named 'pip'. This allows bzlmod and + # non-bzlmod users to access the entry point targets. + actual = yamllint_bin.yamllint, +) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index ce9122810c..84af6dfed1 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -26,6 +26,9 @@ register_toolchains( pip = use_extension("@rules_python//python:extensions.bzl", "pip") pip.parse( name = "pip", + # Generate user friendly alias labels for each dependency that we have and + # enable entry_point support + incompatible_generate_aliases = True, requirements_lock = "//:requirements_lock.txt", requirements_windows = "//:requirements_windows.txt", ) diff --git a/internal_deps.bzl b/internal_deps.bzl index e4d2f69d41..93dac7bd38 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl @@ -53,9 +53,11 @@ def rules_python_internal_deps(): maybe( http_archive, name = "io_bazel_stardoc", - url = "https://github.com/bazelbuild/stardoc/archive/6f274e903009158504a9d9130d7f7d5f3e9421ed.tar.gz", - sha256 = "b5d6891f869d5b5a224316ec4dd9e9d481885a9b1a1c81eb846e20180156f2fa", - strip_prefix = "stardoc-6f274e903009158504a9d9130d7f7d5f3e9421ed", + sha256 = "3fd8fec4ddec3c670bd810904e2e33170bedfe12f90adf943508184be458c8bb", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz", + "https://github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz", + ], ) # The below two deps are required for the integration test with bazel diff --git a/python/BUILD.bazel b/python/BUILD.bazel index d75889d188..c80184499a 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -41,7 +41,7 @@ filegroup( visibility = ["//:__pkg__"], ) -# ========= bzl_library targets end ========= +# ========= bzl_library targets start ========= bzl_library( name = "current_py_toolchain_bzl", @@ -140,6 +140,7 @@ filegroup( name = "bzl", srcs = [ "defs.bzl", + "extensions.bzl", "packaging.bzl", "pip.bzl", "repositories.bzl", diff --git a/python/extensions.bzl b/python/extensions.bzl index 2b0c188554..2fbead451d 100644 --- a/python/extensions.bzl +++ b/python/extensions.bzl @@ -12,7 +12,43 @@ # See the License for the specific language governing permissions and # limitations under the License. -"Module extensions for use with bzlmod" +"""Module extensions for use with bzlmod. + +# python + +## python.toolchain + +TODO + +# pip + +## pip.parse + +You can use the `pip_parse` to access the generate entry_point targets as follows. +First, ensure you use the `incompatible_generate_aliases=True` feature to re-export the +external spoke repository contents in distinct folders in the hub repo: +```starlark +pip = use_extension("@rules_python//python:extensions.bzl", "pip") +pip.parse( + name = "pypi", + # Generate aliases for more ergonomic consumption of dependencies from + # the `pypi` external repo. + incompatible_generate_aliases = True, + requirements_lock = "//:requirements_lock.txt", + requirements_windows = "//:requirements_windows.txt", +) +use_repo(pip, "pip") +``` + +Then, similarly to the legacy usage, you can create an alias for the `flake8` entry_point: +```starlark +load("@pypi//flake8:bin.bzl", "bin") + +alias( + name = "flake8", + actual = bin.flake8, +) +```""" load("@rules_python//python:repositories.bzl", "python_register_toolchains") load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "pip_repository_bzlmod", "use_isolated", "whl_library") @@ -126,4 +162,5 @@ pip = module_extension( tag_classes = { "parse": tag_class(attrs = _pip_parse_ext_attrs()), }, + doc = "NOTE @aignas 2023-05-01: This will not appear in the docs generated with stardoc 0.5.3", ) diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index f58c2afddb..9f53d9b0bf 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -307,6 +307,33 @@ alias( ) rctx.file("{}/BUILD.bazel".format(name), build_content) + # Re-export the opaque struct that contains labels to all of the + # entrypoint labels available in the package repository. + # + # NOTE: We have to do this because the hub repo created by `pip_parse` + # does not know about the existing entrypoints and therefor cannot + # generate the aliases for them. What it knows though is that the + # entrypoint labels will be available as a struct in a particular file + # when the external repository contents will be fetched from PyPI (or + # another index server). + # + # What is more, we should not put re-exports of all available `bin` + # structs into a single file, because that would mean eager fetches of + # all external repos even though the users are using only a single + # entry_point (or none at all, if the re-exports are done in the + # `requirements.bzl` and the user wants to use the `requirement` + # macro). + bin_content = """\ +\"\"\"Autogenerated by 'pip_repository.bzl#_pkg_aliases'.\"\"\" +load("@{repo_name}_{dep}//:bin.bzl", _bin = "bin") + +bin = _bin +""".format( + repo_name = repo_name, + dep = name, + ) + rctx.file("{}/bin.bzl".format(name), bin_content) + def _bzlmod_pkg_aliases(repo_name, bzl_packages): """Create alias declarations for each python dependency. diff --git a/python/pip_install/tools/wheel_installer/wheel_installer.py b/python/pip_install/tools/wheel_installer/wheel_installer.py index 77aa3a406c..c01eaceb8a 100644 --- a/python/pip_install/tools/wheel_installer/wheel_installer.py +++ b/python/pip_install/tools/wheel_installer/wheel_installer.py @@ -22,6 +22,7 @@ import subprocess import sys import textwrap +from dataclasses import dataclass from pathlib import Path from tempfile import NamedTemporaryFile from typing import Dict, Iterable, List, Optional, Set, Tuple @@ -289,6 +290,31 @@ def _generate_build_file_contents( ) +def _generate_bin_bzl_contents(entrypoints: Dict[str, str]) -> str: + """Generate the contents of bin.bzl for each package. + + The presence of the `bin.bzl` and the struct in it named `bin` becomes the API to + the users and to the hub repo on how to consume the labels to the entry_point + targets. + """ + struct_params = sorted( + [ + '{attr} = Label("//:{target_name}"),'.format( + attr=attr, target_name=target_name + ) + for attr, target_name in entrypoints.items() + ] + ) + if not struct_params: + struct_def = "struct()" + else: + struct_def = "struct(\n {}\n)" "".format("\n ".join(struct_params)) + + return "\n# Autogenerated by wheel_installer.py#_generate_bin_bzl_contents\n\nbin = {}\n".format( + struct_def + ) + + def _extract_wheel( wheel_file: str, extras: Dict[str, Set[str]], @@ -329,6 +355,7 @@ def _extract_wheel( ] entry_points = [] + entry_point_targets = {} for name, (module, attribute) in sorted(whl.entry_points().items()): # There is an extreme edge-case with entry_points that end with `.py` # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 @@ -340,6 +367,8 @@ def _extract_wheel( (installation_dir / entry_point_script_name).write_text( _generate_entry_point_contents(module, attribute) ) + + entry_point_targets[entry_point_without_py] = entry_point_target_name entry_points.append( _generate_entry_point_rule( entry_point_target_name, @@ -348,6 +377,9 @@ def _extract_wheel( ) ) + with open(os.path.join(installation_dir, "bin.bzl"), "w") as bin_bzl: + bin_bzl.write(_generate_bin_bzl_contents(entry_point_targets)) + with open(os.path.join(installation_dir, "BUILD.bazel"), "w") as build_file: additional_content = entry_points data = []