From fdb73b02f8d137877614582ad11e2e2b0a6f3ece Mon Sep 17 00:00:00 2001 From: Kevin Lin Date: Sat, 16 Sep 2023 16:10:15 -0700 Subject: [PATCH 1/5] Support netrc-based authentication for python_repository rule --- python/repositories.bzl | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/python/repositories.bzl b/python/repositories.bzl index fbe23bc2e3..b6b65581cd 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -18,7 +18,7 @@ For historic reasons, pip_repositories() is defined in //python:pip.bzl. """ load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archive") -load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe", "read_netrc", "read_user_netrc", "use_netrc") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:coverage_deps.bzl", "coverage_dep") load( @@ -97,11 +97,23 @@ def _python_repository_impl(rctx): release_filename = rctx.attr.release_filename urls = rctx.attr.urls or [rctx.attr.url] + if rctx.attr.netrc: + netrc = read_netrc(rctx, rctx.attr.netrc) + elif "NETRC" in rctx.os.environ: + netrc = read_netrc(rctx, rctx.os.environ["NETRC"]) + else: + netrc = read_user_netrc(rctx) + + auth = None + if netrc: + auth = use_netrc(netrc, urls, rctx.attr.auth_patterns) + if release_filename.endswith(".zst"): rctx.download( url = urls, sha256 = rctx.attr.sha256, output = release_filename, + auth = auth, ) unzstd = rctx.which("unzstd") if not unzstd: @@ -109,6 +121,7 @@ def _python_repository_impl(rctx): rctx.download_and_extract( url = url, sha256 = rctx.attr.zstd_sha256, + auth = auth, ) working_directory = "zstd-{version}".format(version = rctx.attr.zstd_version) @@ -146,6 +159,7 @@ def _python_repository_impl(rctx): url = urls, sha256 = rctx.attr.sha256, stripPrefix = rctx.attr.strip_prefix, + auth = auth, ) patches = rctx.attr.patches @@ -442,6 +456,12 @@ For more information see the official bazel docs "urls": attr.string_list( doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", ), + "netrc": attr.string( + doc = ".netrc file to use for authentication; mirrors the eponymous attribute from http_archive", + ), + "auth_patterns": attr.string_dict( + doc = "Override mapping of hostnames to authorization patterns; mirrors the eponymous attribute from http_archive", + ), "zstd_sha256": attr.string( default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0", ), From 6472028a05aa6915ae51c9b5dd955299bd552864 Mon Sep 17 00:00:00 2001 From: Kevin Lin Date: Sat, 16 Sep 2023 16:18:34 -0700 Subject: [PATCH 2/5] Update CHANGELOG.md --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1385adeee0..e319d28fb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,9 @@ A brief description of the categories of changes: * (gazelle) New `# gazelle:python_generation_mode file` directive to support generating one `py_library` per file. +* (python_repository) Support `netrc` and `auth_patterns` attributes to enable + authentication against private HTTP hosts serving Python toolchain binaries. + ### Removed * (bzlmod) The `entry_point` macro is no longer supported and has been removed @@ -118,5 +121,3 @@ A brief description of the categories of changes: * Expose Python C headers through the toolchain. [0.24.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.24.0 - - From 2d542ad269a0eb880f6443d6f02c090828dd7a97 Mon Sep 17 00:00:00 2001 From: Kevin Lin Date: Sat, 16 Sep 2023 16:26:02 -0700 Subject: [PATCH 3/5] Fix build errors --- python/repositories.bzl | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/python/repositories.bzl b/python/repositories.bzl index b6b65581cd..c363b7b68b 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -103,10 +103,7 @@ def _python_repository_impl(rctx): netrc = read_netrc(rctx, rctx.os.environ["NETRC"]) else: netrc = read_user_netrc(rctx) - - auth = None - if netrc: - auth = use_netrc(netrc, urls, rctx.attr.auth_patterns) + auth = use_netrc(netrc, urls, rctx.attr.auth_patterns) if release_filename.endswith(".zst"): rctx.download( @@ -386,6 +383,9 @@ python_repository = repository_rule( _python_repository_impl, doc = "Fetches the external tools needed for the Python toolchain.", attrs = { + "auth_patterns": attr.string_dict( + doc = "Override mapping of hostnames to authorization patterns; mirrors the eponymous attribute from http_archive", + ), "coverage_tool": attr.string( # Mirrors the definition at # https://github.com/bazelbuild/bazel/blob/master/src/main/starlark/builtins_bzl/common/python/py_runtime_rule.bzl @@ -426,6 +426,9 @@ For more information see the official bazel docs doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.", mandatory = False, ), + "netrc": attr.string( + doc = ".netrc file to use for authentication; mirrors the eponymous attribute from http_archive", + ), "patches": attr.label_list( doc = "A list of patch files to apply to the unpacked interpreter", mandatory = False, @@ -456,12 +459,6 @@ For more information see the official bazel docs "urls": attr.string_list( doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", ), - "netrc": attr.string( - doc = ".netrc file to use for authentication; mirrors the eponymous attribute from http_archive", - ), - "auth_patterns": attr.string_dict( - doc = "Override mapping of hostnames to authorization patterns; mirrors the eponymous attribute from http_archive", - ), "zstd_sha256": attr.string( default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0", ), From e9b67ad8fba8ab4000ae6c91de7aa1ec54e75d96 Mon Sep 17 00:00:00 2001 From: Kevin Lin Date: Thu, 21 Sep 2023 13:56:39 -0700 Subject: [PATCH 4/5] Copy _get_auth helper from Bazel, update attrs in return --- python/repositories.bzl | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/python/repositories.bzl b/python/repositories.bzl index c363b7b68b..84a3d5bd9e 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -85,6 +85,31 @@ def is_standalone_interpreter(rctx, python_interpreter_path): ), ]).return_code == 0 +def _get_auth(rctx, urls): + """ + Convenience utility for automatically retrieving netrc-based authentication parameters for + repository download rules used in python_repository. + + The implementation below is copied directly from Bazel's implementation of `http_archive`. + Accordingly, the return value of this function should be used identically as the `auth` + parameter of `http_archive`. + Reference: https://github.com/bazelbuild/bazel/blob/6.3.2/tools/build_defs/repo/http.bzl#L109 + + Args: + rctx (repository_ctx): The repository rule's context object. + urls: A list of URLs from which assets will be downloaded. + + Returns: + dict: A map of authentication parameters by URL. + """ + if rctx.attr.netrc: + netrc = read_netrc(rctx, rctx.attr.netrc) + elif "NETRC" in rctx.os.environ: + netrc = read_netrc(rctx, rctx.os.environ["NETRC"]) + else: + netrc = read_user_netrc(rctx) + return use_netrc(netrc, urls, rctx.attr.auth_patterns) + def _python_repository_impl(rctx): if rctx.attr.distutils and rctx.attr.distutils_content: fail("Only one of (distutils, distutils_content) should be set.") @@ -96,14 +121,7 @@ def _python_repository_impl(rctx): python_short_version = python_version.rpartition(".")[0] release_filename = rctx.attr.release_filename urls = rctx.attr.urls or [rctx.attr.url] - - if rctx.attr.netrc: - netrc = read_netrc(rctx, rctx.attr.netrc) - elif "NETRC" in rctx.os.environ: - netrc = read_netrc(rctx, rctx.os.environ["NETRC"]) - else: - netrc = read_user_netrc(rctx) - auth = use_netrc(netrc, urls, rctx.attr.auth_patterns) + auth = _get_auth(rctx, urls) if release_filename.endswith(".zst"): rctx.download( @@ -359,11 +377,13 @@ py_cc_toolchain( rctx.file("BUILD.bazel", build_content) attrs = { + "auth_patterns": rctx.attr.auth_patterns, "coverage_tool": rctx.attr.coverage_tool, "distutils": rctx.attr.distutils, "distutils_content": rctx.attr.distutils_content, "ignore_root_user_error": rctx.attr.ignore_root_user_error, "name": rctx.attr.name, + "netrc": rctx.attr.netrc, "patches": rctx.attr.patches, "platform": platform, "python_version": python_version, From b6cbea36e235228dd3e70ba646e9d44bd48d54d8 Mon Sep 17 00:00:00 2001 From: Kevin Lin Date: Thu, 21 Sep 2023 14:07:01 -0700 Subject: [PATCH 5/5] Lint: function-docstring-header --- python/repositories.bzl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/python/repositories.bzl b/python/repositories.bzl index 84a3d5bd9e..ea4a9275db 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -86,13 +86,10 @@ def is_standalone_interpreter(rctx, python_interpreter_path): ]).return_code == 0 def _get_auth(rctx, urls): - """ - Convenience utility for automatically retrieving netrc-based authentication parameters for - repository download rules used in python_repository. + """Utility for retrieving netrc-based authentication parameters for repository download rules used in python_repository. The implementation below is copied directly from Bazel's implementation of `http_archive`. - Accordingly, the return value of this function should be used identically as the `auth` - parameter of `http_archive`. + Accordingly, the return value of this function should be used identically as the `auth` parameter of `http_archive`. Reference: https://github.com/bazelbuild/bazel/blob/6.3.2/tools/build_defs/repo/http.bzl#L109 Args: