From f12f5c32d17753919fd9489ab5251055724da9d7 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 29 Oct 2024 21:04:52 -0700 Subject: [PATCH 01/34] wip: helper to run an arbitrary interpreter or interpreter from a binary Run a specific interpreter: * `bazel run @rules_python//tools/run --@rules_python//python/config_settings:python_version=3.12` Run interpreter from a binary: * `bazel run @rules_python//tools/run --@rules_python//tools/run:bin=//my:binary` --- tools/run/BUILD.bazel | 10 ++++++++++ tools/run/run.bzl | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 tools/run/BUILD.bazel create mode 100644 tools/run/run.bzl diff --git a/tools/run/BUILD.bazel b/tools/run/BUILD.bazel new file mode 100644 index 0000000000..941d06e52d --- /dev/null +++ b/tools/run/BUILD.bazel @@ -0,0 +1,10 @@ +load(":run.bzl", "interpreter") + +interpreter( + name = "run", +) + +label_flag( + name = "bin", + build_setting_default = "//python:none", +) diff --git a/tools/run/run.bzl b/tools/run/run.bzl new file mode 100644 index 0000000000..786d4b5324 --- /dev/null +++ b/tools/run/run.bzl @@ -0,0 +1,40 @@ +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("//python:py_runtime_info.bzl", "PyRuntimeInfo") +load("//python/private:sentinel.bzl", "SentinelInfo") +load("//python/private:toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") + +def _interpreter_impl(ctx): + if SentinelInfo in ctx.attr.binary: + toolchain = ctx.toolchains[TARGET_TOOLCHAIN_TYPE] + runtime = toolchain.py3_runtime + else: + runtime = ctx.attr.binary[PyRuntimeInfo] + + # NOTE: We name the output filename after the underlying file name + # because of things like pyenv: they use $0 to determine what to + # re-exec. If it's not a recognized name, then they fail. + if runtime.interpreter: + executable = ctx.actions.declare_file(runtime.interpreter.basename) + ctx.actions.symlink(output = executable, target_file = runtime.interpreter, is_executable = True) + else: + executable = ctx.actions.declare_symlink(paths.basename(runtime.interpreter_path)) + ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path) + + return [ + DefaultInfo( + executable = executable, + runfiles = ctx.runfiles([executable], transitive_files = runtime.files), + ), + ] + +interpreter = rule( + implementation = _interpreter_impl, + toolchains = [TARGET_TOOLCHAIN_TYPE], + executable = True, + attrs = { + "binary": attr.label( + default = "//tools/run:bin", + ), + }, +) From 75076262173f614f80a103bd60e8fe5167cea084 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Wed, 27 Nov 2024 12:17:19 -0800 Subject: [PATCH 02/34] Basic REPL support $ bazel run //python/bin:repl $ bazel run //python/bin:repl --//python/bin:repl_dep=//python/runfiles --- python/bin/BUILD.bazel | 14 ++++++++++++++ python/bin/repl.py | 17 +++++++++++++++++ python/private/sentinel.bzl | 4 +++- 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 python/bin/BUILD.bazel create mode 100644 python/bin/repl.py diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel new file mode 100644 index 0000000000..a1ba9e6698 --- /dev/null +++ b/python/bin/BUILD.bazel @@ -0,0 +1,14 @@ +load("//python:defs.bzl", "py_binary") + +py_binary( + name = "repl", + srcs = ["repl.py"], + deps = [ + ":repl_dep", + ], +) + +label_flag( + name = "repl_dep", + build_setting_default = "//python:none", +) diff --git a/python/bin/repl.py b/python/bin/repl.py new file mode 100644 index 0000000000..00c7dec02f --- /dev/null +++ b/python/bin/repl.py @@ -0,0 +1,17 @@ +import code +import os +from pathlib import Path + +# Manually implement PYTHONSTARTUP support. We can't just invoke the python +# binary directly as it would skip the bootstrap scripts. +python_startup = os.getenv("PYTHONSTARTUP") +if python_startup: + try: + source = Path(python_startup).read_text() + except Exception as error: + print(f"{type(error).__name__}: {error}") + else: + compiled_code = compile(source, filename=python_startup, mode="exec") + eval(compiled_code, {}) + +code.InteractiveConsole().interact() diff --git a/python/private/sentinel.bzl b/python/private/sentinel.bzl index 6d753e1983..4e4d8e78a2 100644 --- a/python/private/sentinel.bzl +++ b/python/private/sentinel.bzl @@ -18,6 +18,8 @@ Label attributes with defaults cannot accept None, otherwise they fall back to using the default. A sentinel allows detecting an intended None value. """ +load(":py_info.bzl", "PyInfo") + SentinelInfo = provider( doc = "Indicates this was the sentinel target.", fields = [], @@ -25,6 +27,6 @@ SentinelInfo = provider( def _sentinel_impl(ctx): _ = ctx # @unused - return [SentinelInfo()] + return [SentinelInfo(), PyInfo(transitive_sources=depset())] sentinel = rule(implementation = _sentinel_impl) From 9730855c9c965798ce3a022ad087cef4002a3f8c Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 1 Dec 2024 11:39:26 -0800 Subject: [PATCH 03/34] experiment with site packages --- python/private/site_init_template.py | 1 + python/private/stage2_bootstrap_template.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/python/private/site_init_template.py b/python/private/site_init_template.py index 7a32210bff..e6eadfe8be 100644 --- a/python/private/site_init_template.py +++ b/python/private/site_init_template.py @@ -27,6 +27,7 @@ # Runfiles-relative path to the coverage tool entry point, if any. _COVERAGE_TOOL = "%coverage_tool%" +print("Hi") def _is_verbose(): return bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE")) diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index d2c7497795..dcbcb253b8 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -24,6 +24,9 @@ # Runfiles-relative path to the main Python source file. MAIN = "%main%" +# Whether this script is used as a sitecustomize script. +USED_AS_SITECUSTOMIZE = "%used_as_sitecustomize%" + # ===== Template substitutions end ===== @@ -375,6 +378,8 @@ def main(): if runfiles_envkey: os.environ[runfiles_envkey] = runfiles_envvalue + sys.path[0:0] = prepend_path_entries + main_filename = os.path.join(module_space, main_rel_path) main_filename = get_windows_path_with_unc_prefix(main_filename) assert os.path.exists(main_filename), ( @@ -386,8 +391,6 @@ def main(): sys.stdout.flush() - sys.path[0:0] = prepend_path_entries - if os.environ.get("COVERAGE_DIR"): import _bazel_site_init coverage_enabled = _bazel_site_init.COVERAGE_SETUP From 286b4f1400cddb3f8cba2b708e4d641988a9655d Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 1 Dec 2024 12:30:57 -0800 Subject: [PATCH 04/34] consolidate files a bit --- python/bin/BUILD.bazel | 10 ++++++++++ tools/run/run.bzl => python/bin/interpreter.bzl | 0 tools/run/BUILD.bazel | 10 ---------- 3 files changed, 10 insertions(+), 10 deletions(-) rename tools/run/run.bzl => python/bin/interpreter.bzl (100%) delete mode 100644 tools/run/BUILD.bazel diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index a1ba9e6698..5745fc8eee 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -1,4 +1,14 @@ load("//python:defs.bzl", "py_binary") +load(":interpreter.bzl", "interpreter") + +interpreter( + name = "interpreter", +) + +label_flag( + name = "bin", + build_setting_default = "//python:none", +) py_binary( name = "repl", diff --git a/tools/run/run.bzl b/python/bin/interpreter.bzl similarity index 100% rename from tools/run/run.bzl rename to python/bin/interpreter.bzl diff --git a/tools/run/BUILD.bazel b/tools/run/BUILD.bazel deleted file mode 100644 index 941d06e52d..0000000000 --- a/tools/run/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -load(":run.bzl", "interpreter") - -interpreter( - name = "run", -) - -label_flag( - name = "bin", - build_setting_default = "//python:none", -) From 314447d6f634539c38ef213544b36370db383274 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 15 Dec 2024 15:37:57 -0800 Subject: [PATCH 05/34] revert REPL-related things --- python/bin/BUILD.bazel | 13 ------------- python/bin/repl.py | 17 ----------------- python/private/sentinel.bzl | 4 +--- python/private/site_init_template.py | 1 - python/private/stage2_bootstrap_template.py | 7 ++----- 5 files changed, 3 insertions(+), 39 deletions(-) delete mode 100644 python/bin/repl.py diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index 5745fc8eee..4ee55b512a 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -9,16 +9,3 @@ label_flag( name = "bin", build_setting_default = "//python:none", ) - -py_binary( - name = "repl", - srcs = ["repl.py"], - deps = [ - ":repl_dep", - ], -) - -label_flag( - name = "repl_dep", - build_setting_default = "//python:none", -) diff --git a/python/bin/repl.py b/python/bin/repl.py deleted file mode 100644 index 00c7dec02f..0000000000 --- a/python/bin/repl.py +++ /dev/null @@ -1,17 +0,0 @@ -import code -import os -from pathlib import Path - -# Manually implement PYTHONSTARTUP support. We can't just invoke the python -# binary directly as it would skip the bootstrap scripts. -python_startup = os.getenv("PYTHONSTARTUP") -if python_startup: - try: - source = Path(python_startup).read_text() - except Exception as error: - print(f"{type(error).__name__}: {error}") - else: - compiled_code = compile(source, filename=python_startup, mode="exec") - eval(compiled_code, {}) - -code.InteractiveConsole().interact() diff --git a/python/private/sentinel.bzl b/python/private/sentinel.bzl index 4e4d8e78a2..6d753e1983 100644 --- a/python/private/sentinel.bzl +++ b/python/private/sentinel.bzl @@ -18,8 +18,6 @@ Label attributes with defaults cannot accept None, otherwise they fall back to using the default. A sentinel allows detecting an intended None value. """ -load(":py_info.bzl", "PyInfo") - SentinelInfo = provider( doc = "Indicates this was the sentinel target.", fields = [], @@ -27,6 +25,6 @@ SentinelInfo = provider( def _sentinel_impl(ctx): _ = ctx # @unused - return [SentinelInfo(), PyInfo(transitive_sources=depset())] + return [SentinelInfo()] sentinel = rule(implementation = _sentinel_impl) diff --git a/python/private/site_init_template.py b/python/private/site_init_template.py index e6eadfe8be..7a32210bff 100644 --- a/python/private/site_init_template.py +++ b/python/private/site_init_template.py @@ -27,7 +27,6 @@ # Runfiles-relative path to the coverage tool entry point, if any. _COVERAGE_TOOL = "%coverage_tool%" -print("Hi") def _is_verbose(): return bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE")) diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index dcbcb253b8..d2c7497795 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -24,9 +24,6 @@ # Runfiles-relative path to the main Python source file. MAIN = "%main%" -# Whether this script is used as a sitecustomize script. -USED_AS_SITECUSTOMIZE = "%used_as_sitecustomize%" - # ===== Template substitutions end ===== @@ -378,8 +375,6 @@ def main(): if runfiles_envkey: os.environ[runfiles_envkey] = runfiles_envvalue - sys.path[0:0] = prepend_path_entries - main_filename = os.path.join(module_space, main_rel_path) main_filename = get_windows_path_with_unc_prefix(main_filename) assert os.path.exists(main_filename), ( @@ -391,6 +386,8 @@ def main(): sys.stdout.flush() + sys.path[0:0] = prepend_path_entries + if os.environ.get("COVERAGE_DIR"): import _bazel_site_init coverage_enabled = _bazel_site_init.COVERAGE_SETUP From e07a528ef5e8c36e956b636af0568de2efaf8e16 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 15 Dec 2024 18:15:14 -0800 Subject: [PATCH 06/34] add integration test --- .bazelrc | 4 +- python/BUILD.bazel | 1 + python/bin/BUILD.bazel | 13 +++- python/bin/interpreter.bzl | 2 +- tests/integration/BUILD.bazel | 5 ++ tests/integration/interpreter/BUILD.bazel | 13 ++++ tests/integration/interpreter/MODULE.bazel | 21 +++++++ tests/integration/interpreter/WORKSPACE | 0 tests/integration/interpreter_test.py | 72 ++++++++++++++++++++++ tests/integration/runner.py | 3 +- 10 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 tests/integration/interpreter/BUILD.bazel create mode 100644 tests/integration/interpreter/MODULE.bazel create mode 100644 tests/integration/interpreter/WORKSPACE create mode 100644 tests/integration/interpreter_test.py diff --git a/.bazelrc b/.bazelrc index ada5c5a0a7..bfb07ced41 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/interpreter,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/interpreter,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered test --test_output=errors diff --git a/python/BUILD.bazel b/python/BUILD.bazel index b747e2fbc7..8ffa166728 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -35,6 +35,7 @@ filegroup( name = "distribution", srcs = glob(["**"]) + [ "//python/api:distribution", + "//python/bin:distribution", "//python/cc:distribution", "//python/config_settings:distribution", "//python/constraints:distribution", diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index 4ee55b512a..309e0e11bf 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -1,11 +1,22 @@ load("//python:defs.bzl", "py_binary") load(":interpreter.bzl", "interpreter") +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "distribution", + srcs = glob(["**"]), + visibility = ["//python:__pkg__"], +) + interpreter( name = "interpreter", + binary = ":interpreter_src", ) +# The user can modify this flag to source different interpreters for the +# `interpreter` target above. label_flag( - name = "bin", + name = "interpreter_src", build_setting_default = "//python:none", ) diff --git a/python/bin/interpreter.bzl b/python/bin/interpreter.bzl index 786d4b5324..f73d32362f 100644 --- a/python/bin/interpreter.bzl +++ b/python/bin/interpreter.bzl @@ -34,7 +34,7 @@ interpreter = rule( executable = True, attrs = { "binary": attr.label( - default = "//tools/run:bin", + mandatory = True, ), }, ) diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index d178e0f01c..637420a634 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -120,6 +120,11 @@ rules_python_integration_test( py_main = "custom_commands_test.py", ) +rules_python_integration_test( + name = "interpreter_test", + py_main = "interpreter_test.py", +) + py_library( name = "runner_lib", srcs = ["runner.py"], diff --git a/tests/integration/interpreter/BUILD.bazel b/tests/integration/interpreter/BUILD.bazel new file mode 100644 index 0000000000..c9b29fc948 --- /dev/null +++ b/tests/integration/interpreter/BUILD.bazel @@ -0,0 +1,13 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/integration/interpreter/MODULE.bazel b/tests/integration/interpreter/MODULE.bazel new file mode 100644 index 0000000000..5bea8126aa --- /dev/null +++ b/tests/integration/interpreter/MODULE.bazel @@ -0,0 +1,21 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module(name = "module_under_test") + +bazel_dep(name = "rules_python", version = "0.0.0") +local_path_override( + module_name = "rules_python", + path = "../../..", +) diff --git a/tests/integration/interpreter/WORKSPACE b/tests/integration/interpreter/WORKSPACE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/interpreter_test.py b/tests/integration/interpreter_test.py new file mode 100644 index 0000000000..bce33f26f3 --- /dev/null +++ b/tests/integration/interpreter_test.py @@ -0,0 +1,72 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import unittest + +from tests.integration import runner + + +class InterpreterTest(runner.TestCase): + def _run_version_test(self, expected_version): + """Validates that we can successfully execute arbitrary code from the CLI.""" + result = self.run_bazel( + "run", + f"--@rules_python//python/config_settings:python_version={expected_version}", + "@rules_python//python/bin:interpreter", + input = "\r".join([ + "import sys", + "v = sys.version_info", + "print(f'version: {v.major}.{v.minor}')", + ]), + ) + self.assert_result_matches(result, f"version: {expected_version}") + + def test_run_interpreter_3_10(self): + self._run_version_test("3.10") + + def test_run_interpreter_3_11(self): + self._run_version_test("3.11") + + def test_run_interpreter_3_12(self): + self._run_version_test("3.12") + + def _run_module_test(self, version): + """Validates that we can successfully invoke a module from the CLI.""" + result = self.run_bazel( + "run", + f"--@rules_python//python/config_settings:python_version={version}", + "@rules_python//python/bin:interpreter", + "--", + "-m", + "json.tool", + input = '{"json":"obj"}', + ) + self.assert_result_matches(result, r'{\n "json": "obj"\n}') + + def test_run_module_3_10(self): + self._run_module_test("3.10") + + def test_run_module_3_11(self): + self._run_module_test("3.11") + + def test_run_module_3_12(self): + self._run_module_test("3.12") + + + +if __name__ == "__main__": + # Enabling this makes the runner log subprocesses as the test goes along. + # logging.basicConfig(level = "INFO") + unittest.main() diff --git a/tests/integration/runner.py b/tests/integration/runner.py index 9414a865c0..5895828863 100644 --- a/tests/integration/runner.py +++ b/tests/integration/runner.py @@ -86,7 +86,7 @@ def setUp(self): "RUNFILES_DIR": os.environ["TEST_SRCDIR"] } - def run_bazel(self, *args: str, check: bool = True) -> ExecuteResult: + def run_bazel(self, *args: str, input=None, check: bool = True) -> ExecuteResult: """Run a bazel invocation. Args: @@ -104,6 +104,7 @@ def run_bazel(self, *args: str, check: bool = True) -> ExecuteResult: proc_result = subprocess.run( args=args, text=True, + input=input, capture_output=True, cwd=cwd, env=env, From 91d66df12cf33cf35c6296b1be50120238a661e5 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 15 Dec 2024 19:02:37 -0800 Subject: [PATCH 07/34] add some docs --- docs/toolchains.md | 34 ++++++++++++++++++++++++++- tests/integration/interpreter_test.py | 2 ++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/toolchains.md b/docs/toolchains.md index db4c6ba07a..a018e74a19 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -313,7 +313,7 @@ provide `Python.h`. This is typically implemented using {obj}`py_cc_toolchain()`, which provides {obj}`ToolchainInfo` with the field `py_cc_toolchain` set, which is a -{obj}`PyCcToolchainInfo` provider instance. +{obj}`PyCcToolchainInfo` provider instance. This toolchain type is intended to hold only _target configuration_ values relating to the C/C++ information for the Python runtime. As such, when defining @@ -467,3 +467,35 @@ Currently the following flags are used to influence toolchain selection: * {obj}`--@rules_python//python/config_settings:py_linux_libc` for selecting the Linux libc variant. * {obj}`--@rules_python//python/config_settings:py_freethreaded` for selecting the freethreaded experimental Python builds available from `3.13.0` onwards. + +## Accessing the underlying interpreter + +To access the interpreter that bazel manages, you can use the +`@rules_python//python/bin:interpreter` target. This is a binary target with +the executable pointing at the `python3` binary plus the relevent runfiles. + +``` +$ bazel run @rules_python//python/bin:interpreter +Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +$ bazel run @rules_python//python/bin:interpreter --@rules_python//python/config_settings:python_version=3.12 +Python 3.12.0 (main, Oct 3 2023, 01:27:23) [Clang 17.0.1 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +``` + +You can also access a specific binary's interpreter this way by using the +`@rules_python//python/bin:interpreter_src` target. + +``` +$ bazel run @rules_python//python/bin:interpreter --@rules_python//python/bin:interpreter_src=//path/to:bin +Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +``` + +:::{note} +The interpreter target does not provide access to any modules from `py_*` +targets on its own. Work is ongoing to support that. +::: diff --git a/tests/integration/interpreter_test.py b/tests/integration/interpreter_test.py index bce33f26f3..ab5cd10306 100644 --- a/tests/integration/interpreter_test.py +++ b/tests/integration/interpreter_test.py @@ -44,6 +44,7 @@ def test_run_interpreter_3_12(self): def _run_module_test(self, version): """Validates that we can successfully invoke a module from the CLI.""" + # Pass unformatted JSON to the json.tool module. result = self.run_bazel( "run", f"--@rules_python//python/config_settings:python_version={version}", @@ -53,6 +54,7 @@ def _run_module_test(self, version): "json.tool", input = '{"json":"obj"}', ) + # Validate that we get formatted JSON back. self.assert_result_matches(result, r'{\n "json": "obj"\n}') def test_run_module_3_10(self): From c71629fc342e92d6a4288f9bf58941f7bd6b64cf Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 5 Jan 2025 19:31:31 -0800 Subject: [PATCH 08/34] rename to :python and incorporate readme feedback --- docs/toolchains.md | 16 ++++++++-------- python/bin/BUILD.bazel | 8 ++++---- tests/integration/interpreter_test.py | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/toolchains.md b/docs/toolchains.md index a018e74a19..819b4e9613 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -468,34 +468,34 @@ Currently the following flags are used to influence toolchain selection: * {obj}`--@rules_python//python/config_settings:py_freethreaded` for selecting the freethreaded experimental Python builds available from `3.13.0` onwards. -## Accessing the underlying interpreter +## Running the underlying interpreter -To access the interpreter that bazel manages, you can use the -`@rules_python//python/bin:interpreter` target. This is a binary target with +To run the interpreter that Bazel will use, you can use the +`@rules_python//python/bin:python` target. This is a binary target with the executable pointing at the `python3` binary plus the relevent runfiles. ``` -$ bazel run @rules_python//python/bin:interpreter +$ bazel run @rules_python//python/bin:python Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux Type "help", "copyright", "credits" or "license" for more information. >>> -$ bazel run @rules_python//python/bin:interpreter --@rules_python//python/config_settings:python_version=3.12 +$ bazel run @rules_python//python/bin:python --@rules_python//python/config_settings:python_version=3.12 Python 3.12.0 (main, Oct 3 2023, 01:27:23) [Clang 17.0.1 ] on linux Type "help", "copyright", "credits" or "license" for more information. >>> ``` You can also access a specific binary's interpreter this way by using the -`@rules_python//python/bin:interpreter_src` target. +`@rules_python//python/bin:python_src` target. ``` -$ bazel run @rules_python//python/bin:interpreter --@rules_python//python/bin:interpreter_src=//path/to:bin +$ bazel run @rules_python//python/bin:python --@rules_python//python/bin:interpreter_src=//path/to:bin Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux Type "help", "copyright", "credits" or "license" for more information. >>> ``` :::{note} -The interpreter target does not provide access to any modules from `py_*` +The `python` target does not provide access to any modules from `py_*` targets on its own. Work is ongoing to support that. ::: diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index 309e0e11bf..a90773b743 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -10,13 +10,13 @@ filegroup( ) interpreter( - name = "interpreter", - binary = ":interpreter_src", + name = "python", + binary = ":python_src", ) # The user can modify this flag to source different interpreters for the -# `interpreter` target above. +# `python` target above. label_flag( - name = "interpreter_src", + name = "python_src", build_setting_default = "//python:none", ) diff --git a/tests/integration/interpreter_test.py b/tests/integration/interpreter_test.py index ab5cd10306..a7f015305d 100644 --- a/tests/integration/interpreter_test.py +++ b/tests/integration/interpreter_test.py @@ -24,7 +24,7 @@ def _run_version_test(self, expected_version): result = self.run_bazel( "run", f"--@rules_python//python/config_settings:python_version={expected_version}", - "@rules_python//python/bin:interpreter", + "@rules_python//python/bin:python", input = "\r".join([ "import sys", "v = sys.version_info", @@ -48,7 +48,7 @@ def _run_module_test(self, version): result = self.run_bazel( "run", f"--@rules_python//python/config_settings:python_version={version}", - "@rules_python//python/bin:interpreter", + "@rules_python//python/bin:python", "--", "-m", "json.tool", From 2ee2311120a3591c1b4114c0c4b1e989b7bf24d7 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 5 Jan 2025 19:42:37 -0800 Subject: [PATCH 09/34] move implementation to //python/private --- python/bin/interpreter.bzl | 41 ++-------------------------------- python/private/interpreter.bzl | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 39 deletions(-) create mode 100644 python/private/interpreter.bzl diff --git a/python/bin/interpreter.bzl b/python/bin/interpreter.bzl index f73d32362f..40282e0790 100644 --- a/python/bin/interpreter.bzl +++ b/python/bin/interpreter.bzl @@ -1,40 +1,3 @@ -load("@bazel_skylib//lib:paths.bzl", "paths") -load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") -load("//python:py_runtime_info.bzl", "PyRuntimeInfo") -load("//python/private:sentinel.bzl", "SentinelInfo") -load("//python/private:toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") +load("//python/private:interpreter.bzl", _interpeter="interpreter") -def _interpreter_impl(ctx): - if SentinelInfo in ctx.attr.binary: - toolchain = ctx.toolchains[TARGET_TOOLCHAIN_TYPE] - runtime = toolchain.py3_runtime - else: - runtime = ctx.attr.binary[PyRuntimeInfo] - - # NOTE: We name the output filename after the underlying file name - # because of things like pyenv: they use $0 to determine what to - # re-exec. If it's not a recognized name, then they fail. - if runtime.interpreter: - executable = ctx.actions.declare_file(runtime.interpreter.basename) - ctx.actions.symlink(output = executable, target_file = runtime.interpreter, is_executable = True) - else: - executable = ctx.actions.declare_symlink(paths.basename(runtime.interpreter_path)) - ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path) - - return [ - DefaultInfo( - executable = executable, - runfiles = ctx.runfiles([executable], transitive_files = runtime.files), - ), - ] - -interpreter = rule( - implementation = _interpreter_impl, - toolchains = [TARGET_TOOLCHAIN_TYPE], - executable = True, - attrs = { - "binary": attr.label( - mandatory = True, - ), - }, -) +interpreter = _interpeter diff --git a/python/private/interpreter.bzl b/python/private/interpreter.bzl new file mode 100644 index 0000000000..8754b47cb2 --- /dev/null +++ b/python/private/interpreter.bzl @@ -0,0 +1,40 @@ +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("//python:py_runtime_info.bzl", "PyRuntimeInfo") +load(":sentinel.bzl", "SentinelInfo") +load(":toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") + +def _interpreter_impl(ctx): + if SentinelInfo in ctx.attr.binary: + toolchain = ctx.toolchains[TARGET_TOOLCHAIN_TYPE] + runtime = toolchain.py3_runtime + else: + runtime = ctx.attr.binary[PyRuntimeInfo] + + # NOTE: We name the output filename after the underlying file name + # because of things like pyenv: they use $0 to determine what to + # re-exec. If it's not a recognized name, then they fail. + if runtime.interpreter: + executable = ctx.actions.declare_file(runtime.interpreter.basename) + ctx.actions.symlink(output = executable, target_file = runtime.interpreter, is_executable = True) + else: + executable = ctx.actions.declare_symlink(paths.basename(runtime.interpreter_path)) + ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path) + + return [ + DefaultInfo( + executable = executable, + runfiles = ctx.runfiles([executable], transitive_files = runtime.files), + ), + ] + +interpreter = rule( + implementation = _interpreter_impl, + toolchains = [TARGET_TOOLCHAIN_TYPE], + executable = True, + attrs = { + "binary": attr.label( + mandatory = True, + ), + }, +) From 620bf31e4daecf9ab3c9ec8771a344c2b2622ce8 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Thu, 23 Jan 2025 20:17:07 -0800 Subject: [PATCH 10/34] Add a test using transitions instead of an integration test --- tests/interpreter/BUILD.bazel | 35 ++++++++++++ tests/interpreter/interpreter_test.py | 71 +++++++++++++++++++++++ tests/interpreter/run_json_tool_test.sh | 76 +++++++++++++++++++++++++ 3 files changed, 182 insertions(+) create mode 100644 tests/interpreter/BUILD.bazel create mode 100644 tests/interpreter/interpreter_test.py create mode 100644 tests/interpreter/run_json_tool_test.sh diff --git a/tests/interpreter/BUILD.bazel b/tests/interpreter/BUILD.bazel new file mode 100644 index 0000000000..72c15c3e05 --- /dev/null +++ b/tests/interpreter/BUILD.bazel @@ -0,0 +1,35 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") + +[py_reconfig_test( + name = "interpreter_%s_test" % python_version, + srcs = ["interpreter_test.py"], + main = "interpreter_test.py", + data = [ + "//python/bin:python", + ], + deps = [ + "//python/runfiles", + ], + python_version = python_version, + env = { + "EXPECTED_PYTHON_VERSION": python_version, + }, +) for python_version in ( + "3.10", + "3.11", + "3.12", +)] diff --git a/tests/interpreter/interpreter_test.py b/tests/interpreter/interpreter_test.py new file mode 100644 index 0000000000..1694ddb546 --- /dev/null +++ b/tests/interpreter/interpreter_test.py @@ -0,0 +1,71 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import logging +import unittest +import subprocess + +from python.runfiles import runfiles + + +class InterpreterTest(unittest.TestCase): + def setUp(self): + r = runfiles.Create() + self.interpreter = r.Rlocation("rules_python/python/bin/python3") + + def test_version(self): + """Validates that we can successfully execute arbitrary code from the CLI.""" + expected_version = os.environ["EXPECTED_PYTHON_VERSION"] + + result = subprocess.check_output([self.interpreter], + text = True, + input = "\r".join([ + "import sys", + "v = sys.version_info", + "print(f'version: {v.major}.{v.minor}')", + ]), + ).strip() + self.assertEqual(result, f"version: {expected_version}") + + #def _run_module_test(self, version): + # """Validates that we can successfully invoke a module from the CLI.""" + # # Pass unformatted JSON to the json.tool module. + # result = self.run_bazel( + # "run", + # f"--@rules_python//python/config_settings:python_version={version}", + # "@rules_python//python/bin:python", + # "--", + # "-m", + # "json.tool", + # input = '{"json":"obj"}', + # ) + # # Validate that we get formatted JSON back. + # self.assert_result_matches(result, r'{\n "json": "obj"\n}') + + #def test_run_module_3_10(self): + # self._run_module_test("3.10") + + #def test_run_module_3_11(self): + # self._run_module_test("3.11") + + #def test_run_module_3_12(self): + # self._run_module_test("3.12") + + + +if __name__ == "__main__": + # Enabling this makes the runner log subprocesses as the test goes along. + # logging.basicConfig(level = "INFO") + unittest.main() diff --git a/tests/interpreter/run_json_tool_test.sh b/tests/interpreter/run_json_tool_test.sh new file mode 100644 index 0000000000..728efdd583 --- /dev/null +++ b/tests/interpreter/run_json_tool_test.sh @@ -0,0 +1,76 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# --- begin runfiles.bash initialization v3 --- +# Copy-pasted from the Bazel Bash runfiles library v3. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v3 --- +set +e + +bin=$(rlocation $BIN_RLOCATION) +if [[ -z "$bin" ]]; then + echo "Unable to locate test binary: $BIN_RLOCATION" + exit 1 +fi + +function + +function test_invocation() { + actual=$($bin) + # How we detect if a zip file was executed from depends on which bootstrap + # is used. + # bootstrap_impl=script outputs RULES_PYTHON_ZIP_DIR= + # bootstrap_impl=system_python outputs file:.*Bazel.runfiles + expected_pattern="Hello" + if ! (echo "$actual" | grep "$expected_pattern" ) >/dev/null; then + echo "Test case failed: $1" + echo "expected output to match: $expected_pattern" + echo "but got:\n$actual" + exit 1 + fi +} + +# Test invocation with RUNFILES_DIR set +unset RUNFILES_MANIFEST_FILE +if [[ ! -e "$RUNFILES_DIR" ]]; then + echo "Runfiles doesn't exist: $RUNFILES_DIR" + exit 1 +fi +test_invocation "using RUNFILES_DIR" + + +orig_runfiles_dir="$RUNFILES_DIR" +unset RUNFILES_DIR + +# Test invocation using manifest within runfiles directory (output manifest) +# NOTE: this file may not actually exist in our test, but that's OK; the +# bootstrap just uses the path to find the runfiles directory. +export RUNFILES_MANIFEST_FILE="$orig_runfiles_dir/MANIFEST" +test_invocation "using RUNFILES_MANIFEST_FILE with output manifest" + +# Test invocation using manifest outside runfiles (input manifest) +# NOTE: this file may not actually exist in our test, but that's OK; the +# bootstrap just uses the path to find the runfiles directory. +export RUNFILES_MANIFEST_FILE="${orig_runfiles_dir%%.runfiles}.runfiles_manifest" +test_invocation "using RUNFILES_MANIFEST_FILE with input manifest" + +# Test invocation without any runfiles env vars set +unset RUNFILES_MANIFEST_FILE +test_invocation "using no runfiles env vars" From d4b79788dbcc063f9ceac6f2063381fa57c7c5cc Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Thu, 23 Jan 2025 20:36:26 -0800 Subject: [PATCH 11/34] Add python_src tests --- tests/interpreter/BUILD.bazel | 23 ++++++++++++++- tests/interpreter/bin.py | 1 + tests/interpreter/interpreter_test.py | 40 +++++++++------------------ tests/support/sh_py_run_test.bzl | 6 ++++ 4 files changed, 42 insertions(+), 28 deletions(-) create mode 100644 tests/interpreter/bin.py diff --git a/tests/interpreter/BUILD.bazel b/tests/interpreter/BUILD.bazel index 72c15c3e05..2d1bbf55be 100644 --- a/tests/interpreter/BUILD.bazel +++ b/tests/interpreter/BUILD.bazel @@ -15,7 +15,7 @@ load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") [py_reconfig_test( - name = "interpreter_%s_test" % python_version, + name = "python_version_%s_test" % python_version, srcs = ["interpreter_test.py"], main = "interpreter_test.py", data = [ @@ -33,3 +33,24 @@ load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") "3.11", "3.12", )] + +[py_reconfig_test( + name = "python_src_%s_test" % python_version, + srcs = ["interpreter_test.py"], + main = "interpreter_test.py", + data = [ + "//python/bin:python", + ], + deps = [ + "//python/runfiles", + ], + python_src = "//tools/publish:twine", + python_version = python_version, + env = { + "EXPECTED_PYTHON_VERSION": "3.11", + }, +) for python_version in ( + "3.10", + "3.11", + "3.12", +)] diff --git a/tests/interpreter/bin.py b/tests/interpreter/bin.py new file mode 100644 index 0000000000..97f3ba8057 --- /dev/null +++ b/tests/interpreter/bin.py @@ -0,0 +1 @@ +# Intentionally empty. diff --git a/tests/interpreter/interpreter_test.py b/tests/interpreter/interpreter_test.py index 1694ddb546..241cab275d 100644 --- a/tests/interpreter/interpreter_test.py +++ b/tests/interpreter/interpreter_test.py @@ -13,7 +13,6 @@ # limitations under the License. import os -import logging import unittest import subprocess @@ -39,33 +38,20 @@ def test_version(self): ).strip() self.assertEqual(result, f"version: {expected_version}") - #def _run_module_test(self, version): - # """Validates that we can successfully invoke a module from the CLI.""" - # # Pass unformatted JSON to the json.tool module. - # result = self.run_bazel( - # "run", - # f"--@rules_python//python/config_settings:python_version={version}", - # "@rules_python//python/bin:python", - # "--", - # "-m", - # "json.tool", - # input = '{"json":"obj"}', - # ) - # # Validate that we get formatted JSON back. - # self.assert_result_matches(result, r'{\n "json": "obj"\n}') - - #def test_run_module_3_10(self): - # self._run_module_test("3.10") - - #def test_run_module_3_11(self): - # self._run_module_test("3.11") - - #def test_run_module_3_12(self): - # self._run_module_test("3.12") - + def test_json_tool(self): + """Validates that we can successfully invoke a module from the CLI.""" + # Pass unformatted JSON to the json.tool module. + result = subprocess.check_output([ + self.interpreter, + "-m", + "json.tool", + ], + text=True, + input = '{"json":"obj"}', + ).strip() + # Validate that we get formatted JSON back. + self.assertEqual(result, '{\n "json": "obj"\n}') if __name__ == "__main__": - # Enabling this makes the runner log subprocesses as the test goes along. - # logging.basicConfig(level = "INFO") unittest.main() diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl index 7fb7016eec..1d22879a1d 100644 --- a/tests/support/sh_py_run_test.bzl +++ b/tests/support/sh_py_run_test.bzl @@ -33,6 +33,8 @@ def _perform_transition_impl(input_settings, attr): settings["//command_line_option:extra_toolchains"] = attr.extra_toolchains if attr.python_version: settings["//python/config_settings:python_version"] = attr.python_version + if attr.python_src: + settings["//python/bin:python_src"] = attr.python_src return settings _perform_transition = transition( @@ -41,12 +43,14 @@ _perform_transition = transition( "//python/config_settings:bootstrap_impl", "//command_line_option:extra_toolchains", "//python/config_settings:python_version", + "//python/bin:python_src", ], outputs = [ "//command_line_option:build_python_zip", "//command_line_option:extra_toolchains", "//python/config_settings:bootstrap_impl", "//python/config_settings:python_version", + "//python/bin:python_src", VISIBLE_FOR_TESTING, ], ) @@ -106,6 +110,7 @@ toolchain. """, ), "python_version": attr.string(), + "python_src": attr.label(), "target": attr.label(executable = True, cfg = "target"), "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", @@ -135,6 +140,7 @@ def py_reconfig_test(*, name, **kwargs): reconfig_kwargs["python_version"] = kwargs.pop("python_version", None) reconfig_kwargs["env"] = kwargs.get("env") reconfig_kwargs["target_compatible_with"] = kwargs.get("target_compatible_with") + reconfig_kwargs["python_src"] = kwargs.pop("python_src", None) inner_name = "_{}_inner".format(name) _py_reconfig_test( From 7ede58a86382ae36e184c87404dfa09874d2b6d2 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Thu, 23 Jan 2025 21:25:19 -0800 Subject: [PATCH 12/34] black --- tests/interpreter/interpreter_test.py | 32 +++++++++++++++------------ 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tests/interpreter/interpreter_test.py b/tests/interpreter/interpreter_test.py index 241cab275d..92c7b41b76 100644 --- a/tests/interpreter/interpreter_test.py +++ b/tests/interpreter/interpreter_test.py @@ -28,26 +28,30 @@ def test_version(self): """Validates that we can successfully execute arbitrary code from the CLI.""" expected_version = os.environ["EXPECTED_PYTHON_VERSION"] - result = subprocess.check_output([self.interpreter], - text = True, - input = "\r".join([ - "import sys", - "v = sys.version_info", - "print(f'version: {v.major}.{v.minor}')", - ]), + result = subprocess.check_output( + [self.interpreter], + text=True, + input="\r".join( + [ + "import sys", + "v = sys.version_info", + "print(f'version: {v.major}.{v.minor}')", + ] + ), ).strip() self.assertEqual(result, f"version: {expected_version}") def test_json_tool(self): """Validates that we can successfully invoke a module from the CLI.""" # Pass unformatted JSON to the json.tool module. - result = subprocess.check_output([ - self.interpreter, - "-m", - "json.tool", - ], - text=True, - input = '{"json":"obj"}', + result = subprocess.check_output( + [ + self.interpreter, + "-m", + "json.tool", + ], + text=True, + input='{"json":"obj"}', ).strip() # Validate that we get formatted JSON back. self.assertEqual(result, '{\n "json": "obj"\n}') From 8cd6de0991f458a67c06fa92c9f52eb548bcb5a5 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Thu, 23 Jan 2025 21:26:00 -0800 Subject: [PATCH 13/34] delete integration test --- tests/integration/interpreter/BUILD.bazel | 13 ------------- tests/integration/interpreter/MODULE.bazel | 21 --------------------- tests/integration/interpreter/WORKSPACE | 0 tests/integration/runner.py | 3 +-- 4 files changed, 1 insertion(+), 36 deletions(-) delete mode 100644 tests/integration/interpreter/BUILD.bazel delete mode 100644 tests/integration/interpreter/MODULE.bazel delete mode 100644 tests/integration/interpreter/WORKSPACE diff --git a/tests/integration/interpreter/BUILD.bazel b/tests/integration/interpreter/BUILD.bazel deleted file mode 100644 index c9b29fc948..0000000000 --- a/tests/integration/interpreter/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/integration/interpreter/MODULE.bazel b/tests/integration/interpreter/MODULE.bazel deleted file mode 100644 index 5bea8126aa..0000000000 --- a/tests/integration/interpreter/MODULE.bazel +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module(name = "module_under_test") - -bazel_dep(name = "rules_python", version = "0.0.0") -local_path_override( - module_name = "rules_python", - path = "../../..", -) diff --git a/tests/integration/interpreter/WORKSPACE b/tests/integration/interpreter/WORKSPACE deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/integration/runner.py b/tests/integration/runner.py index 5895828863..9414a865c0 100644 --- a/tests/integration/runner.py +++ b/tests/integration/runner.py @@ -86,7 +86,7 @@ def setUp(self): "RUNFILES_DIR": os.environ["TEST_SRCDIR"] } - def run_bazel(self, *args: str, input=None, check: bool = True) -> ExecuteResult: + def run_bazel(self, *args: str, check: bool = True) -> ExecuteResult: """Run a bazel invocation. Args: @@ -104,7 +104,6 @@ def run_bazel(self, *args: str, input=None, check: bool = True) -> ExecuteResult proc_result = subprocess.run( args=args, text=True, - input=input, capture_output=True, cwd=cwd, env=env, From 8c2495bd6a151372b1e64c56aef61b383e863f68 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Thu, 23 Jan 2025 21:27:17 -0800 Subject: [PATCH 14/34] delete unused files --- tests/interpreter/bin.py | 1 - tests/interpreter/run_json_tool_test.sh | 76 ------------------------- 2 files changed, 77 deletions(-) delete mode 100644 tests/interpreter/bin.py delete mode 100644 tests/interpreter/run_json_tool_test.sh diff --git a/tests/interpreter/bin.py b/tests/interpreter/bin.py deleted file mode 100644 index 97f3ba8057..0000000000 --- a/tests/interpreter/bin.py +++ /dev/null @@ -1 +0,0 @@ -# Intentionally empty. diff --git a/tests/interpreter/run_json_tool_test.sh b/tests/interpreter/run_json_tool_test.sh deleted file mode 100644 index 728efdd583..0000000000 --- a/tests/interpreter/run_json_tool_test.sh +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# --- begin runfiles.bash initialization v3 --- -# Copy-pasted from the Bazel Bash runfiles library v3. -set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash -source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ - source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ - source "$0.runfiles/$f" 2>/dev/null || \ - source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ - source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ - { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e -# --- end runfiles.bash initialization v3 --- -set +e - -bin=$(rlocation $BIN_RLOCATION) -if [[ -z "$bin" ]]; then - echo "Unable to locate test binary: $BIN_RLOCATION" - exit 1 -fi - -function - -function test_invocation() { - actual=$($bin) - # How we detect if a zip file was executed from depends on which bootstrap - # is used. - # bootstrap_impl=script outputs RULES_PYTHON_ZIP_DIR= - # bootstrap_impl=system_python outputs file:.*Bazel.runfiles - expected_pattern="Hello" - if ! (echo "$actual" | grep "$expected_pattern" ) >/dev/null; then - echo "Test case failed: $1" - echo "expected output to match: $expected_pattern" - echo "but got:\n$actual" - exit 1 - fi -} - -# Test invocation with RUNFILES_DIR set -unset RUNFILES_MANIFEST_FILE -if [[ ! -e "$RUNFILES_DIR" ]]; then - echo "Runfiles doesn't exist: $RUNFILES_DIR" - exit 1 -fi -test_invocation "using RUNFILES_DIR" - - -orig_runfiles_dir="$RUNFILES_DIR" -unset RUNFILES_DIR - -# Test invocation using manifest within runfiles directory (output manifest) -# NOTE: this file may not actually exist in our test, but that's OK; the -# bootstrap just uses the path to find the runfiles directory. -export RUNFILES_MANIFEST_FILE="$orig_runfiles_dir/MANIFEST" -test_invocation "using RUNFILES_MANIFEST_FILE with output manifest" - -# Test invocation using manifest outside runfiles (input manifest) -# NOTE: this file may not actually exist in our test, but that's OK; the -# bootstrap just uses the path to find the runfiles directory. -export RUNFILES_MANIFEST_FILE="${orig_runfiles_dir%%.runfiles}.runfiles_manifest" -test_invocation "using RUNFILES_MANIFEST_FILE with input manifest" - -# Test invocation without any runfiles env vars set -unset RUNFILES_MANIFEST_FILE -test_invocation "using no runfiles env vars" From d7bce2563a5f19ce49db8bff07bd7e04f5b9c981 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Thu, 23 Jan 2025 21:27:47 -0800 Subject: [PATCH 15/34] delete unused files --- tests/integration/BUILD.bazel | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index 637420a634..d178e0f01c 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -120,11 +120,6 @@ rules_python_integration_test( py_main = "custom_commands_test.py", ) -rules_python_integration_test( - name = "interpreter_test", - py_main = "interpreter_test.py", -) - py_library( name = "runner_lib", srcs = ["runner.py"], From d0c83045b20f2d7690e88dbf5e1c56711c02636f Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Thu, 23 Jan 2025 21:28:14 -0800 Subject: [PATCH 16/34] delete unused files --- tests/integration/interpreter_test.py | 74 --------------------------- 1 file changed, 74 deletions(-) delete mode 100644 tests/integration/interpreter_test.py diff --git a/tests/integration/interpreter_test.py b/tests/integration/interpreter_test.py deleted file mode 100644 index a7f015305d..0000000000 --- a/tests/integration/interpreter_test.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import unittest - -from tests.integration import runner - - -class InterpreterTest(runner.TestCase): - def _run_version_test(self, expected_version): - """Validates that we can successfully execute arbitrary code from the CLI.""" - result = self.run_bazel( - "run", - f"--@rules_python//python/config_settings:python_version={expected_version}", - "@rules_python//python/bin:python", - input = "\r".join([ - "import sys", - "v = sys.version_info", - "print(f'version: {v.major}.{v.minor}')", - ]), - ) - self.assert_result_matches(result, f"version: {expected_version}") - - def test_run_interpreter_3_10(self): - self._run_version_test("3.10") - - def test_run_interpreter_3_11(self): - self._run_version_test("3.11") - - def test_run_interpreter_3_12(self): - self._run_version_test("3.12") - - def _run_module_test(self, version): - """Validates that we can successfully invoke a module from the CLI.""" - # Pass unformatted JSON to the json.tool module. - result = self.run_bazel( - "run", - f"--@rules_python//python/config_settings:python_version={version}", - "@rules_python//python/bin:python", - "--", - "-m", - "json.tool", - input = '{"json":"obj"}', - ) - # Validate that we get formatted JSON back. - self.assert_result_matches(result, r'{\n "json": "obj"\n}') - - def test_run_module_3_10(self): - self._run_module_test("3.10") - - def test_run_module_3_11(self): - self._run_module_test("3.11") - - def test_run_module_3_12(self): - self._run_module_test("3.12") - - - -if __name__ == "__main__": - # Enabling this makes the runner log subprocesses as the test goes along. - # logging.basicConfig(level = "INFO") - unittest.main() From 63924fe3976c806294864d7e3f0d4f3dd110c03e Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Thu, 23 Jan 2025 21:28:48 -0800 Subject: [PATCH 17/34] revert unused changes --- .bazelrc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bazelrc b/.bazelrc index bfb07ced41..ada5c5a0a7 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/interpreter,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/interpreter,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered test --test_output=errors From 94d5ee3a648e8e73efc89fe706959c2f8adc70be Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Thu, 23 Jan 2025 21:35:36 -0800 Subject: [PATCH 18/34] pre-commit --- python/bin/BUILD.bazel | 1 - python/bin/interpreter.bzl | 2 +- python/private/interpreter.bzl | 1 - .../dependency_resolver.py | 4 +++- python/private/site_init_template.py | 4 +++- python/private/stage2_bootstrap_template.py | 1 + tests/interpreter/BUILD.bazel | 22 +++++++++---------- tests/interpreter/interpreter_test.py | 2 +- 8 files changed, 20 insertions(+), 17 deletions(-) diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index a90773b743..aec266c3e5 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -1,4 +1,3 @@ -load("//python:defs.bzl", "py_binary") load(":interpreter.bzl", "interpreter") package(default_visibility = ["//visibility:public"]) diff --git a/python/bin/interpreter.bzl b/python/bin/interpreter.bzl index 40282e0790..5a5ed8a9e8 100644 --- a/python/bin/interpreter.bzl +++ b/python/bin/interpreter.bzl @@ -1,3 +1,3 @@ -load("//python/private:interpreter.bzl", _interpeter="interpreter") +load("//python/private:interpreter.bzl", _interpeter = "interpreter") interpreter = _interpeter diff --git a/python/private/interpreter.bzl b/python/private/interpreter.bzl index 8754b47cb2..1952db2bdd 100644 --- a/python/private/interpreter.bzl +++ b/python/private/interpreter.bzl @@ -1,5 +1,4 @@ load("@bazel_skylib//lib:paths.bzl", "paths") -load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("//python:py_runtime_info.bzl", "PyRuntimeInfo") load(":sentinel.bzl", "SentinelInfo") load(":toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") diff --git a/python/private/pypi/dependency_resolver/dependency_resolver.py b/python/private/pypi/dependency_resolver/dependency_resolver.py index 6f6c20241b..28f7c9af09 100644 --- a/python/private/pypi/dependency_resolver/dependency_resolver.py +++ b/python/private/pypi/dependency_resolver/dependency_resolver.py @@ -173,7 +173,9 @@ def main( # the lines like: # via -r /absolute/path/to/ # with: # via -r # For Windows, we should explicitly call .as_posix() to convert \\ -> / - absolute_src_prefixes = [Path(src).absolute().parent.as_posix() + "/" for src in src_files] + absolute_src_prefixes = [ + Path(src).absolute().parent.as_posix() + "/" for src in src_files + ] if UPDATE: print("Updating " + requirements_file_relative) diff --git a/python/private/site_init_template.py b/python/private/site_init_template.py index dcbd799909..40fb4e4139 100644 --- a/python/private/site_init_template.py +++ b/python/private/site_init_template.py @@ -163,7 +163,9 @@ def _maybe_add_path(path): if cov_tool: _print_verbose_coverage(f"Using toolchain coverage_tool {cov_tool}") elif cov_tool := os.environ.get("PYTHON_COVERAGE"): - _print_verbose_coverage(f"Using env var coverage: PYTHON_COVERAGE={cov_tool}") + _print_verbose_coverage( + f"Using env var coverage: PYTHON_COVERAGE={cov_tool}" + ) if cov_tool: if os.path.isabs(cov_tool): diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index b1f6b031aa..f7daafb63b 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -398,6 +398,7 @@ def main(): if os.environ.get("COVERAGE_DIR"): import _bazel_site_init + coverage_enabled = _bazel_site_init.COVERAGE_SETUP else: coverage_enabled = False diff --git a/tests/interpreter/BUILD.bazel b/tests/interpreter/BUILD.bazel index 2d1bbf55be..ee1ef22ab6 100644 --- a/tests/interpreter/BUILD.bazel +++ b/tests/interpreter/BUILD.bazel @@ -17,17 +17,17 @@ load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") [py_reconfig_test( name = "python_version_%s_test" % python_version, srcs = ["interpreter_test.py"], - main = "interpreter_test.py", data = [ "//python/bin:python", ], - deps = [ - "//python/runfiles", - ], - python_version = python_version, env = { "EXPECTED_PYTHON_VERSION": python_version, }, + main = "interpreter_test.py", + python_version = python_version, + deps = [ + "//python/runfiles", + ], ) for python_version in ( "3.10", "3.11", @@ -37,18 +37,18 @@ load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") [py_reconfig_test( name = "python_src_%s_test" % python_version, srcs = ["interpreter_test.py"], - main = "interpreter_test.py", data = [ "//python/bin:python", ], - deps = [ - "//python/runfiles", - ], - python_src = "//tools/publish:twine", - python_version = python_version, env = { "EXPECTED_PYTHON_VERSION": "3.11", }, + main = "interpreter_test.py", + python_src = "//tools/publish:twine", + python_version = python_version, + deps = [ + "//python/runfiles", + ], ) for python_version in ( "3.10", "3.11", diff --git a/tests/interpreter/interpreter_test.py b/tests/interpreter/interpreter_test.py index 92c7b41b76..f517bbcd61 100644 --- a/tests/interpreter/interpreter_test.py +++ b/tests/interpreter/interpreter_test.py @@ -13,8 +13,8 @@ # limitations under the License. import os -import unittest import subprocess +import unittest from python.runfiles import runfiles From 8e935f430157b8b74eee43e580cb1035ad16c011 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sat, 25 Jan 2025 17:56:27 -0800 Subject: [PATCH 19/34] Fix Windows hopefully --- docs/toolchains.md | 17 ++++++++++---- tests/interpreter/BUILD.bazel | 32 +++++++++++++++++---------- tests/interpreter/interpreter_test.py | 13 ++++++++--- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/docs/toolchains.md b/docs/toolchains.md index 888a378953..c11d285b3a 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -516,7 +516,7 @@ To run the interpreter that Bazel will use, you can use the `@rules_python//python/bin:python` target. This is a binary target with the executable pointing at the `python3` binary plus the relevent runfiles. -``` +```console $ bazel run @rules_python//python/bin:python Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux Type "help", "copyright", "credits" or "license" for more information. @@ -528,14 +528,23 @@ Type "help", "copyright", "credits" or "license" for more information. ``` You can also access a specific binary's interpreter this way by using the -`@rules_python//python/bin:python_src` target. +`@rules_python//python/bin:python_src` target. In the example below, it is +assumed that the `@rules_python//tools/publish:twine` binary is fixed at Python +3.11. -``` -$ bazel run @rules_python//python/bin:python --@rules_python//python/bin:interpreter_src=//path/to:bin +```console +$ bazel run @rules_python//python/bin:python --@rules_python//python/bin:interpreter_src=@rules_python//tools/publish:twine +Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +$ bazel run @rules_python//python/bin:python --@rules_python//python/bin:interpreter_src=@rules_python//tools/publish:twine --@rules_python//python/config_settings:python_version=3.12 Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux Type "help", "copyright", "credits" or "license" for more information. >>> ``` +Despite setting the Python version explicitly to 3.12 in the example above, the +interpreter comes from the `@rules_python//tools/publish:twine` binary. That is +a fixed version. :::{note} The `python` target does not provide access to any modules from `py_*` diff --git a/tests/interpreter/BUILD.bazel b/tests/interpreter/BUILD.bazel index ee1ef22ab6..3975ff1d31 100644 --- a/tests/interpreter/BUILD.bazel +++ b/tests/interpreter/BUILD.bazel @@ -14,6 +14,12 @@ load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") +PYTHON_VERSIONS_TO_TEST = ( + "3.10", + "3.11", + "3.12", +) + [py_reconfig_test( name = "python_version_%s_test" % python_version, srcs = ["interpreter_test.py"], @@ -21,18 +27,18 @@ load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") "//python/bin:python", ], env = { - "EXPECTED_PYTHON_VERSION": python_version, + # Both the interpreter and the test itself are expected to run under + # the same version. + "EXPECTED_INTERPRETER_VERSION": python_version, + "EXPECTED_SELF_VERSION": python_version, + "PYTHON_BIN": "$(location //python/bin:python)", }, main = "interpreter_test.py", python_version = python_version, deps = [ "//python/runfiles", ], -) for python_version in ( - "3.10", - "3.11", - "3.12", -)] +) for python_version in PYTHON_VERSIONS_TO_TEST] [py_reconfig_test( name = "python_src_%s_test" % python_version, @@ -41,7 +47,13 @@ load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") "//python/bin:python", ], env = { - "EXPECTED_PYTHON_VERSION": "3.11", + # Since we're grabbing the interpreter from a binary with a fixed + # version, we expect to always see that version. It doesn't matter what + # Python version the test itself is running with. + "EXPECTED_INTERPRETER_VERSION": "3.11", + # The test itself is still running under different Python versions. + "EXPECTED_SELF_VERSION": python_version, + "PYTHON_BIN": "$(rootpath //python/bin:python)", }, main = "interpreter_test.py", python_src = "//tools/publish:twine", @@ -49,8 +61,4 @@ load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") deps = [ "//python/runfiles", ], -) for python_version in ( - "3.10", - "3.11", - "3.12", -)] +) for python_version in PYTHON_VERSIONS_TO_TEST] diff --git a/tests/interpreter/interpreter_test.py b/tests/interpreter/interpreter_test.py index f517bbcd61..f28259231f 100644 --- a/tests/interpreter/interpreter_test.py +++ b/tests/interpreter/interpreter_test.py @@ -14,6 +14,7 @@ import os import subprocess +import sys import unittest from python.runfiles import runfiles @@ -22,11 +23,17 @@ class InterpreterTest(unittest.TestCase): def setUp(self): r = runfiles.Create() - self.interpreter = r.Rlocation("rules_python/python/bin/python3") + self.interpreter = os.environ["PYTHON_BIN"] - def test_version(self): + def test_self_version(self): + """Performs a sanity check on the Python version used for this test.""" + expected_version = os.environ["EXPECTED_SELF_VERSION"] + v = sys.version_info + self.assertEqual(expected_version, f"{v.major}.{v.minor}") + + def test_interpreter_version(self): """Validates that we can successfully execute arbitrary code from the CLI.""" - expected_version = os.environ["EXPECTED_PYTHON_VERSION"] + expected_version = os.environ["EXPECTED_INTERPRETER_VERSION"] result = subprocess.check_output( [self.interpreter], From d39a0a1a71ca05be29b899a6d2ab2746b16766ec Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sat, 25 Jan 2025 17:56:59 -0800 Subject: [PATCH 20/34] fixup --- tests/interpreter/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/interpreter/BUILD.bazel b/tests/interpreter/BUILD.bazel index 3975ff1d31..00298dc84b 100644 --- a/tests/interpreter/BUILD.bazel +++ b/tests/interpreter/BUILD.bazel @@ -31,7 +31,7 @@ PYTHON_VERSIONS_TO_TEST = ( # the same version. "EXPECTED_INTERPRETER_VERSION": python_version, "EXPECTED_SELF_VERSION": python_version, - "PYTHON_BIN": "$(location //python/bin:python)", + "PYTHON_BIN": "$(rootpath //python/bin:python)", }, main = "interpreter_test.py", python_version = python_version, From abcc5e9d70b734fc716f2d6fd533fa691b91e686 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sat, 25 Jan 2025 18:03:34 -0800 Subject: [PATCH 21/34] fix: Enable location expansion for `sh_py_run_test` I noticed that my `$(location //path/to:target)` wasn't getting expanded when writing a test. This patch fixes the issue by forwarding the already-expanded environment from the inner target to the outer target. --- tests/support/sh_py_run_test.bzl | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl index 7fb7016eec..9bf0a7402e 100644 --- a/tests/support/sh_py_run_test.bzl +++ b/tests/support/sh_py_run_test.bzl @@ -86,16 +86,14 @@ def _py_reconfig_impl(ctx): default_info.default_runfiles, ), ), - testing.TestEnvironment( - environment = ctx.attr.env, - ), + # Inherit the expanded environment from the inner target. + ctx.attr.target[RunEnvironmentInfo], ] def _make_reconfig_rule(**kwargs): attrs = { "bootstrap_impl": attr.string(), "build_python_zip": attr.string(default = "auto"), - "env": attr.string_dict(), "extra_toolchains": attr.string_list( doc = """ Value for the --extra_toolchains flag. @@ -133,7 +131,6 @@ def py_reconfig_test(*, name, **kwargs): reconfig_kwargs["bootstrap_impl"] = kwargs.pop("bootstrap_impl", None) reconfig_kwargs["extra_toolchains"] = kwargs.pop("extra_toolchains", None) reconfig_kwargs["python_version"] = kwargs.pop("python_version", None) - reconfig_kwargs["env"] = kwargs.get("env") reconfig_kwargs["target_compatible_with"] = kwargs.get("target_compatible_with") inner_name = "_{}_inner".format(name) @@ -172,7 +169,7 @@ def sh_py_run_test(*, name, sh_src, py_src, **kwargs): py_binary_kwargs = { key: kwargs.pop(key) - for key in ("imports", "deps") + for key in ("imports", "deps", "env") if key in kwargs } From 81b26c6b44fdff031350e7d1be1ddb7988caefb6 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sat, 25 Jan 2025 18:13:12 -0800 Subject: [PATCH 22/34] buildifier fixes hopefully --- python/bin/interpreter.bzl | 16 ++++++++++++++++ python/private/interpreter.bzl | 16 ++++++++++++++++ tests/support/sh_py_run_test.bzl | 4 ++-- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/python/bin/interpreter.bzl b/python/bin/interpreter.bzl index 5a5ed8a9e8..6a391e1d5a 100644 --- a/python/bin/interpreter.bzl +++ b/python/bin/interpreter.bzl @@ -1,3 +1,19 @@ +# Copyright 2025 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Rules to access the underlying Python interpreter.""" + load("//python/private:interpreter.bzl", _interpeter = "interpreter") interpreter = _interpeter diff --git a/python/private/interpreter.bzl b/python/private/interpreter.bzl index 1952db2bdd..cc542fa91f 100644 --- a/python/private/interpreter.bzl +++ b/python/private/interpreter.bzl @@ -1,3 +1,19 @@ +# Copyright 2025 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of the rules to access the underlying Python interpreter.""" + load("@bazel_skylib//lib:paths.bzl", "paths") load("//python:py_runtime_info.bzl", "PyRuntimeInfo") load(":sentinel.bzl", "SentinelInfo") diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl index 6bd32a6b94..7b76f325d0 100644 --- a/tests/support/sh_py_run_test.bzl +++ b/tests/support/sh_py_run_test.bzl @@ -107,8 +107,8 @@ to make the RBE presubmits happy, which disable auto-detection of a CC toolchain. """, ), - "python_version": attr.string(), "python_src": attr.label(), + "python_version": attr.string(), "target": attr.label(executable = True, cfg = "target"), "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", @@ -135,9 +135,9 @@ def py_reconfig_test(*, name, **kwargs): reconfig_kwargs = {} reconfig_kwargs["bootstrap_impl"] = kwargs.pop("bootstrap_impl", None) reconfig_kwargs["extra_toolchains"] = kwargs.pop("extra_toolchains", None) + reconfig_kwargs["python_src"] = kwargs.pop("python_src", None) reconfig_kwargs["python_version"] = kwargs.pop("python_version", None) reconfig_kwargs["target_compatible_with"] = kwargs.get("target_compatible_with") - reconfig_kwargs["python_src"] = kwargs.pop("python_src", None) inner_name = "_{}_inner".format(name) _py_reconfig_test( From cfd96c8a7f2285429514b19906f4711b474033fe Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sat, 25 Jan 2025 18:18:15 -0800 Subject: [PATCH 23/34] delete superfluous runfiles --- tests/interpreter/BUILD.bazel | 6 ------ tests/interpreter/interpreter_test.py | 3 --- 2 files changed, 9 deletions(-) diff --git a/tests/interpreter/BUILD.bazel b/tests/interpreter/BUILD.bazel index 00298dc84b..77af0aa71c 100644 --- a/tests/interpreter/BUILD.bazel +++ b/tests/interpreter/BUILD.bazel @@ -35,9 +35,6 @@ PYTHON_VERSIONS_TO_TEST = ( }, main = "interpreter_test.py", python_version = python_version, - deps = [ - "//python/runfiles", - ], ) for python_version in PYTHON_VERSIONS_TO_TEST] [py_reconfig_test( @@ -58,7 +55,4 @@ PYTHON_VERSIONS_TO_TEST = ( main = "interpreter_test.py", python_src = "//tools/publish:twine", python_version = python_version, - deps = [ - "//python/runfiles", - ], ) for python_version in PYTHON_VERSIONS_TO_TEST] diff --git a/tests/interpreter/interpreter_test.py b/tests/interpreter/interpreter_test.py index f28259231f..aaac04bfbd 100644 --- a/tests/interpreter/interpreter_test.py +++ b/tests/interpreter/interpreter_test.py @@ -17,12 +17,9 @@ import sys import unittest -from python.runfiles import runfiles - class InterpreterTest(unittest.TestCase): def setUp(self): - r = runfiles.Create() self.interpreter = os.environ["PYTHON_BIN"] def test_self_version(self): From cadfc9acaadf66962bcad151b0072bbfaeb752fb Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Mon, 3 Feb 2025 21:40:54 -0800 Subject: [PATCH 24/34] get remote execution working --- python/private/interpreter.bzl | 20 +++++++++-- python/private/interpreter_tmpl.sh | 19 ++++++++++ tests/interpreter/interpreter_test.py | 52 ++++++++++++++++----------- 3 files changed, 69 insertions(+), 22 deletions(-) create mode 100644 python/private/interpreter_tmpl.sh diff --git a/python/private/interpreter.bzl b/python/private/interpreter.bzl index cc542fa91f..bd8cb15d50 100644 --- a/python/private/interpreter.bzl +++ b/python/private/interpreter.bzl @@ -31,7 +31,14 @@ def _interpreter_impl(ctx): # re-exec. If it's not a recognized name, then they fail. if runtime.interpreter: executable = ctx.actions.declare_file(runtime.interpreter.basename) - ctx.actions.symlink(output = executable, target_file = runtime.interpreter, is_executable = True) + ctx.actions.expand_template( + template = ctx.file._template, + output = executable, + substitutions = { + "%target_file%": runtime.interpreter.short_path, + }, + is_executable = True, + ) else: executable = ctx.actions.declare_symlink(paths.basename(runtime.interpreter_path)) ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path) @@ -39,7 +46,9 @@ def _interpreter_impl(ctx): return [ DefaultInfo( executable = executable, - runfiles = ctx.runfiles([executable], transitive_files = runtime.files), + runfiles = ctx.runfiles([executable], transitive_files = runtime.files).merge_all([ + ctx.attr._bash_runfiles[DefaultInfo].default_runfiles, + ]), ), ] @@ -51,5 +60,12 @@ interpreter = rule( "binary": attr.label( mandatory = True, ), + "_template": attr.label( + default = "//python/private:interpreter_tmpl.sh", + allow_single_file = True, + ), + "_bash_runfiles": attr.label( + default = "@bazel_tools//tools/bash/runfiles", + ), }, ) diff --git a/python/private/interpreter_tmpl.sh b/python/private/interpreter_tmpl.sh new file mode 100644 index 0000000000..888c12e805 --- /dev/null +++ b/python/private/interpreter_tmpl.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# --- begin runfiles.bash initialization v3 --- +# Copy-pasted from the Bazel Bash runfiles library v3. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +# shellcheck disable=SC1090 +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v3 --- + +readonly TARGET_FILE="%target_file%" + +MAIN_BIN="$(rlocation "${TARGET_FILE#*/}")" + +exec "${MAIN_BIN}" "$@" diff --git a/tests/interpreter/interpreter_test.py b/tests/interpreter/interpreter_test.py index aaac04bfbd..3a79c7cccb 100644 --- a/tests/interpreter/interpreter_test.py +++ b/tests/interpreter/interpreter_test.py @@ -32,31 +32,43 @@ def test_interpreter_version(self): """Validates that we can successfully execute arbitrary code from the CLI.""" expected_version = os.environ["EXPECTED_INTERPRETER_VERSION"] - result = subprocess.check_output( - [self.interpreter], - text=True, - input="\r".join( - [ - "import sys", - "v = sys.version_info", - "print(f'version: {v.major}.{v.minor}')", - ] - ), - ).strip() + try: + result = subprocess.check_output( + [self.interpreter], + text=True, + stderr=subprocess.STDOUT, + input="\r".join( + [ + "import sys", + "v = sys.version_info", + "print(f'version: {v.major}.{v.minor}')", + ] + ), + ).strip() + except subprocess.CalledProcessError as error: + print("OUTPUT:", error.stdout) + raise + self.assertEqual(result, f"version: {expected_version}") def test_json_tool(self): """Validates that we can successfully invoke a module from the CLI.""" # Pass unformatted JSON to the json.tool module. - result = subprocess.check_output( - [ - self.interpreter, - "-m", - "json.tool", - ], - text=True, - input='{"json":"obj"}', - ).strip() + try: + result = subprocess.check_output( + [ + self.interpreter, + "-m", + "json.tool", + ], + text=True, + stderr=subprocess.STDOUT, + input='{"json":"obj"}', + ).strip() + except subprocess.CalledProcessError as error: + print("OUTPUT:", error.stdout) + raise + # Validate that we get formatted JSON back. self.assertEqual(result, '{\n "json": "obj"\n}') From 155489fca58301efd9094529e2dcfd6ec768637c Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Mon, 3 Feb 2025 21:51:00 -0800 Subject: [PATCH 25/34] incorporate feedback --- docs/toolchains.md | 4 ++-- python/bin/BUILD.bazel | 9 ++++----- python/bin/interpreter.bzl | 19 ------------------- python/private/interpreter.bzl | 6 +++--- tests/interpreter/interpreter_test.py | 1 + 5 files changed, 10 insertions(+), 29 deletions(-) delete mode 100644 python/bin/interpreter.bzl diff --git a/docs/toolchains.md b/docs/toolchains.md index 57aa5ba2f4..3294c1732a 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -562,7 +562,7 @@ Currently the following flags are used to influence toolchain selection: To run the interpreter that Bazel will use, you can use the `@rules_python//python/bin:python` target. This is a binary target with -the executable pointing at the `python3` binary plus the relevent runfiles. +the executable pointing at the `python3` binary plus its relevent runfiles. ```console $ bazel run @rules_python//python/bin:python @@ -596,5 +596,5 @@ a fixed version. :::{note} The `python` target does not provide access to any modules from `py_*` -targets on its own. Work is ongoing to support that. +targets on its own. Please file a feature request if this is desired. ::: diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index aec266c3e5..be363e3809 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -1,16 +1,15 @@ -load(":interpreter.bzl", "interpreter") - -package(default_visibility = ["//visibility:public"]) +load("//python/private:interpreter.bzl", _interpreter_binary="interpreter_binary") filegroup( name = "distribution", srcs = glob(["**"]), - visibility = ["//python:__pkg__"], + visibility = ["//:__subpackages__"], ) -interpreter( +_interpreter_binary( name = "python", binary = ":python_src", + visibility = ["//visibility:public"], ) # The user can modify this flag to source different interpreters for the diff --git a/python/bin/interpreter.bzl b/python/bin/interpreter.bzl deleted file mode 100644 index 6a391e1d5a..0000000000 --- a/python/bin/interpreter.bzl +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Rules to access the underlying Python interpreter.""" - -load("//python/private:interpreter.bzl", _interpeter = "interpreter") - -interpreter = _interpeter diff --git a/python/private/interpreter.bzl b/python/private/interpreter.bzl index bd8cb15d50..57faa52df0 100644 --- a/python/private/interpreter.bzl +++ b/python/private/interpreter.bzl @@ -19,7 +19,7 @@ load("//python:py_runtime_info.bzl", "PyRuntimeInfo") load(":sentinel.bzl", "SentinelInfo") load(":toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") -def _interpreter_impl(ctx): +def _interpreter_binary_impl(ctx): if SentinelInfo in ctx.attr.binary: toolchain = ctx.toolchains[TARGET_TOOLCHAIN_TYPE] runtime = toolchain.py3_runtime @@ -52,8 +52,8 @@ def _interpreter_impl(ctx): ), ] -interpreter = rule( - implementation = _interpreter_impl, +interpreter_binary = rule( + implementation = _interpreter_binary_impl, toolchains = [TARGET_TOOLCHAIN_TYPE], executable = True, attrs = { diff --git a/tests/interpreter/interpreter_test.py b/tests/interpreter/interpreter_test.py index 3a79c7cccb..ccd68a433b 100644 --- a/tests/interpreter/interpreter_test.py +++ b/tests/interpreter/interpreter_test.py @@ -20,6 +20,7 @@ class InterpreterTest(unittest.TestCase): def setUp(self): + super().setUp() self.interpreter = os.environ["PYTHON_BIN"] def test_self_version(self): From 83a774db8b31fcd46aab355246a7cf887ca0f985 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Mon, 3 Feb 2025 21:54:41 -0800 Subject: [PATCH 26/34] buildifier --- python/bin/BUILD.bazel | 2 +- python/private/interpreter.bzl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index be363e3809..458edfd435 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -1,4 +1,4 @@ -load("//python/private:interpreter.bzl", _interpreter_binary="interpreter_binary") +load("//python/private:interpreter.bzl", _interpreter_binary = "interpreter_binary") filegroup( name = "distribution", diff --git a/python/private/interpreter.bzl b/python/private/interpreter.bzl index 57faa52df0..2ba36c14e2 100644 --- a/python/private/interpreter.bzl +++ b/python/private/interpreter.bzl @@ -60,12 +60,12 @@ interpreter_binary = rule( "binary": attr.label( mandatory = True, ), + "_bash_runfiles": attr.label( + default = "@bazel_tools//tools/bash/runfiles", + ), "_template": attr.label( default = "//python/private:interpreter_tmpl.sh", allow_single_file = True, ), - "_bash_runfiles": attr.label( - default = "@bazel_tools//tools/bash/runfiles", - ), }, ) From 4d15a73b0d0b1bb6636af50d4d967ae5945c6457 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 9 Feb 2025 16:19:41 -0800 Subject: [PATCH 27/34] explore using ctx.actions.declare_symlink --- python/bin/BUILD.bazel | 4 ++++ python/private/interpreter.bzl | 16 ++++++++++++++++ python/private/py_executable.bzl | 12 ++++++------ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index 458edfd435..fac50fa970 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -10,6 +10,10 @@ _interpreter_binary( name = "python", binary = ":python_src", visibility = ["//visibility:public"], + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), ) # The user can modify this flag to source different interpreters for the diff --git a/python/private/interpreter.bzl b/python/private/interpreter.bzl index 2ba36c14e2..91a8b7ea77 100644 --- a/python/private/interpreter.bzl +++ b/python/private/interpreter.bzl @@ -18,6 +18,7 @@ load("@bazel_skylib//lib:paths.bzl", "paths") load("//python:py_runtime_info.bzl", "PyRuntimeInfo") load(":sentinel.bzl", "SentinelInfo") load(":toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") +load(":py_executable.bzl", "relative_path", "runfiles_root_path") def _interpreter_binary_impl(ctx): if SentinelInfo in ctx.attr.binary: @@ -30,6 +31,13 @@ def _interpreter_binary_impl(ctx): # because of things like pyenv: they use $0 to determine what to # re-exec. If it's not a recognized name, then they fail. if runtime.interpreter: + # Option 1: + # Works locally, but not remotely. + #executable = ctx.actions.declare_file(runtime.interpreter.basename) + #ctx.actions.symlink(output = executable, target_file = runtime.interpreter) + + # Option 2: + # Works locally, and remotely, but not on Windows. executable = ctx.actions.declare_file(runtime.interpreter.basename) ctx.actions.expand_template( template = ctx.file._template, @@ -39,6 +47,14 @@ def _interpreter_binary_impl(ctx): }, is_executable = True, ) + + # Option 3: + # Works locally and remotely, but only in runfiles, doesn't work via "bazel run". + #executable = ctx.actions.declare_symlink(runtime.interpreter.basename) + #ctx.actions.symlink(output = executable, target_path = relative_path( + # from_ = paths.dirname(runfiles_root_path(ctx, executable.short_path)), + # to = runfiles_root_path(ctx, runtime.interpreter.short_path), + #)) else: executable = ctx.actions.declare_symlink(paths.basename(runtime.interpreter_path)) ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 18a7a707fc..b6512d20d0 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -447,7 +447,7 @@ def _create_executable( ) def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details, venv): - python_binary = _runfiles_root_path(ctx, venv.interpreter.short_path) + python_binary = runfiles_root_path(ctx, venv.interpreter.short_path) python_binary_actual = venv.interpreter_actual_path # The location of this file doesn't really matter. It's added to @@ -522,7 +522,7 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): if not venvs_use_declare_symlink_enabled: if runtime.interpreter: - interpreter_actual_path = _runfiles_root_path(ctx, runtime.interpreter.short_path) + interpreter_actual_path = runfiles_root_path(ctx, runtime.interpreter.short_path) else: interpreter_actual_path = runtime.interpreter_path @@ -543,11 +543,11 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): # may choose to write what symlink() points to instead. interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename)) - interpreter_actual_path = _runfiles_root_path(ctx, runtime.interpreter.short_path) + interpreter_actual_path = runfiles_root_path(ctx, runtime.interpreter.short_path) rel_path = relative_path( # dirname is necessary because a relative symlink is relative to # the directory the symlink resides within. - from_ = paths.dirname(_runfiles_root_path(ctx, interpreter.short_path)), + from_ = paths.dirname(runfiles_root_path(ctx, interpreter.short_path)), to = interpreter_actual_path, ) @@ -646,7 +646,7 @@ def _create_stage2_bootstrap( ) return output -def _runfiles_root_path(ctx, short_path): +def runfiles_root_path(ctx, short_path): """Compute a runfiles-root relative path from `File.short_path` Args: @@ -676,7 +676,7 @@ def _create_stage1_bootstrap( runtime = runtime_details.effective_runtime if venv: - python_binary_path = _runfiles_root_path(ctx, venv.interpreter.short_path) + python_binary_path = runfiles_root_path(ctx, venv.interpreter.short_path) else: python_binary_path = runtime_details.executable_interpreter_path From 77a75d78463a96efa44865ee6063e315c34086aa Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 9 Feb 2025 16:27:22 -0800 Subject: [PATCH 28/34] revert unnecessary change --- python/private/stage2_bootstrap_template.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index f7daafb63b..b1f6b031aa 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -398,7 +398,6 @@ def main(): if os.environ.get("COVERAGE_DIR"): import _bazel_site_init - coverage_enabled = _bazel_site_init.COVERAGE_SETUP else: coverage_enabled = False From ddcf7385f6318535ed581336f3d96fd1b44f3177 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 16 Feb 2025 10:48:29 -0800 Subject: [PATCH 29/34] incorporate feedback --- python/private/interpreter.bzl | 29 ++++++++++----------- python/private/interpreter_tmpl.sh | 2 ++ python/private/py_executable.bzl | 12 ++++----- tests/interpreter/BUILD.bazel | 41 ++++++++++++------------------ 4 files changed, 37 insertions(+), 47 deletions(-) diff --git a/python/private/interpreter.bzl b/python/private/interpreter.bzl index 91a8b7ea77..57c497642b 100644 --- a/python/private/interpreter.bzl +++ b/python/private/interpreter.bzl @@ -18,7 +18,6 @@ load("@bazel_skylib//lib:paths.bzl", "paths") load("//python:py_runtime_info.bzl", "PyRuntimeInfo") load(":sentinel.bzl", "SentinelInfo") load(":toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") -load(":py_executable.bzl", "relative_path", "runfiles_root_path") def _interpreter_binary_impl(ctx): if SentinelInfo in ctx.attr.binary: @@ -31,30 +30,28 @@ def _interpreter_binary_impl(ctx): # because of things like pyenv: they use $0 to determine what to # re-exec. If it's not a recognized name, then they fail. if runtime.interpreter: - # Option 1: - # Works locally, but not remotely. - #executable = ctx.actions.declare_file(runtime.interpreter.basename) - #ctx.actions.symlink(output = executable, target_file = runtime.interpreter) - - # Option 2: - # Works locally, and remotely, but not on Windows. + # In order for this to work both locally and remotely, we create a + # shell script here that re-exec's into the real interpreter. Ideally, + # we'd just use a symlink, but that breaks under certain conditions. If + # we use a ctx.actions.symlink(target=...) then it fails under remote + # execution. If we use ctx.actions.symlink(target_path=...) then it + # behaves differently inside the runfiles tree and outside the runfiles + # tree. + # + # This currently does not work on Windows. Need to find a way to enable + # that. executable = ctx.actions.declare_file(runtime.interpreter.basename) ctx.actions.expand_template( template = ctx.file._template, output = executable, substitutions = { + # Since we never invoke this rule from within the interpreter's + # own repository, the short_path here should give us a + # predictable path of "..//". "%target_file%": runtime.interpreter.short_path, }, is_executable = True, ) - - # Option 3: - # Works locally and remotely, but only in runfiles, doesn't work via "bazel run". - #executable = ctx.actions.declare_symlink(runtime.interpreter.basename) - #ctx.actions.symlink(output = executable, target_path = relative_path( - # from_ = paths.dirname(runfiles_root_path(ctx, executable.short_path)), - # to = runfiles_root_path(ctx, runtime.interpreter.short_path), - #)) else: executable = ctx.actions.declare_symlink(paths.basename(runtime.interpreter_path)) ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path) diff --git a/python/private/interpreter_tmpl.sh b/python/private/interpreter_tmpl.sh index 888c12e805..694fbb4171 100644 --- a/python/private/interpreter_tmpl.sh +++ b/python/private/interpreter_tmpl.sh @@ -14,6 +14,8 @@ source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ readonly TARGET_FILE="%target_file%" +# Strip the leading "../" from "..//" so that we can do +# a runfiles lookup. MAIN_BIN="$(rlocation "${TARGET_FILE#*/}")" exec "${MAIN_BIN}" "$@" diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index dc73d7c384..2b2bf6636a 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -447,7 +447,7 @@ def _create_executable( ) def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details, venv): - python_binary = runfiles_root_path(ctx, venv.interpreter.short_path) + python_binary = _runfiles_root_path(ctx, venv.interpreter.short_path) python_binary_actual = venv.interpreter_actual_path # The location of this file doesn't really matter. It's added to @@ -522,7 +522,7 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): if not venvs_use_declare_symlink_enabled: if runtime.interpreter: - interpreter_actual_path = runfiles_root_path(ctx, runtime.interpreter.short_path) + interpreter_actual_path = _runfiles_root_path(ctx, runtime.interpreter.short_path) else: interpreter_actual_path = runtime.interpreter_path @@ -543,11 +543,11 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): # may choose to write what symlink() points to instead. interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename)) - interpreter_actual_path = runfiles_root_path(ctx, runtime.interpreter.short_path) + interpreter_actual_path = _runfiles_root_path(ctx, runtime.interpreter.short_path) rel_path = relative_path( # dirname is necessary because a relative symlink is relative to # the directory the symlink resides within. - from_ = paths.dirname(runfiles_root_path(ctx, interpreter.short_path)), + from_ = paths.dirname(_runfiles_root_path(ctx, interpreter.short_path)), to = interpreter_actual_path, ) @@ -646,7 +646,7 @@ def _create_stage2_bootstrap( ) return output -def runfiles_root_path(ctx, short_path): +def _runfiles_root_path(ctx, short_path): """Compute a runfiles-root relative path from `File.short_path` Args: @@ -676,7 +676,7 @@ def _create_stage1_bootstrap( runtime = runtime_details.effective_runtime if venv: - python_binary_path = runfiles_root_path(ctx, venv.interpreter.short_path) + python_binary_path = _runfiles_root_path(ctx, venv.interpreter.short_path) else: python_binary_path = runtime_details.executable_interpreter_path diff --git a/tests/interpreter/BUILD.bazel b/tests/interpreter/BUILD.bazel index 77af0aa71c..3e3491af8c 100644 --- a/tests/interpreter/BUILD.bazel +++ b/tests/interpreter/BUILD.bazel @@ -12,47 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") +load(":interpreter_tests.bzl", "py_reconfig_interpreter_tests", "PYTHON_VERSIONS_TO_TEST") -PYTHON_VERSIONS_TO_TEST = ( - "3.10", - "3.11", - "3.12", -) - -[py_reconfig_test( - name = "python_version_%s_test" % python_version, +py_reconfig_interpreter_tests( + name = "interpreter_version_test", + python_versions = PYTHON_VERSIONS_TO_TEST, + # Both the interpreter and the test itself are expected to run under + # the same version. + expected_interpreter_version = None, srcs = ["interpreter_test.py"], data = [ "//python/bin:python", ], env = { - # Both the interpreter and the test itself are expected to run under - # the same version. - "EXPECTED_INTERPRETER_VERSION": python_version, - "EXPECTED_SELF_VERSION": python_version, "PYTHON_BIN": "$(rootpath //python/bin:python)", }, main = "interpreter_test.py", - python_version = python_version, -) for python_version in PYTHON_VERSIONS_TO_TEST] +) -[py_reconfig_test( - name = "python_src_%s_test" % python_version, +py_reconfig_interpreter_tests( + name = "python_src_test", srcs = ["interpreter_test.py"], data = [ "//python/bin:python", ], env = { - # Since we're grabbing the interpreter from a binary with a fixed - # version, we expect to always see that version. It doesn't matter what - # Python version the test itself is running with. - "EXPECTED_INTERPRETER_VERSION": "3.11", - # The test itself is still running under different Python versions. - "EXPECTED_SELF_VERSION": python_version, "PYTHON_BIN": "$(rootpath //python/bin:python)", }, + python_versions = PYTHON_VERSIONS_TO_TEST, + # Since we're grabbing the interpreter from a binary with a fixed + # version, we expect to always see that version. It doesn't matter what + # Python version the test itself is running with. + expected_interpreter_version = "3.11", main = "interpreter_test.py", python_src = "//tools/publish:twine", - python_version = python_version, -) for python_version in PYTHON_VERSIONS_TO_TEST] +) From faa37b3e016287944e8c14165efdceea2946e3c7 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 16 Feb 2025 10:50:37 -0800 Subject: [PATCH 30/34] add missing file --- tests/interpreter/interpreter_tests.bzl | 38 +++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/interpreter/interpreter_tests.bzl diff --git a/tests/interpreter/interpreter_tests.bzl b/tests/interpreter/interpreter_tests.bzl new file mode 100644 index 0000000000..9045ab1b13 --- /dev/null +++ b/tests/interpreter/interpreter_tests.bzl @@ -0,0 +1,38 @@ +# Copyright 2025 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") + +PYTHON_VERSIONS_TO_TEST = ( + "3.10", + "3.11", + "3.12", +) + +def py_reconfig_interpreter_tests(name, python_versions, expected_interpreter_version=None, env={}, **kwargs): + for python_version in python_versions: + py_reconfig_test( + name = "{}_{}".format(name, python_version), + env = env | { + "EXPECTED_INTERPRETER_VERSION": expected_interpreter_version or python_version, + "EXPECTED_SELF_VERSION": python_version, + }, + python_version = python_version, + **kwargs + ) + + native.test_suite( + name = name, + tests = [":{}_{}".format(name, python_version) for python_version in python_versions], + ) From 682762d3ecb71de886b5b7e6057b9d7724ccb775 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 16 Feb 2025 10:58:38 -0800 Subject: [PATCH 31/34] lint --- python/bin/BUILD.bazel | 2 +- tests/interpreter/BUILD.bazel | 12 ++++++------ tests/interpreter/interpreter_tests.bzl | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index fac50fa970..57bee34378 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -9,11 +9,11 @@ filegroup( _interpreter_binary( name = "python", binary = ":python_src", - visibility = ["//visibility:public"], target_compatible_with = select({ "@platforms//os:windows": ["@platforms//:incompatible"], "//conditions:default": [], }), + visibility = ["//visibility:public"], ) # The user can modify this flag to source different interpreters for the diff --git a/tests/interpreter/BUILD.bazel b/tests/interpreter/BUILD.bazel index 3e3491af8c..57209a0d9c 100644 --- a/tests/interpreter/BUILD.bazel +++ b/tests/interpreter/BUILD.bazel @@ -12,14 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -load(":interpreter_tests.bzl", "py_reconfig_interpreter_tests", "PYTHON_VERSIONS_TO_TEST") +load(":interpreter_tests.bzl", "PYTHON_VERSIONS_TO_TEST", "py_reconfig_interpreter_tests") py_reconfig_interpreter_tests( name = "interpreter_version_test", - python_versions = PYTHON_VERSIONS_TO_TEST, - # Both the interpreter and the test itself are expected to run under - # the same version. - expected_interpreter_version = None, srcs = ["interpreter_test.py"], data = [ "//python/bin:python", @@ -27,7 +23,11 @@ py_reconfig_interpreter_tests( env = { "PYTHON_BIN": "$(rootpath //python/bin:python)", }, + # Both the interpreter and the test itself are expected to run under + # the same version. + expected_interpreter_version = None, main = "interpreter_test.py", + python_versions = PYTHON_VERSIONS_TO_TEST, ) py_reconfig_interpreter_tests( @@ -39,11 +39,11 @@ py_reconfig_interpreter_tests( env = { "PYTHON_BIN": "$(rootpath //python/bin:python)", }, - python_versions = PYTHON_VERSIONS_TO_TEST, # Since we're grabbing the interpreter from a binary with a fixed # version, we expect to always see that version. It doesn't matter what # Python version the test itself is running with. expected_interpreter_version = "3.11", main = "interpreter_test.py", python_src = "//tools/publish:twine", + python_versions = PYTHON_VERSIONS_TO_TEST, ) diff --git a/tests/interpreter/interpreter_tests.bzl b/tests/interpreter/interpreter_tests.bzl index 9045ab1b13..1fe68a3de1 100644 --- a/tests/interpreter/interpreter_tests.bzl +++ b/tests/interpreter/interpreter_tests.bzl @@ -20,7 +20,7 @@ PYTHON_VERSIONS_TO_TEST = ( "3.12", ) -def py_reconfig_interpreter_tests(name, python_versions, expected_interpreter_version=None, env={}, **kwargs): +def py_reconfig_interpreter_tests(name, python_versions, expected_interpreter_version = None, env = {}, **kwargs): for python_version in python_versions: py_reconfig_test( name = "{}_{}".format(name, python_version), From 69d647bcf6c2dcaf539e229aaab158e3cc784a73 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 16 Feb 2025 11:00:26 -0800 Subject: [PATCH 32/34] add docstring --- tests/interpreter/interpreter_tests.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/interpreter/interpreter_tests.bzl b/tests/interpreter/interpreter_tests.bzl index 1fe68a3de1..8863740d38 100644 --- a/tests/interpreter/interpreter_tests.bzl +++ b/tests/interpreter/interpreter_tests.bzl @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""This file contains helpers for testing the interpreter rule.""" + load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") PYTHON_VERSIONS_TO_TEST = ( From ebe1842e4961a9dd36b9a5ea44631753bb87c887 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 16 Feb 2025 11:13:15 -0800 Subject: [PATCH 33/34] simplify the code a bit --- tests/interpreter/BUILD.bazel | 17 ++++++++++------- tests/interpreter/interpreter_test.py | 8 +++++--- tests/interpreter/interpreter_tests.bzl | 18 ++++++++++++++++-- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/tests/interpreter/BUILD.bazel b/tests/interpreter/BUILD.bazel index 57209a0d9c..5d89ede28a 100644 --- a/tests/interpreter/BUILD.bazel +++ b/tests/interpreter/BUILD.bazel @@ -14,6 +14,9 @@ load(":interpreter_tests.bzl", "PYTHON_VERSIONS_TO_TEST", "py_reconfig_interpreter_tests") +# For this test the interpreter is sourced from the current configuration. That +# means both the interpreter and the test itself are expected to run under the +# same Python version. py_reconfig_interpreter_tests( name = "interpreter_version_test", srcs = ["interpreter_test.py"], @@ -23,13 +26,13 @@ py_reconfig_interpreter_tests( env = { "PYTHON_BIN": "$(rootpath //python/bin:python)", }, - # Both the interpreter and the test itself are expected to run under - # the same version. - expected_interpreter_version = None, main = "interpreter_test.py", python_versions = PYTHON_VERSIONS_TO_TEST, ) +# For this test the interpreter is sourced from a binary pinned at a specific +# Python version. That means the interpreter and the test itself can run +# different Python versions. py_reconfig_interpreter_tests( name = "python_src_test", srcs = ["interpreter_test.py"], @@ -37,12 +40,12 @@ py_reconfig_interpreter_tests( "//python/bin:python", ], env = { + # Since we're grabbing the interpreter from a binary with a fixed + # version, we expect to always see that version. It doesn't matter what + # Python version the test itself is running with. + "EXPECTED_INTERPRETER_VERSION": "3.11", "PYTHON_BIN": "$(rootpath //python/bin:python)", }, - # Since we're grabbing the interpreter from a binary with a fixed - # version, we expect to always see that version. It doesn't matter what - # Python version the test itself is running with. - expected_interpreter_version = "3.11", main = "interpreter_test.py", python_src = "//tools/publish:twine", python_versions = PYTHON_VERSIONS_TO_TEST, diff --git a/tests/interpreter/interpreter_test.py b/tests/interpreter/interpreter_test.py index ccd68a433b..0971fa2eba 100644 --- a/tests/interpreter/interpreter_test.py +++ b/tests/interpreter/interpreter_test.py @@ -23,15 +23,17 @@ def setUp(self): super().setUp() self.interpreter = os.environ["PYTHON_BIN"] + v = sys.version_info + self.version = f"{v.major}.{v.minor}" + def test_self_version(self): """Performs a sanity check on the Python version used for this test.""" expected_version = os.environ["EXPECTED_SELF_VERSION"] - v = sys.version_info - self.assertEqual(expected_version, f"{v.major}.{v.minor}") + self.assertEqual(expected_version, self.version) def test_interpreter_version(self): """Validates that we can successfully execute arbitrary code from the CLI.""" - expected_version = os.environ["EXPECTED_INTERPRETER_VERSION"] + expected_version = os.environ.get("EXPECTED_INTERPRETER_VERSION", self.version) try: result = subprocess.check_output( diff --git a/tests/interpreter/interpreter_tests.bzl b/tests/interpreter/interpreter_tests.bzl index 8863740d38..ad94f43423 100644 --- a/tests/interpreter/interpreter_tests.bzl +++ b/tests/interpreter/interpreter_tests.bzl @@ -16,18 +16,32 @@ load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") +# The versions of Python that we want to run the interpreter tests against. PYTHON_VERSIONS_TO_TEST = ( "3.10", "3.11", "3.12", ) -def py_reconfig_interpreter_tests(name, python_versions, expected_interpreter_version = None, env = {}, **kwargs): +def py_reconfig_interpreter_tests(name, python_versions, env = {}, **kwargs): + """Runs the specified test against each of the specified Python versions. + + One test gets generated for each Python version. The following environment + variable gets set for the test: + + EXPECTED_SELF_VERSION: Contains the Python version that the test itself + is running under. + + Args: + name: Name of the test. + python_versions: A list of Python versions to test. + env: The environment to set on the test. + **kwargs: Passed to the underlying py_reconfig_test targets. + """ for python_version in python_versions: py_reconfig_test( name = "{}_{}".format(name, python_version), env = env | { - "EXPECTED_INTERPRETER_VERSION": expected_interpreter_version or python_version, "EXPECTED_SELF_VERSION": python_version, }, python_version = python_version, From ccf77153dc0a23b27043ff2a518f38ea9b2d7bec Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 16 Feb 2025 13:10:38 -0800 Subject: [PATCH 34/34] use full runfiles path; add basic docs --- docs/api/rules_python/python/bin/index.md | 41 +++++++++++++++++++++++ python/private/common.bzl | 17 ++++++++++ python/private/interpreter.bzl | 6 ++-- python/private/interpreter_tmpl.sh | 10 +++--- python/private/py_executable.bzl | 28 ++++------------ 5 files changed, 72 insertions(+), 30 deletions(-) create mode 100644 docs/api/rules_python/python/bin/index.md diff --git a/docs/api/rules_python/python/bin/index.md b/docs/api/rules_python/python/bin/index.md new file mode 100644 index 0000000000..ad6a4e7ed5 --- /dev/null +++ b/docs/api/rules_python/python/bin/index.md @@ -0,0 +1,41 @@ +:::{default-domain} bzl +::: +:::{bzl:currentfile} //python/bin:BUILD.bazel +::: + +# //python/bin + +:::{bzl:target} python + +A target to directly run a Python interpreter. + +By default, it uses the Python version that toolchain resolution matches +(typically the one marked `is_default=True` in `MODULE.bazel`). + +This runs a Python interpreter in a similar manner as when running `python3` +on the command line. It can be invoked using `bazel run`. Remember that in +order to pass flags onto the program `--` must be specified to separate +Bazel flags from the program flags. + +An example that will run Python 3.12 and have it print the version + +``` +bazel run @rules_python//python/bin:python \ + `--@rule_python//python/config_settings:python_verion=3.12 \ + -- \ + --version +``` + +::::{seealso} +The {flag}`--python_src` flag for using the intepreter a binary/test uses. +:::: + +::::{versionadded} VERSION_NEXT_FEATURE +:::: +::: + +:::{bzl:flag} python_src + +The target (one providing `PyRuntimeInfo`) whose python interpreter to use for +{obj}`:python`. +::: diff --git a/python/private/common.bzl b/python/private/common.bzl index b6a54532d3..137f0d23f3 100644 --- a/python/private/common.bzl +++ b/python/private/common.bzl @@ -543,3 +543,20 @@ def target_platform_has_any_constraint(ctx, constraints): if ctx.target_platform_has_constraint(constraint_value): return True return False + +def runfiles_root_path(ctx, short_path): + """Compute a runfiles-root relative path from `File.short_path` + + Args: + ctx: current target ctx + short_path: str, a main-repo relative path from `File.short_path` + + Returns: + {type}`str`, a runflies-root relative path + """ + + # The ../ comes from short_path is for files in other repos. + if short_path.startswith("../"): + return short_path[3:] + else: + return "{}/{}".format(ctx.workspace_name, short_path) diff --git a/python/private/interpreter.bzl b/python/private/interpreter.bzl index 57c497642b..c66d3dc21e 100644 --- a/python/private/interpreter.bzl +++ b/python/private/interpreter.bzl @@ -16,6 +16,7 @@ load("@bazel_skylib//lib:paths.bzl", "paths") load("//python:py_runtime_info.bzl", "PyRuntimeInfo") +load(":common.bzl", "runfiles_root_path") load(":sentinel.bzl", "SentinelInfo") load(":toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") @@ -45,10 +46,7 @@ def _interpreter_binary_impl(ctx): template = ctx.file._template, output = executable, substitutions = { - # Since we never invoke this rule from within the interpreter's - # own repository, the short_path here should give us a - # predictable path of "..//". - "%target_file%": runtime.interpreter.short_path, + "%target_file%": runfiles_root_path(ctx, runtime.interpreter.short_path), }, is_executable = True, ) diff --git a/python/private/interpreter_tmpl.sh b/python/private/interpreter_tmpl.sh index 694fbb4171..cfe85ec1be 100644 --- a/python/private/interpreter_tmpl.sh +++ b/python/private/interpreter_tmpl.sh @@ -12,10 +12,12 @@ source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e # --- end runfiles.bash initialization v3 --- +set +e # allow us to check for errors more easily readonly TARGET_FILE="%target_file%" +MAIN_BIN=$(rlocation "$TARGET_FILE") -# Strip the leading "../" from "..//" so that we can do -# a runfiles lookup. -MAIN_BIN="$(rlocation "${TARGET_FILE#*/}")" - +if [[ -z "$MAIN_BIN" || ! -e "$MAIN_BIN" ]]; then + echo "ERROR: interpreter executable not found: $MAIN_BIN (from $TARGET_FILE)" + exit 1 +fi exec "${MAIN_BIN}" "$@" diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 2b2bf6636a..a2ccdc65f3 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -48,6 +48,7 @@ load( "filter_to_py_srcs", "get_imports", "is_bool", + "runfiles_root_path", "target_platform_has_any_constraint", "union_attrs", ) @@ -447,7 +448,7 @@ def _create_executable( ) def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details, venv): - python_binary = _runfiles_root_path(ctx, venv.interpreter.short_path) + python_binary = runfiles_root_path(ctx, venv.interpreter.short_path) python_binary_actual = venv.interpreter_actual_path # The location of this file doesn't really matter. It's added to @@ -522,7 +523,7 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): if not venvs_use_declare_symlink_enabled: if runtime.interpreter: - interpreter_actual_path = _runfiles_root_path(ctx, runtime.interpreter.short_path) + interpreter_actual_path = runfiles_root_path(ctx, runtime.interpreter.short_path) else: interpreter_actual_path = runtime.interpreter_path @@ -543,11 +544,11 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): # may choose to write what symlink() points to instead. interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename)) - interpreter_actual_path = _runfiles_root_path(ctx, runtime.interpreter.short_path) + interpreter_actual_path = runfiles_root_path(ctx, runtime.interpreter.short_path) rel_path = relative_path( # dirname is necessary because a relative symlink is relative to # the directory the symlink resides within. - from_ = paths.dirname(_runfiles_root_path(ctx, interpreter.short_path)), + from_ = paths.dirname(runfiles_root_path(ctx, interpreter.short_path)), to = interpreter_actual_path, ) @@ -646,23 +647,6 @@ def _create_stage2_bootstrap( ) return output -def _runfiles_root_path(ctx, short_path): - """Compute a runfiles-root relative path from `File.short_path` - - Args: - ctx: current target ctx - short_path: str, a main-repo relative path from `File.short_path` - - Returns: - {type}`str`, a runflies-root relative path - """ - - # The ../ comes from short_path is for files in other repos. - if short_path.startswith("../"): - return short_path[3:] - else: - return "{}/{}".format(ctx.workspace_name, short_path) - def _create_stage1_bootstrap( ctx, *, @@ -676,7 +660,7 @@ def _create_stage1_bootstrap( runtime = runtime_details.effective_runtime if venv: - python_binary_path = _runfiles_root_path(ctx, venv.interpreter.short_path) + python_binary_path = runfiles_root_path(ctx, venv.interpreter.short_path) else: python_binary_path = runtime_details.executable_interpreter_path