From 8cc8aec5364464cf2f868e57fdb7972ac578ac0b Mon Sep 17 00:00:00 2001 From: Armando Montanez Date: Mon, 16 Jun 2025 12:01:03 -0700 Subject: [PATCH 1/6] fix: Fix bazel vendor support for requirements with environment markers Fixes `bazel vendor` support for requirements files that contain environment markers. During a vendored `bazel build`, when evaluate_markers_py() is run it needs PYTHONHOME set to properly find the home of the vendored libraries. Resolves (#2996) --- .bazelci/presubmit.yml | 9 +++++++++ examples/pip_parse/.bazelignore | 1 + examples/pip_parse/.gitignore | 1 + examples/pip_parse/requirements.in | 4 +++- examples/pip_parse/requirements_lock.txt | 2 +- python/private/pypi/evaluate_markers.bzl | 13 ++++++++----- 6 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 examples/pip_parse/.bazelignore diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 01af217924..6afdfad362 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -495,6 +495,15 @@ tasks: name: "tests/integration bazel-in-bazel: Debian" platform: debian11 + + integration_test_bazel_vendor_ubuntu: + <<: *reusable_build_test_all + name: "compile_pip_requirements: Ubuntu" + working_directory: tests/integration/pip_parse + platform: ubuntu2004 + shell_commands: + - "bazel vendor --vendor_dir=./vendor //..." + - "bazel build --vendor_dir=./vendor //..." integration_test_compile_pip_requirements_ubuntu: <<: *reusable_build_test_all name: "compile_pip_requirements: Ubuntu" diff --git a/examples/pip_parse/.bazelignore b/examples/pip_parse/.bazelignore new file mode 100644 index 0000000000..48b8bf9072 --- /dev/null +++ b/examples/pip_parse/.bazelignore @@ -0,0 +1 @@ +vendor/ diff --git a/examples/pip_parse/.gitignore b/examples/pip_parse/.gitignore index e5ae073b3c..29ab74b9be 100644 --- a/examples/pip_parse/.gitignore +++ b/examples/pip_parse/.gitignore @@ -1,4 +1,5 @@ # git ignore patterns /bazel-* +/vendor user.bazelrc diff --git a/examples/pip_parse/requirements.in b/examples/pip_parse/requirements.in index 9d9e766d21..6c9b405e23 100644 --- a/examples/pip_parse/requirements.in +++ b/examples/pip_parse/requirements.in @@ -1,5 +1,7 @@ requests~=2.25.1 s3cmd~=2.1.0 -yamllint~=1.28.0 +# Just to ensure environment markers are handled properly. +yamllint~=1.28.0; python_version == '3.9' +yamllint~=1.28.0; python_version != '3.9' sphinx sphinxcontrib-serializinghtml diff --git a/examples/pip_parse/requirements_lock.txt b/examples/pip_parse/requirements_lock.txt index dc34b45a45..072b887e5c 100644 --- a/examples/pip_parse/requirements_lock.txt +++ b/examples/pip_parse/requirements_lock.txt @@ -226,7 +226,7 @@ urllib3==1.26.20 \ # via # -c ./constraints_urllib3.txt # requests -yamllint==1.28.0 \ +yamllint==1.28.0 ; python_version == "3.9" \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b # via -r requirements.in diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl index 191933596e..1cd3d66f2a 100644 --- a/python/private/pypi/evaluate_markers.bzl +++ b/python/private/pypi/evaluate_markers.bzl @@ -75,14 +75,16 @@ def evaluate_markers_py(mrctx, *, requirements, python_interpreter, python_inter out_file = mrctx.path("requirements_with_markers.out.json") mrctx.file(in_file, json.encode(requirements)) + interpreter = pypi_repo_utils.resolve_python_interpreter( + mrctx, + python_interpreter = python_interpreter, + python_interpreter_target = python_interpreter_target, + ) + pypi_repo_utils.execute_checked( mrctx, op = "ResolveRequirementEnvMarkers({})".format(in_file), - python = pypi_repo_utils.resolve_python_interpreter( - mrctx, - python_interpreter = python_interpreter, - python_interpreter_target = python_interpreter_target, - ), + python = interpreter, arguments = [ "-m", "python.private.pypi.requirements_parser.resolve_target_platforms", @@ -95,6 +97,7 @@ def evaluate_markers_py(mrctx, *, requirements, python_interpreter, python_inter Label("@pypi__packaging//:BUILD.bazel"), Label("//:BUILD.bazel"), ], + "PYTHONHOME": str(interpreter.dirname), }, logger = logger, ) From 561fb6c0edbcf087d9feee293f5cd7c19a9b3b21 Mon Sep 17 00:00:00 2001 From: Armando Montanez Date: Mon, 16 Jun 2025 12:08:37 -0700 Subject: [PATCH 2/6] Delete vendor dir after integration test --- .bazelci/presubmit.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 6afdfad362..5df8292170 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -504,6 +504,7 @@ tasks: shell_commands: - "bazel vendor --vendor_dir=./vendor //..." - "bazel build --vendor_dir=./vendor //..." + - "rm -rf ./vendor" integration_test_compile_pip_requirements_ubuntu: <<: *reusable_build_test_all name: "compile_pip_requirements: Ubuntu" From 861c346f9f046b0e22dc0862a316cbfcffb2d995 Mon Sep 17 00:00:00 2001 From: Armando Montanez Date: Fri, 20 Jun 2025 09:19:00 -0700 Subject: [PATCH 3/6] Switch check to use examples/bzlmod --- .bazelci/presubmit.yml | 4 ++-- examples/bzlmod/.bazelignore | 1 + examples/bzlmod/.gitignore | 1 + examples/pip_parse/.bazelignore | 1 - examples/pip_parse/.gitignore | 1 - examples/pip_parse/requirements.in | 4 +--- examples/pip_parse/requirements_lock.txt | 2 +- 7 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 examples/pip_parse/.bazelignore diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 5df8292170..85e6d23d19 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -498,8 +498,8 @@ tasks: integration_test_bazel_vendor_ubuntu: <<: *reusable_build_test_all - name: "compile_pip_requirements: Ubuntu" - working_directory: tests/integration/pip_parse + name: "test_bazel_vendor: Ubuntu" + working_directory: examples/bzlmod platform: ubuntu2004 shell_commands: - "bazel vendor --vendor_dir=./vendor //..." diff --git a/examples/bzlmod/.bazelignore b/examples/bzlmod/.bazelignore index 3927f8e910..536ded93a6 100644 --- a/examples/bzlmod/.bazelignore +++ b/examples/bzlmod/.bazelignore @@ -1,2 +1,3 @@ other_module py_proto_library/foo_external +vendor diff --git a/examples/bzlmod/.gitignore b/examples/bzlmod/.gitignore index ac51a054d2..0f6c6316dd 100644 --- a/examples/bzlmod/.gitignore +++ b/examples/bzlmod/.gitignore @@ -1 +1,2 @@ bazel-* +vendor/ diff --git a/examples/pip_parse/.bazelignore b/examples/pip_parse/.bazelignore deleted file mode 100644 index 48b8bf9072..0000000000 --- a/examples/pip_parse/.bazelignore +++ /dev/null @@ -1 +0,0 @@ -vendor/ diff --git a/examples/pip_parse/.gitignore b/examples/pip_parse/.gitignore index 29ab74b9be..e5ae073b3c 100644 --- a/examples/pip_parse/.gitignore +++ b/examples/pip_parse/.gitignore @@ -1,5 +1,4 @@ # git ignore patterns /bazel-* -/vendor user.bazelrc diff --git a/examples/pip_parse/requirements.in b/examples/pip_parse/requirements.in index 6c9b405e23..9d9e766d21 100644 --- a/examples/pip_parse/requirements.in +++ b/examples/pip_parse/requirements.in @@ -1,7 +1,5 @@ requests~=2.25.1 s3cmd~=2.1.0 -# Just to ensure environment markers are handled properly. -yamllint~=1.28.0; python_version == '3.9' -yamllint~=1.28.0; python_version != '3.9' +yamllint~=1.28.0 sphinx sphinxcontrib-serializinghtml diff --git a/examples/pip_parse/requirements_lock.txt b/examples/pip_parse/requirements_lock.txt index 072b887e5c..dc34b45a45 100644 --- a/examples/pip_parse/requirements_lock.txt +++ b/examples/pip_parse/requirements_lock.txt @@ -226,7 +226,7 @@ urllib3==1.26.20 \ # via # -c ./constraints_urllib3.txt # requests -yamllint==1.28.0 ; python_version == "3.9" \ +yamllint==1.28.0 \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b # via -r requirements.in From 668e873019c824a3fc5a37d45c7583200ce02f4e Mon Sep 17 00:00:00 2001 From: Armando Montanez Date: Fri, 20 Jun 2025 09:23:36 -0700 Subject: [PATCH 4/6] Add changelog note --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2fa98f73f..b3d2695f82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,9 @@ END_UNRELEASED_TEMPLATE {#v0-0-0-fixed} ### Fixed -* Nothing fixed. +* (pypi) Fixes an issue where builds using a `bazel vendor` vendor directory + would fail if the constraints file contained environment markers. Fixes + [#2996](https://github.com/bazel-contrib/rules_python/issues/2996). {#v0-0-0-added} ### Added From b2ad12ebc7105af85cd86737057145f83bc52452 Mon Sep 17 00:00:00 2001 From: Armando Montanez Date: Fri, 20 Jun 2025 09:26:25 -0700 Subject: [PATCH 5/6] Fix buildifier lint findings --- python/private/pypi/evaluate_markers.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl index d03e61e456..2b805c33e6 100644 --- a/python/private/pypi/evaluate_markers.bzl +++ b/python/private/pypi/evaluate_markers.bzl @@ -96,11 +96,11 @@ def evaluate_markers_py(mrctx, *, requirements, python_interpreter, python_inter ], srcs = srcs, environment = { + "PYTHONHOME": str(interpreter.dirname), "PYTHONPATH": [ Label("@pypi__packaging//:BUILD.bazel"), Label("//:BUILD.bazel"), ], - "PYTHONHOME": str(interpreter.dirname), }, logger = logger, ) From a410e2951e0bcd27c315642bd4d6290ae3597b41 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Jun 2025 11:09:01 +0900 Subject: [PATCH 6/6] move ci block next to all examples/bzlmod invocations --- .bazelci/presubmit.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 85e6d23d19..07ffa4eaac 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -272,6 +272,15 @@ tasks: working_directory: examples/bzlmod platform: debian11 bazel: 7.x + integration_test_bzlmod_ubuntu_vendor: + <<: *reusable_build_test_all + name: "examples/bzlmod: bazel vendor" + working_directory: examples/bzlmod + platform: ubuntu2004 + shell_commands: + - "bazel vendor --vendor_dir=./vendor //..." + - "bazel build --vendor_dir=./vendor //..." + - "rm -rf ./vendor" integration_test_bzlmod_macos: <<: *reusable_build_test_all <<: *coverage_targets_example_bzlmod @@ -495,16 +504,6 @@ tasks: name: "tests/integration bazel-in-bazel: Debian" platform: debian11 - - integration_test_bazel_vendor_ubuntu: - <<: *reusable_build_test_all - name: "test_bazel_vendor: Ubuntu" - working_directory: examples/bzlmod - platform: ubuntu2004 - shell_commands: - - "bazel vendor --vendor_dir=./vendor //..." - - "bazel build --vendor_dir=./vendor //..." - - "rm -rf ./vendor" integration_test_compile_pip_requirements_ubuntu: <<: *reusable_build_test_all name: "compile_pip_requirements: Ubuntu"