From ee2b59a6905ba03d4c12a38917900b62c229c01d Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 8 Feb 2023 15:02:56 -0800 Subject: [PATCH 01/14] fix: checked-in requirements imports generated requirements Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- .bazelci/presubmit.yml | 8 ++++++ python/pip_install/requirements.bzl | 4 +++ .../dependency_resolver.py | 26 ++++--------------- tests/compile_pip_requirements/BUILD.bazel | 3 ++- .../compile_pip_requirements/requirements.txt | 1 + 5 files changed, 20 insertions(+), 22 deletions(-) create mode 100644 tests/compile_pip_requirements/requirements.txt diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 76f9d8b5aa..a59bfe69da 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -237,21 +237,29 @@ tasks: name: compile_pip_requirements integration tests on Ubuntu working_directory: tests/compile_pip_requirements platform: ubuntu2004 + run_targets: + - "//:requirements.update" integration_test_compile_pip_requirements_debian: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on Debian working_directory: tests/compile_pip_requirements platform: debian11 + run_targets: + - "//:requirements.update" integration_test_compile_pip_requirements_macos: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on macOS working_directory: tests/compile_pip_requirements platform: macos + run_targets: + - "//:requirements.update" integration_test_compile_pip_requirements_windows: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on Windows working_directory: tests/compile_pip_requirements platform: windows + run_targets: + - "//:requirements.update" integration_test_pip_repository_entry_points_ubuntu: <<: *reusable_build_test_all diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl index 51e34a2246..00c0f09f05 100644 --- a/python/pip_install/requirements.bzl +++ b/python/pip_install/requirements.bzl @@ -103,6 +103,8 @@ def compile_pip_requirements( tags = tags or [] tags.append("requires-network") + tags.append("no-remote") + tags.append("no-sandbox") attrs = { "args": args, "data": data, @@ -113,6 +115,8 @@ def compile_pip_requirements( "visibility": visibility, } + deps.append(Label("//python/runfiles")) + # cheap way to detect the bazel version _bazel_version_4_or_greater = "propeller_optimize" in dir(native) diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index db84977a0d..d1201b09fd 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -22,6 +22,8 @@ from piptools.scripts.compile import cli +from rules_python.python.runfiles import runfiles + def _select_golden_requirements_file( requirements_txt, requirements_linux, requirements_darwin, requirements_windows @@ -62,7 +64,9 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): ) sys.exit(1) - parse_str_none = lambda s: None if s == "None" else s + r = runfiles.Create() + + parse_str_none = lambda s: None if s == "None" else r.Rlocation(s) requirements_in = sys.argv.pop(1) requirements_txt = sys.argv.pop(1) @@ -101,26 +105,6 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): ) copyfile(requirements_txt, requirements_out) - elif "BUILD_WORKSPACE_DIRECTORY" in os.environ: - # This value, populated when running under `bazel run`, is a path to the - # "root of the workspace where the build was run." - # This matches up with the values passed in via the macro using the 'rootpath' Make variable, - # which for source files provides a path "relative to your workspace root." - # - # Changing to the WORKSPACE root avoids 'file not found' errors when the `.update` target is run - # from different directories within the WORKSPACE. - os.chdir(os.environ["BUILD_WORKSPACE_DIRECTORY"]) - else: - err_msg = ( - "Expected to find BUILD_WORKSPACE_DIRECTORY (running under `bazel run`) or " - "TEST_TMPDIR (running under `bazel test`) in environment." - ) - print( - err_msg, - file=sys.stderr, - ) - sys.exit(1) - update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run %s" % ( update_target_label, ) diff --git a/tests/compile_pip_requirements/BUILD.bazel b/tests/compile_pip_requirements/BUILD.bazel index 3a67dcca47..d6ac0086ab 100644 --- a/tests/compile_pip_requirements/BUILD.bazel +++ b/tests/compile_pip_requirements/BUILD.bazel @@ -22,12 +22,13 @@ EOF compile_pip_requirements( name = "requirements", data = [ + "requirements.in", "requirements_extra.in", ], extra_args = [ "--allow-unsafe", "--resolver=backtracking", ], - requirements_in = "requirements.in", + requirements_in = "requirements.txt", requirements_txt = "requirements_lock.txt", ) diff --git a/tests/compile_pip_requirements/requirements.txt b/tests/compile_pip_requirements/requirements.txt new file mode 100644 index 0000000000..4826399f01 --- /dev/null +++ b/tests/compile_pip_requirements/requirements.txt @@ -0,0 +1 @@ +-r requirements.in From c6eb1af9ab8a0b2aca5c99e5d22a39ed8cf55451 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 8 Feb 2023 15:14:30 -0800 Subject: [PATCH 02/14] fix: shutil.copy instead of shutil.copyfile This allows copying from one filesystem to another, as the `os.rename` (used by copyfile) doesn't work with multiple filesystems. Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- .../tools/dependency_resolver/dependency_resolver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index d1201b09fd..ad4c3fa2e9 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -18,7 +18,7 @@ import re import sys from pathlib import Path -from shutil import copyfile +from shutil import copy from piptools.scripts.compile import cli @@ -103,7 +103,7 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): requirements_out = os.path.join( os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out" ) - copyfile(requirements_txt, requirements_out) + copy(requirements_txt, requirements_out) update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run %s" % ( update_target_label, From 706863ed11334c04b0063b74f3497ffb937b679f Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 8 Feb 2023 15:24:15 -0800 Subject: [PATCH 03/14] fix: patch os.replace to use shutil.copy Same as the previous commit. Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- .../tools/dependency_resolver/dependency_resolver.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index ad4c3fa2e9..0e36ed1732 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -16,14 +16,16 @@ import os import re +import shutil import sys from pathlib import Path -from shutil import copy from piptools.scripts.compile import cli from rules_python.python.runfiles import runfiles +os.replace = shutil.copy + def _select_golden_requirements_file( requirements_txt, requirements_linux, requirements_darwin, requirements_windows @@ -103,7 +105,7 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): requirements_out = os.path.join( os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out" ) - copy(requirements_txt, requirements_out) + shutil.copy(requirements_txt, requirements_out) update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run %s" % ( update_target_label, From 4a65474f22e3987c0ab106d03057fe68ff3fb27c Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 8 Feb 2023 15:29:09 -0800 Subject: [PATCH 04/14] fix: runfiles.Rlocation requires paths to be normalized Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- .../tools/dependency_resolver/dependency_resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index 0e36ed1732..05c20c15c1 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -68,7 +68,7 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): r = runfiles.Create() - parse_str_none = lambda s: None if s == "None" else r.Rlocation(s) + parse_str_none = lambda s: None if s == "None" else r.Rlocation(os.path.normpath(s)) requirements_in = sys.argv.pop(1) requirements_txt = sys.argv.pop(1) From 850ad7ea82302d5325b631865507cf4b725ad14f Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 8 Feb 2023 15:37:36 -0800 Subject: [PATCH 05/14] fix: drop rules_python from import This is not compatible with bzlmod. Importing python.runfiles works for both ways. Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- .../tools/dependency_resolver/dependency_resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index 05c20c15c1..9f5cacd61f 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -22,7 +22,7 @@ from piptools.scripts.compile import cli -from rules_python.python.runfiles import runfiles +from python.runfiles import runfiles os.replace = shutil.copy From b89468e9f22ddc7404be83b100cc186fbe1a9150 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 8 Feb 2023 15:52:26 -0800 Subject: [PATCH 06/14] fix: remove unnecessary runfiles Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- python/pip_install/requirements.bzl | 2 -- .../tools/dependency_resolver/dependency_resolver.py | 6 +----- tools/publish/BUILD.bazel | 8 -------- 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl index 00c0f09f05..738a2d1c3c 100644 --- a/python/pip_install/requirements.bzl +++ b/python/pip_install/requirements.bzl @@ -115,8 +115,6 @@ def compile_pip_requirements( "visibility": visibility, } - deps.append(Label("//python/runfiles")) - # cheap way to detect the bazel version _bazel_version_4_or_greater = "propeller_optimize" in dir(native) diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index 9f5cacd61f..185573978e 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -22,8 +22,6 @@ from piptools.scripts.compile import cli -from python.runfiles import runfiles - os.replace = shutil.copy @@ -66,9 +64,7 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): ) sys.exit(1) - r = runfiles.Create() - - parse_str_none = lambda s: None if s == "None" else r.Rlocation(os.path.normpath(s)) + parse_str_none = lambda s: None if s == "None" else s requirements_in = sys.argv.pop(1) requirements_txt = sys.argv.pop(1) diff --git a/tools/publish/BUILD.bazel b/tools/publish/BUILD.bazel index 8c4b3ab4a2..065e56bd69 100644 --- a/tools/publish/BUILD.bazel +++ b/tools/publish/BUILD.bazel @@ -4,12 +4,4 @@ compile_pip_requirements( name = "requirements", requirements_darwin = "requirements_darwin.txt", requirements_windows = "requirements_windows.txt", - # This fails on RBE right now, and we don't need coverage there: - # WARNING: Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) - # after connection broken by 'NewConnectionError(': - # Failed to establish a new connection: [Errno -3] Temporary failure in name resolution')': /simple/twine/ - # - # ERROR: Could not find a version that satisfies the requirement twine==4.0.2 - # (from -r tools/publish/requirements.in (line 1)) (from versions: none) - tags = ["no-remote-exec"], ) From 7ea085ad3d890a2f5760c4589c64248e6dad55c8 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 8 Feb 2023 15:55:40 -0800 Subject: [PATCH 07/14] doc: why os.replace = shutil.copy Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- .../tools/dependency_resolver/dependency_resolver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index 185573978e..6c59b00456 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -22,6 +22,8 @@ from piptools.scripts.compile import cli +# Replace the os.replace function with shutil.copy to work around os.replace not being able to +# replace or move files across filesystems. os.replace = shutil.copy From 08096d4b89a8bde9728ee7c495541eaad4836c68 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 8 Feb 2023 16:14:39 -0800 Subject: [PATCH 08/14] fix: allow the test to still be remote cacheable Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- python/pip_install/requirements.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl index 738a2d1c3c..af3c194d18 100644 --- a/python/pip_install/requirements.bzl +++ b/python/pip_install/requirements.bzl @@ -103,7 +103,7 @@ def compile_pip_requirements( tags = tags or [] tags.append("requires-network") - tags.append("no-remote") + tags.append("no-remote-exec") tags.append("no-sandbox") attrs = { "args": args, From 60916fd321de686a762fcf8fffc8110dbc2992f1 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 8 Feb 2023 16:43:20 -0800 Subject: [PATCH 09/14] doc: why shutil.copy Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- .../tools/dependency_resolver/dependency_resolver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index 6c59b00456..0ad4acd12f 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -103,6 +103,8 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): requirements_out = os.path.join( os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out" ) + # Those two files won't necessarily be on the same filesystem, so we can't use os.replace + # or shutil.copyfile, as they will fail with OSError: [Errno 18] Invalid cross-device link. shutil.copy(requirements_txt, requirements_out) update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run %s" % ( From 1e1279e4be74eb9239bebbf94ada54571ef7dbbf Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 8 Feb 2023 16:43:52 -0800 Subject: [PATCH 10/14] doc: add missing punctuation Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- .../tools/dependency_resolver/dependency_resolver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index 0ad4acd12f..561c7d438c 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -90,7 +90,7 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): os.environ["LANG"] = "C.UTF-8" UPDATE = True - # Detect if we are running under `bazel test` + # Detect if we are running under `bazel test`. if "TEST_TMPDIR" in os.environ: UPDATE = False # pip-compile wants the cache files to be writeable, but if we point @@ -99,7 +99,7 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): # In theory this makes the test more hermetic as well. sys.argv.append("--cache-dir") sys.argv.append(os.environ["TEST_TMPDIR"]) - # Make a copy for pip-compile to read and mutate + # Make a copy for pip-compile to read and mutate. requirements_out = os.path.join( os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out" ) From e8ca1002646e327be13858b44aaa163b9d3a0e13 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 8 Feb 2023 16:44:12 -0800 Subject: [PATCH 11/14] fix: remove unnecessary _fix_up_requirements_in_path Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- .../dependency_resolver.py | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index 561c7d438c..5dbf472ad1 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -45,19 +45,6 @@ def _select_golden_requirements_file( return requirements_txt -def _fix_up_requirements_in_path(absolute_prefix, output_file): - """Fix up references to the input file inside of the generated requirements file. - - We don't want fully resolved, absolute paths in the generated requirements file. - The paths could differ for every invocation. Replace them with a predictable path. - """ - output_file = Path(output_file) - contents = output_file.read_text() - contents = contents.replace(absolute_prefix, "") - contents = re.sub(r"\\(?!(\n|\r\n))", "/", contents) - output_file.write_text(contents) - - if __name__ == "__main__": if len(sys.argv) < 4: print( @@ -79,7 +66,6 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): # absolute prefixes in the locked requirements output file. requirements_in_path = Path(requirements_in) resolved_requirements_in = str(requirements_in_path.resolve()) - absolute_prefix = resolved_requirements_in[: -len(str(requirements_in_path))] # Before loading click, set the locale for its parser. # If it leaks through to the system setting, it may fail: @@ -123,12 +109,7 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): if UPDATE: print("Updating " + requirements_txt) - try: - cli() - except SystemExit as e: - if e.code == 0: - _fix_up_requirements_in_path(absolute_prefix, requirements_txt) - raise + cli() else: # cli will exit(0) on success try: @@ -146,7 +127,6 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): ) sys.exit(1) elif e.code == 0: - _fix_up_requirements_in_path(absolute_prefix, requirements_out) golden_filename = _select_golden_requirements_file( requirements_txt, requirements_linux, From ab53ad2e265fcf05a93d16a35bdeb3a8b22bfba9 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Mon, 13 Feb 2023 12:56:27 -0800 Subject: [PATCH 12/14] test: make sure the locked requirements is updated Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- .bazelci/presubmit.yml | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 8c61d89161..5f92fbff04 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -289,29 +289,49 @@ tasks: name: compile_pip_requirements integration tests on Ubuntu working_directory: tests/compile_pip_requirements platform: ubuntu2004 - run_targets: - - "//:requirements.update" + shell_commands: + # Make a change to the locked requirements and then assert that //:requirements.update does the + # right thing. + - "echo '' > requirements_lock.txt" + - "! git diff --exit-code" + - "bazel run //:requirements.update" + - "git diff --exit-code" integration_test_compile_pip_requirements_debian: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on Debian working_directory: tests/compile_pip_requirements platform: debian11 - run_targets: - - "//:requirements.update" + shell_commands: + # Make a change to the locked requirements and then assert that //:requirements.update does the + # right thing. + - "echo '' > requirements_lock.txt" + - "! git diff --exit-code" + - "bazel run //:requirements.update" + - "git diff --exit-code" integration_test_compile_pip_requirements_macos: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on macOS working_directory: tests/compile_pip_requirements platform: macos - run_targets: - - "//:requirements.update" + shell_commands: + # Make a change to the locked requirements and then assert that //:requirements.update does the + # right thing. + - "echo '' > requirements_lock.txt" + - "! git diff --exit-code" + - "bazel run //:requirements.update" + - "git diff --exit-code" integration_test_compile_pip_requirements_windows: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on Windows working_directory: tests/compile_pip_requirements platform: windows - run_targets: - - "//:requirements.update" + shell_commands: + # Make a change to the locked requirements and then assert that //:requirements.update does the + # right thing. + - "echo '' > requirements_lock.txt" + - "! git diff --exit-code" + - "bazel run //:requirements.update" + - "git diff --exit-code" integration_test_pip_repository_entry_points_ubuntu: <<: *reusable_build_test_all From a41b8cadfd97f5081c484439cc50118796257992 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Mon, 13 Feb 2023 14:09:24 -0800 Subject: [PATCH 13/14] fix: copy requirements back into src tree if needed Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- .../tools/dependency_resolver/dependency_resolver.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index 5dbf472ad1..59230629d3 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -14,8 +14,8 @@ "Set defaults for the pip-compile command to run it under Bazel" +import atexit import os -import re import shutil import sys from pathlib import Path @@ -109,6 +109,16 @@ def _select_golden_requirements_file( if UPDATE: print("Updating " + requirements_txt) + if "BUILD_WORKSPACE_DIRECTORY" in os.environ: + workspace = os.environ["BUILD_WORKSPACE_DIRECTORY"] + requirements_txt_tree = os.path.join(workspace, requirements_txt) + # In most cases, requirements_txt will be a symlink to the real file in the source tree. + # If symlinks are not enabled (e.g. on Windows), then requirements_txt will be a copy, + # and we should copy the updated requirements back to the source tree. + if not os.path.samefile(requirements_txt, requirements_txt_tree): + atexit.register( + lambda: shutil.copy(requirements_txt, requirements_txt_tree) + ) cli() else: # cli will exit(0) on success From 5b60e91e5897b8bba0e1fe65d5a7d600fae31278 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Mon, 13 Feb 2023 14:24:20 -0800 Subject: [PATCH 14/14] fix: make sure windows uses forward slashes Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- .../dependency_resolver.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index 59230629d3..e636febd93 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -20,12 +20,33 @@ import sys from pathlib import Path +import piptools.writer as piptools_writer from piptools.scripts.compile import cli # Replace the os.replace function with shutil.copy to work around os.replace not being able to # replace or move files across filesystems. os.replace = shutil.copy +# Next, we override the annotation_style_split and annotation_style_line functions to replace the +# backslashes in the paths with forward slashes. This is so that we can have the same requirements +# file on Windows and Unix-like. +original_annotation_style_split = piptools_writer.annotation_style_split +original_annotation_style_line = piptools_writer.annotation_style_line + + +def annotation_style_split(required_by) -> str: + required_by = set([v.replace("\\", "/") for v in required_by]) + return original_annotation_style_split(required_by) + + +def annotation_style_line(required_by) -> str: + required_by = set([v.replace("\\", "/") for v in required_by]) + return original_annotation_style_line(required_by) + + +piptools_writer.annotation_style_split = annotation_style_split +piptools_writer.annotation_style_line = annotation_style_line + def _select_golden_requirements_file( requirements_txt, requirements_linux, requirements_darwin, requirements_windows