From 4dd16ed2a53af5c51f868882a9eeaa00be9fb5c3 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 30 Nov 2022 12:37:50 -0800 Subject: [PATCH 1/5] fix: pip_compile to handle multiple generated requirements.in Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- examples/pip_parse/BUILD | 10 ------ ...quirements.in.template => requirements.in} | 0 python/pip_install/pip_compile.py | 24 +++++--------- tests/compile_pip_requirements/.bazelrc | 5 +++ tests/compile_pip_requirements/.gitignore | 1 + tests/compile_pip_requirements/BUILD.bazel | 33 +++++++++++++++++++ tests/compile_pip_requirements/README.md | 3 ++ tests/compile_pip_requirements/WORKSPACE | 8 +++++ .../requirements_lock.txt | 16 +++++++++ 9 files changed, 75 insertions(+), 25 deletions(-) rename examples/pip_parse/{requirements.in.template => requirements.in} (100%) create mode 100644 tests/compile_pip_requirements/.bazelrc create mode 100644 tests/compile_pip_requirements/.gitignore create mode 100644 tests/compile_pip_requirements/BUILD.bazel create mode 100644 tests/compile_pip_requirements/README.md create mode 100644 tests/compile_pip_requirements/WORKSPACE create mode 100644 tests/compile_pip_requirements/requirements_lock.txt diff --git a/examples/pip_parse/BUILD b/examples/pip_parse/BUILD index 1b6ba55f5e..653f75ce2b 100644 --- a/examples/pip_parse/BUILD +++ b/examples/pip_parse/BUILD @@ -55,16 +55,6 @@ alias( actual = entry_point("yamllint"), ) -# The requirements.in file can be checked in to the source tree or it can be -# generated. Pretend that we do some generating of the file. For this example, -# the "template" is already the file we want. -genrule( - name = "generate_requirements_in", - srcs = ["requirements.in.template"], - outs = ["requirements.in"], - cmd = "cp $(SRCS) $(OUTS)", -) - # This rule adds a convenient way to update the requirements file. compile_pip_requirements( name = "requirements", diff --git a/examples/pip_parse/requirements.in.template b/examples/pip_parse/requirements.in similarity index 100% rename from examples/pip_parse/requirements.in.template rename to examples/pip_parse/requirements.in diff --git a/python/pip_install/pip_compile.py b/python/pip_install/pip_compile.py index c9bcf32115..c60b866581 100644 --- a/python/pip_install/pip_compile.py +++ b/python/pip_install/pip_compile.py @@ -26,18 +26,14 @@ def _select_golden_requirements_file( return requirements_txt -def _fix_up_requirements_in_path( - resolved_requirements_in, requirements_in, output_file -): +def _fix_up_requirements_in_path(absolute_prefix, output_file): """Fix up references to the input file inside of the generated requirements file. We don't want fully resolved, absolute paths in the generated requirements file. The paths could differ for every invocation. Replace them with a predictable path. """ output_file = Path(output_file) - fixed_requirements_text = output_file.read_text().replace( - resolved_requirements_in, requirements_in - ) + fixed_requirements_text = output_file.read_text().replace(absolute_prefix, "") output_file.write_text(fixed_requirements_text) @@ -58,9 +54,11 @@ def _fix_up_requirements_in_path( requirements_windows = parse_str_none(sys.argv.pop(1)) update_target_label = sys.argv.pop(1) - # The requirements_in file could be generated. We need to get the path to it before we change - # directory into the workspace directory. - resolved_requirements_in = str(Path(requirements_in).resolve()) + # The requirements_in file could be generated, so we will need to remove the + # absolute prefixes in the locked requirements output file. + requirements_in_path = Path(requirements_in) + resolved_requirements_in = str(requirements_in_path.resolve()) + absolute_prefix = resolved_requirements_in[: -len(str(requirements_in_path))] # Before loading click, set the locale for its parser. # If it leaks through to the system setting, it may fail: @@ -124,9 +122,7 @@ def _fix_up_requirements_in_path( cli() except SystemExit as e: if e.code == 0: - _fix_up_requirements_in_path( - resolved_requirements_in, requirements_in, requirements_txt - ) + _fix_up_requirements_in_path(absolute_prefix, requirements_txt) raise else: # cli will exit(0) on success @@ -145,9 +141,7 @@ def _fix_up_requirements_in_path( ) sys.exit(1) elif e.code == 0: - _fix_up_requirements_in_path( - resolved_requirements_in, requirements_in, requirements_out - ) + _fix_up_requirements_in_path(absolute_prefix, requirements_out) golden_filename = _select_golden_requirements_file( requirements_txt, requirements_linux, diff --git a/tests/compile_pip_requirements/.bazelrc b/tests/compile_pip_requirements/.bazelrc new file mode 100644 index 0000000000..f23315a7a1 --- /dev/null +++ b/tests/compile_pip_requirements/.bazelrc @@ -0,0 +1,5 @@ +test --test_output=errors + +# Windows requires these for multi-python support: +build --enable_runfiles +startup --windows_enable_symlinks diff --git a/tests/compile_pip_requirements/.gitignore b/tests/compile_pip_requirements/.gitignore new file mode 100644 index 0000000000..ac51a054d2 --- /dev/null +++ b/tests/compile_pip_requirements/.gitignore @@ -0,0 +1 @@ +bazel-* diff --git a/tests/compile_pip_requirements/BUILD.bazel b/tests/compile_pip_requirements/BUILD.bazel new file mode 100644 index 0000000000..258bb71ad5 --- /dev/null +++ b/tests/compile_pip_requirements/BUILD.bazel @@ -0,0 +1,33 @@ +load("@rules_python//python:pip.bzl", "compile_pip_requirements") + +genrule( + name = "generate_requirements_extra_in", + srcs = [], + outs = ["requirements_extra.in"], + cmd = "echo 'setuptools~=65.6.3' > $@", +) + +genrule( + name = "generate_requirements_in", + srcs = [], + outs = ["requirements.in"], + cmd = """ +cat > $@ < Date: Wed, 30 Nov 2022 12:49:22 -0800 Subject: [PATCH 2/5] fix: only use requirements.in absolute when generated Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- python/pip_install/pip_compile.py | 6 ++++-- tests/compile_pip_requirements/requirements_lock.txt | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/python/pip_install/pip_compile.py b/python/pip_install/pip_compile.py index c60b866581..89efbd911e 100644 --- a/python/pip_install/pip_compile.py +++ b/python/pip_install/pip_compile.py @@ -33,7 +33,7 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): The paths could differ for every invocation. Replace them with a predictable path. """ output_file = Path(output_file) - fixed_requirements_text = output_file.read_text().replace(absolute_prefix, "") + fixed_requirements_text = output_file.read_text().replace(absolute_prefix, "./") output_file.write_text(fixed_requirements_text) @@ -114,7 +114,9 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): sys.argv.append("--generate-hashes") sys.argv.append("--output-file") sys.argv.append(requirements_txt if UPDATE else requirements_out) - sys.argv.append(resolved_requirements_in) + sys.argv.append( + requirements_in if requirements_in_path.exists() else resolved_requirements_in + ) if UPDATE: print("Updating " + requirements_txt) diff --git a/tests/compile_pip_requirements/requirements_lock.txt b/tests/compile_pip_requirements/requirements_lock.txt index f32fbb9fbf..d8c453ddb7 100644 --- a/tests/compile_pip_requirements/requirements_lock.txt +++ b/tests/compile_pip_requirements/requirements_lock.txt @@ -9,8 +9,8 @@ pip==22.3.1 \ --hash=sha256:65fd48317359f3af8e593943e6ae1506b66325085ea64b706a998c6e83eeaf38 \ --hash=sha256:908c78e6bc29b676ede1c4d57981d490cb892eb45cd8c214ab6298125119e077 - # via -r requirements.in + # via -r ./requirements.in setuptools==65.6.3 \ --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 - # via -r requirements_extra.in + # via -r ./requirements_extra.in From bd753792c659322785f5ca38f9ce2da88d69af55 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 30 Nov 2022 12:51:26 -0800 Subject: [PATCH 3/5] fix: use hermetic toolchain to lock the requirements Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- tests/compile_pip_requirements/WORKSPACE | 7 +++++++ tests/compile_pip_requirements/requirements_lock.txt | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/compile_pip_requirements/WORKSPACE b/tests/compile_pip_requirements/WORKSPACE index be6661dee0..d3fd700911 100644 --- a/tests/compile_pip_requirements/WORKSPACE +++ b/tests/compile_pip_requirements/WORKSPACE @@ -6,3 +6,10 @@ local_repository( load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") pip_install_dependencies() + +load("@rules_python//python:repositories.bzl", "python_register_toolchains") + +python_register_toolchains( + name = "python39", + python_version = "3.9", +) diff --git a/tests/compile_pip_requirements/requirements_lock.txt b/tests/compile_pip_requirements/requirements_lock.txt index d8c453ddb7..78ced94440 100644 --- a/tests/compile_pip_requirements/requirements_lock.txt +++ b/tests/compile_pip_requirements/requirements_lock.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # bazel run //:pip_dependencies.update From 1eeabd827000e55aef843bdbee994e2ac6be8ea3 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 30 Nov 2022 12:54:27 -0800 Subject: [PATCH 4/5] fix: tests on CI Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- .bazelci/presubmit.yml | 16 ++++++++++++++++ .bazelrc | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 56516e975b..641416f2cb 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -167,6 +167,22 @@ tasks: working_directory: examples/pip_repository_annotations platform: windows + integration_test_compile_pip_requirements_linux: + <<: *reusable_build_test_all + name: compile_pip_requirements integration tests on Linux + working_directory: tests/compile_pip_requirements + platform: ubuntu2004 + integration_test_compile_pip_requirements_macos: + <<: *reusable_build_test_all + name: compile_pip_requirements integration tests on macOS + working_directory: tests/compile_pip_requirements + platform: macos + integration_test_compile_pip_requirements_windows: + <<: *reusable_build_test_all + name: compile_pip_requirements integration tests on Windows + working_directory: tests/compile_pip_requirements + platform: windows + integration_test_pip_repository_entry_points_linux: <<: *reusable_build_test_all name: pip_repository_entry_points integration tests on Linux diff --git a/.bazelrc b/.bazelrc index 510191b847..f1c7b7a210 100644 --- a/.bazelrc +++ b/.bazelrc @@ -3,8 +3,8 @@ # This lets us glob() up all the files inside the examples to make them inputs to tests # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh -build --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/pip_repository_entry_points,tests/pip_deps -query --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/pip_repository_entry_points,tests/pip_deps +build --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/compile_pip_requirements,tests/pip_repository_entry_points,tests/pip_deps +query --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/compile_pip_requirements,tests/pip_repository_entry_points,tests/pip_deps test --test_output=errors From 0175da84cea5ed2806411146db160f8210bd77c2 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Wed, 30 Nov 2022 16:19:13 -0800 Subject: [PATCH 5/5] fix: windows path separator madness Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- python/pip_install/pip_compile.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/pip_install/pip_compile.py b/python/pip_install/pip_compile.py index 89efbd911e..8d6cf1bd47 100644 --- a/python/pip_install/pip_compile.py +++ b/python/pip_install/pip_compile.py @@ -1,6 +1,7 @@ "Set defaults for the pip-compile command to run it under Bazel" import os +import re import sys from pathlib import Path from shutil import copyfile @@ -33,8 +34,10 @@ def _fix_up_requirements_in_path(absolute_prefix, output_file): The paths could differ for every invocation. Replace them with a predictable path. """ output_file = Path(output_file) - fixed_requirements_text = output_file.read_text().replace(absolute_prefix, "./") - output_file.write_text(fixed_requirements_text) + contents = output_file.read_text() + contents = contents.replace(absolute_prefix, "") + contents = re.sub(r"\\(?!(\n|\r\n))", "/", contents) + output_file.write_text(contents) if __name__ == "__main__":