diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 56516e975b..641416f2cb 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -167,6 +167,22 @@ tasks: working_directory: examples/pip_repository_annotations platform: windows + integration_test_compile_pip_requirements_linux: + <<: *reusable_build_test_all + name: compile_pip_requirements integration tests on Linux + working_directory: tests/compile_pip_requirements + platform: ubuntu2004 + integration_test_compile_pip_requirements_macos: + <<: *reusable_build_test_all + name: compile_pip_requirements integration tests on macOS + working_directory: tests/compile_pip_requirements + platform: macos + integration_test_compile_pip_requirements_windows: + <<: *reusable_build_test_all + name: compile_pip_requirements integration tests on Windows + working_directory: tests/compile_pip_requirements + platform: windows + integration_test_pip_repository_entry_points_linux: <<: *reusable_build_test_all name: pip_repository_entry_points integration tests on Linux diff --git a/.bazelrc b/.bazelrc index 510191b847..f1c7b7a210 100644 --- a/.bazelrc +++ b/.bazelrc @@ -3,8 +3,8 @@ # This lets us glob() up all the files inside the examples to make them inputs to tests # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh -build --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/pip_repository_entry_points,tests/pip_deps -query --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/pip_repository_entry_points,tests/pip_deps +build --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/compile_pip_requirements,tests/pip_repository_entry_points,tests/pip_deps +query --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/compile_pip_requirements,tests/pip_repository_entry_points,tests/pip_deps test --test_output=errors diff --git a/examples/pip_parse/BUILD b/examples/pip_parse/BUILD index 1b6ba55f5e..653f75ce2b 100644 --- a/examples/pip_parse/BUILD +++ b/examples/pip_parse/BUILD @@ -55,16 +55,6 @@ alias( actual = entry_point("yamllint"), ) -# The requirements.in file can be checked in to the source tree or it can be -# generated. Pretend that we do some generating of the file. For this example, -# the "template" is already the file we want. -genrule( - name = "generate_requirements_in", - srcs = ["requirements.in.template"], - outs = ["requirements.in"], - cmd = "cp $(SRCS) $(OUTS)", -) - # This rule adds a convenient way to update the requirements file. compile_pip_requirements( name = "requirements", diff --git a/examples/pip_parse/requirements.in.template b/examples/pip_parse/requirements.in similarity index 100% rename from examples/pip_parse/requirements.in.template rename to examples/pip_parse/requirements.in diff --git a/python/pip_install/pip_compile.py b/python/pip_install/pip_compile.py index c9bcf32115..8d6cf1bd47 100644 --- a/python/pip_install/pip_compile.py +++ b/python/pip_install/pip_compile.py @@ -1,6 +1,7 @@ "Set defaults for the pip-compile command to run it under Bazel" import os +import re import sys from pathlib import Path from shutil import copyfile @@ -26,19 +27,17 @@ def _select_golden_requirements_file( return requirements_txt -def _fix_up_requirements_in_path( - resolved_requirements_in, requirements_in, output_file -): +def _fix_up_requirements_in_path(absolute_prefix, output_file): """Fix up references to the input file inside of the generated requirements file. We don't want fully resolved, absolute paths in the generated requirements file. The paths could differ for every invocation. Replace them with a predictable path. """ output_file = Path(output_file) - fixed_requirements_text = output_file.read_text().replace( - resolved_requirements_in, requirements_in - ) - output_file.write_text(fixed_requirements_text) + contents = output_file.read_text() + contents = contents.replace(absolute_prefix, "") + contents = re.sub(r"\\(?!(\n|\r\n))", "/", contents) + output_file.write_text(contents) if __name__ == "__main__": @@ -58,9 +57,11 @@ def _fix_up_requirements_in_path( requirements_windows = parse_str_none(sys.argv.pop(1)) update_target_label = sys.argv.pop(1) - # The requirements_in file could be generated. We need to get the path to it before we change - # directory into the workspace directory. - resolved_requirements_in = str(Path(requirements_in).resolve()) + # The requirements_in file could be generated, so we will need to remove the + # absolute prefixes in the locked requirements output file. + requirements_in_path = Path(requirements_in) + resolved_requirements_in = str(requirements_in_path.resolve()) + absolute_prefix = resolved_requirements_in[: -len(str(requirements_in_path))] # Before loading click, set the locale for its parser. # If it leaks through to the system setting, it may fail: @@ -116,7 +117,9 @@ def _fix_up_requirements_in_path( sys.argv.append("--generate-hashes") sys.argv.append("--output-file") sys.argv.append(requirements_txt if UPDATE else requirements_out) - sys.argv.append(resolved_requirements_in) + sys.argv.append( + requirements_in if requirements_in_path.exists() else resolved_requirements_in + ) if UPDATE: print("Updating " + requirements_txt) @@ -124,9 +127,7 @@ def _fix_up_requirements_in_path( cli() except SystemExit as e: if e.code == 0: - _fix_up_requirements_in_path( - resolved_requirements_in, requirements_in, requirements_txt - ) + _fix_up_requirements_in_path(absolute_prefix, requirements_txt) raise else: # cli will exit(0) on success @@ -145,9 +146,7 @@ def _fix_up_requirements_in_path( ) sys.exit(1) elif e.code == 0: - _fix_up_requirements_in_path( - resolved_requirements_in, requirements_in, requirements_out - ) + _fix_up_requirements_in_path(absolute_prefix, requirements_out) golden_filename = _select_golden_requirements_file( requirements_txt, requirements_linux, diff --git a/tests/compile_pip_requirements/.bazelrc b/tests/compile_pip_requirements/.bazelrc new file mode 100644 index 0000000000..f23315a7a1 --- /dev/null +++ b/tests/compile_pip_requirements/.bazelrc @@ -0,0 +1,5 @@ +test --test_output=errors + +# Windows requires these for multi-python support: +build --enable_runfiles +startup --windows_enable_symlinks diff --git a/tests/compile_pip_requirements/.gitignore b/tests/compile_pip_requirements/.gitignore new file mode 100644 index 0000000000..ac51a054d2 --- /dev/null +++ b/tests/compile_pip_requirements/.gitignore @@ -0,0 +1 @@ +bazel-* diff --git a/tests/compile_pip_requirements/BUILD.bazel b/tests/compile_pip_requirements/BUILD.bazel new file mode 100644 index 0000000000..258bb71ad5 --- /dev/null +++ b/tests/compile_pip_requirements/BUILD.bazel @@ -0,0 +1,33 @@ +load("@rules_python//python:pip.bzl", "compile_pip_requirements") + +genrule( + name = "generate_requirements_extra_in", + srcs = [], + outs = ["requirements_extra.in"], + cmd = "echo 'setuptools~=65.6.3' > $@", +) + +genrule( + name = "generate_requirements_in", + srcs = [], + outs = ["requirements.in"], + cmd = """ +cat > $@ <