From d2a72f8724c320fe01658db160de0e1674ca7848 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Thu, 5 Sep 2024 21:15:16 +0200 Subject: [PATCH 1/2] [build] pin setuptools to avoid installation failures This PR applies the recommendation from the generated requirements file to use the `--allow-unsafe`: ``` ``` The PR also removes the pinned minor version of the base image. Closes #768 --- Makefile | 7 ++++++- monitoring/Dockerfile | 2 +- requirements.in | 1 + requirements.txt | 16 +++++++++++----- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 27a82a496c..ccc203f7de 100644 --- a/Makefile +++ b/Makefile @@ -120,8 +120,13 @@ restart-all: stop-uss-mocks down-locally start-locally start-uss-mocks restart-uss-mocks: stop-uss-mocks start-uss-mocks # To be run locally whenever a direct dependency has been updated in requirements.in +# --allow-unsafe allows us to pin setuptools .PHONY: update-pinned-dependencies update-pinned-dependencies: - ./scripts/pip_tools/pip_compile.sh --generate-hashes --output-file=requirements.txt requirements.in + ./scripts/pip_tools/pip_compile.sh \ + --generate-hashes \ + --allow-unsafe \ + --output-file=requirements.txt \ + requirements.in diff --git a/monitoring/Dockerfile b/monitoring/Dockerfile index 641350a1c0..7eac3b9303 100644 --- a/monitoring/Dockerfile +++ b/monitoring/Dockerfile @@ -9,7 +9,7 @@ # # This image is intended to be built from the repository root context/folder. -FROM python:3.12.4-slim +FROM python:3.12-slim # Not -alpine because: https://stackoverflow.com/a/58028091/651139 # Install system tools diff --git a/requirements.in b/requirements.in index 4897aa14b0..86aaa038ce 100644 --- a/requirements.in +++ b/requirements.in @@ -43,6 +43,7 @@ pyyaml==6.0.1 requests==2.31.0 s2sphere==0.2.5 scipy==1.13.0 +setuptools==72.1.0 shapely==1.7.1 structlog==21.5.0 # deployment_manager termcolor==1.1.0 diff --git a/requirements.txt b/requirements.txt index e2d85ad997..bceaf11644 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --generate-hashes --output-file=requirements.txt requirements.in +# pip-compile --allow-unsafe --generate-hashes --output-file=requirements.txt requirements.in # aiohttp==3.9.2 \ --hash=sha256:00a9abcea793c81e7f8778ca195a1714a64f6d7436c4c0bb168ad2a212627000 \ @@ -1778,7 +1778,13 @@ zope-interface==6.3 \ --hash=sha256:f95bebd0afe86b2adc074df29edb6848fc4d474ff24075e2c263d698774e108d # via gevent -# WARNING: The following packages were not pinned, but pip requires them to be -# pinned when the requirements file includes hashes and the requirement is not -# satisfied by a package already installed. Consider using the --allow-unsafe flag. -# setuptools +# The following packages are considered to be unsafe in a requirements file: +setuptools==72.1.0 \ + --hash=sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1 \ + --hash=sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec + # via + # -r requirements.in + # gunicorn + # kubernetes + # zope-event + # zope-interface From 5806cb6b47737f62b4370337a84bac812db3fd05 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Tue, 10 Sep 2024 10:36:27 +0200 Subject: [PATCH 2/2] Add comments/doc about the --allow-unsafe option --- Makefile | 3 ++- introduction_to_repository.md | 2 ++ monitoring/Dockerfile | 2 +- requirements.in | 5 +++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ccc203f7de..2f550b2cae 100644 --- a/Makefile +++ b/Makefile @@ -120,7 +120,8 @@ restart-all: stop-uss-mocks down-locally start-locally start-uss-mocks restart-uss-mocks: stop-uss-mocks start-uss-mocks # To be run locally whenever a direct dependency has been updated in requirements.in -# --allow-unsafe allows us to pin setuptools +# --allow-unsafe allows us to pin setuptools, without it we run into issues +# such as https://github.com/interuss/monitoring/issues/768 .PHONY: update-pinned-dependencies update-pinned-dependencies: ./scripts/pip_tools/pip_compile.sh \ diff --git a/introduction_to_repository.md b/introduction_to_repository.md index 36a1dcf25f..92d40b03f4 100644 --- a/introduction_to_repository.md +++ b/introduction_to_repository.md @@ -54,3 +54,5 @@ When `requirements.in` is updated, the pinned dependencies must be updated by ru ```bash make update-pinned-dependencies ``` + +Please note that we invoke `pip-tools` with the `--allow-unsafe` option to avoid issues such as https://github.com/interuss/monitoring/issues/768 diff --git a/monitoring/Dockerfile b/monitoring/Dockerfile index 7eac3b9303..51be0d5e50 100644 --- a/monitoring/Dockerfile +++ b/monitoring/Dockerfile @@ -9,7 +9,7 @@ # # This image is intended to be built from the repository root context/folder. -FROM python:3.12-slim +FROM python:3.12.6-slim # Not -alpine because: https://stackoverflow.com/a/58028091/651139 # Install system tools diff --git a/requirements.in b/requirements.in index 86aaa038ce..61f14c77de 100644 --- a/requirements.in +++ b/requirements.in @@ -7,6 +7,11 @@ # When adding a dependency, the version should usually be specified exactly to # avoid breakages when new versions are released. +# Note that when deriving a requirements.txt file from the present dependencies, +# you many need to add the --allow-unsafe option to the call to pip-compile to allow it +# to pin setuptools. +# Without it we run into issues such as https://github.com/interuss/monitoring/issues/768 + aiohttp==3.9.2 arrow==1.1.0 bc-jsonpath-ng==1.5.9 # uss_qualifier