From 47fb6097626b7c56e9961085c819365a14843311 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sun, 1 Feb 2026 13:18:37 +0000 Subject: [PATCH 1/3] Enforce that `base.Dockerfile` installs all pip dependencies At the start of `finish.Dockerfile, download setuptools and wheel to /pip-wheels for pip's PEP 517 build isolation. Then use PIP_NO_INDEX=1 PIP_FIND_LINKS=/pip-wheels for all pip install commands. If any runtime dependency was not properly installed by `base.Dockerfile` and is missing, pip fails with "No matching distribution found" plus a custom error message explaining the issue. Changes to `base.Dockerfile` generation: - Install `requirements-pip.txt` first (pins pip/setuptools versions) - Remove redundant `pip install --upgrade pip` (version now pinned) - Install `requirements-dev.txt` for DEVELOP builds Changes to finish.Dockerfile generation: - Download setuptools/wheel to /pip-wheels for build isolation - Remove redundant `pip install` of `requirements-dev.txt` (now in base) - Use `PIP_NO_INDEX=1 PIP_FIND_LINKS=/pip-wheels` for all pip installs - Add custom error message when pip install fails - Add `pip check` after installation to verify dependency consistency - Add `inspect` target to generated Makefile with `pip check` - Make `test` target depend on `inspect` Supersedes #218 --- build_image.py | 78 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 8 deletions(-) diff --git a/build_image.py b/build_image.py index 6bc501eda..c4e67724f 100644 --- a/build_image.py +++ b/build_image.py @@ -217,6 +217,16 @@ def create_dockerfile_base(config: OfrakImageConfig) -> str: "", ] + # Read pinned pip/setuptools versions + pip_reqs_path = "requirements-pip.txt" + pip_reqs = [] + if os.path.exists(pip_reqs_path): + with open(pip_reqs_path) as pip_reqs_handle: + pip_reqs = [ + str(requirement) + for requirement in pkg_resources.parse_requirements(pip_reqs_handle) + ] + requirement_suffixes = ["", "-non-pypi"] if config.install_target is InstallTarget.DEVELOP: requirement_suffixes += ["-docs", "-test"] @@ -242,13 +252,44 @@ def create_dockerfile_base(config: OfrakImageConfig) -> str: for requirement in pkg_resources.parse_requirements(requirements_handle) ] if python_reqs: + if pip_reqs: + # Install pinned pip/setuptools versions first for consistent builds + dockerfile_base_parts += [ + f"### Python build tools from {pip_reqs_path}", + "RUN python3 -m pip install '" + "' '".join(pip_reqs) + "'", + "", + ] + pip_reqs = [] dockerfile_base_parts += [ f"### Python dependencies from the {package_path} requirements file[s]", - "RUN python3 -m pip install --upgrade pip &&\\", - " python3 -m pip install '" + "' '".join(python_reqs) + "'", + "RUN python3 -m pip install '" + "' '".join(python_reqs) + "'", "", ] + # For develop builds, also install root-level requirements-dev.txt + if config.install_target is InstallTarget.DEVELOP: + dev_reqs_path = "requirements-dev.txt" + if os.path.exists(dev_reqs_path): + with open(dev_reqs_path) as dev_reqs_handle: + dev_reqs = [ + str(requirement) + for requirement in pkg_resources.parse_requirements(dev_reqs_handle) + ] + if dev_reqs: + if pip_reqs: + # Install pinned pip/setuptools versions first for consistent builds + dockerfile_base_parts += [ + f"### Python build tools from {pip_reqs_path}", + "RUN python3 -m pip install '" + "' '".join(pip_reqs) + "'", + "", + ] + pip_reqs = [] + dockerfile_base_parts += [ + f"### Python dependencies from {dev_reqs_path}", + "RUN python3 -m pip install '" + "' '".join(dev_reqs) + "'", + "", + ] + return "\n".join(dockerfile_base_parts) @@ -256,6 +297,10 @@ def create_dockerfile_finish(config: OfrakImageConfig) -> str: full_base_image_name = "/".join((config.registry, config.base_image_name)) dockerfile_finish_parts = [ f"FROM {full_base_image_name}:{config.image_revision}\n\n", + # Download build tools for pip build isolation (PEP 517 default: setuptools + wheel). + # Used with --find-links=/pip-wheels --no-index to enforce that all runtime deps + # were pre-installed in base.Dockerfile, while still allowing build isolation. + "RUN python3 -m pip download -d /pip-wheels setuptools wheel\n\n", f"ARG OFRAK_SRC_DIR=/\n", ] @@ -277,12 +322,18 @@ def create_dockerfile_finish(config: OfrakImageConfig) -> str: dockerfile_finish_parts.append("\nWORKDIR /\n") dockerfile_finish_parts.append("ARG INSTALL_TARGET\n") if config.install_target is InstallTarget.DEVELOP: - dockerfile_finish_parts.append(f"ADD {ofrak_dir_prefix}requirements-dev.txt /\n") - dockerfile_finish_parts.append("RUN python3 -m pip install -r requirements-dev.txt\n") dockerfile_finish_parts.append( f"ADD '{ofrak_dir_prefix}pytest_ofrak' $OFRAK_SRC_DIR/pytest_ofrak\n" ) - dockerfile_finish_parts.append(f"RUN make -C $OFRAK_SRC_DIR/pytest_ofrak develop\n") + # PIP_NO_INDEX + PIP_FIND_LINKS ensures all runtime deps were pre-installed + # in base.Dockerfile, while allowing build isolation to find setuptools/wheel. + dockerfile_finish_parts.append( + "RUN PIP_NO_INDEX=1 PIP_FIND_LINKS=/pip-wheels " + f"make -C $OFRAK_SRC_DIR/pytest_ofrak develop || " + '(echo "ERROR: pip install of an OFRAK package failed when prohibited from downloading from PyPI. ' + "A dependency may be missing from base.Dockerfile. " + 'Add it to the appropriate requirements.txt file." && exit 1)\n' + ) develop_makefile = "\\n\\\n".join( [ "$INSTALL_TARGET:", @@ -293,12 +344,23 @@ def create_dockerfile_finish(config: OfrakImageConfig) -> str: ] ) dockerfile_finish_parts.append(f'RUN printf "{develop_makefile}" >> Makefile\n') - dockerfile_finish_parts.append("RUN make $INSTALL_TARGET\n\n") + # PIP_NO_INDEX + PIP_FIND_LINKS ensures all runtime deps were pre-installed + # in base.Dockerfile, while allowing build isolation to find setuptools/wheel. + dockerfile_finish_parts.append( + "RUN PIP_NO_INDEX=1 PIP_FIND_LINKS=/pip-wheels make $INSTALL_TARGET || " + '(echo "ERROR: pip install of an OFRAK package failed when prohibited from downloading from PyPI. ' + "A dependency may be missing from base.Dockerfile. " + 'Add it to the appropriate requirements.txt file." && exit 1)\n' + ) + # Verify all dependencies are consistent + dockerfile_finish_parts.append("RUN python3 -m pip check\n\n") test_names = " ".join([f"test_{package_name}" for package_name in package_names]) finish_makefile = "\\n\\\n".join( [ - ".PHONY: test " + test_names, - "test: " + test_names, + ".PHONY: test inspect " + test_names, + "inspect:", + "\tpython3 -m pip check", + "test: inspect " + test_names, ] + [ f"test_{package_name}:\\n\\\n\t\\$(MAKE) -C {package_name} test" From 67233fdd4391c3358d0b16a4aa245bf476620c70 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Fri, 6 Feb 2026 21:10:54 -0800 Subject: [PATCH 2/3] Clarify the error message --- build_image.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build_image.py b/build_image.py index c4e67724f..e02a87c08 100644 --- a/build_image.py +++ b/build_image.py @@ -331,8 +331,8 @@ def create_dockerfile_finish(config: OfrakImageConfig) -> str: "RUN PIP_NO_INDEX=1 PIP_FIND_LINKS=/pip-wheels " f"make -C $OFRAK_SRC_DIR/pytest_ofrak develop || " '(echo "ERROR: pip install of an OFRAK package failed when prohibited from downloading from PyPI. ' - "A dependency may be missing from base.Dockerfile. " - 'Add it to the appropriate requirements.txt file." && exit 1)\n' + "A dependency may be missing from base.Dockerfile or several incompatible requirements are present. " + 'Add it to the appropriate requirements.txt file and make sure all requirements agree." && exit 1)\n' ) develop_makefile = "\\n\\\n".join( [ @@ -349,8 +349,8 @@ def create_dockerfile_finish(config: OfrakImageConfig) -> str: dockerfile_finish_parts.append( "RUN PIP_NO_INDEX=1 PIP_FIND_LINKS=/pip-wheels make $INSTALL_TARGET || " '(echo "ERROR: pip install of an OFRAK package failed when prohibited from downloading from PyPI. ' - "A dependency may be missing from base.Dockerfile. " - 'Add it to the appropriate requirements.txt file." && exit 1)\n' + "A dependency may be missing from base.Dockerfile or several incompatible requirements are present. " + 'Add it to the appropriate requirements.txt file and make sure all requirements agree." && exit 1)\n' ) # Verify all dependencies are consistent dockerfile_finish_parts.append("RUN python3 -m pip check\n\n") From 35caab9fd274f3cabe33cc8c57726d42580d154e Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Mon, 9 Feb 2026 02:58:55 -0500 Subject: [PATCH 3/3] This branch requires a follow-up for PR708 --- build_image.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/build_image.py b/build_image.py index 1519d8978..799d6a892 100644 --- a/build_image.py +++ b/build_image.py @@ -194,6 +194,16 @@ def check_package_contents(package_path: str): return +def read_requirements(requirements_path: str) -> List[str]: + python_reqs: List[str] = [] + with open(requirements_path) as requirements_handle: + for line in requirements_handle: + line = line.split("#")[0].strip() + if line: + python_reqs.append(line) + return python_reqs + + def create_dockerfile_base(config: OfrakImageConfig) -> str: dockerfile_base_parts = [ "# syntax = docker/dockerfile:1.3", @@ -220,11 +230,7 @@ def create_dockerfile_base(config: OfrakImageConfig) -> str: pip_reqs_path = "requirements-pip.txt" pip_reqs = [] if os.path.exists(pip_reqs_path): - with open(pip_reqs_path) as pip_reqs_handle: - pip_reqs = [ - str(requirement) - for requirement in pkg_resources.parse_requirements(pip_reqs_handle) - ] + pip_reqs = read_requirements(pip_reqs_path) requirement_suffixes = ["", "-non-pypi"] if config.install_target is InstallTarget.DEVELOP: @@ -245,11 +251,7 @@ def create_dockerfile_base(config: OfrakImageConfig) -> str: requirements_path = os.path.join(package_path, f"requirements{suff}.txt") if not os.path.exists(requirements_path): continue - with open(requirements_path) as requirements_handle: - for line in requirements_handle: - line = line.split("#")[0].strip() - if line: - python_reqs.append(line) + python_reqs += read_requirements(requirements_path) if python_reqs: if pip_reqs: # Install pinned pip/setuptools versions first for consistent builds @@ -269,11 +271,7 @@ def create_dockerfile_base(config: OfrakImageConfig) -> str: if config.install_target is InstallTarget.DEVELOP: dev_reqs_path = "requirements-dev.txt" if os.path.exists(dev_reqs_path): - with open(dev_reqs_path) as dev_reqs_handle: - dev_reqs = [ - str(requirement) - for requirement in pkg_resources.parse_requirements(dev_reqs_handle) - ] + dev_reqs = read_requirements(dev_reqs_path) if dev_reqs: if pip_reqs: # Install pinned pip/setuptools versions first for consistent builds