diff --git a/build_image.py b/build_image.py index 23848f0f2..799d6a892 100644 --- a/build_image.py +++ b/build_image.py @@ -194,6 +194,16 @@ def check_package_contents(package_path: str): return +def read_requirements(requirements_path: str) -> List[str]: + python_reqs: List[str] = [] + with open(requirements_path) as requirements_handle: + for line in requirements_handle: + line = line.split("#")[0].strip() + if line: + python_reqs.append(line) + return python_reqs + + def create_dockerfile_base(config: OfrakImageConfig) -> str: dockerfile_base_parts = [ "# syntax = docker/dockerfile:1.3", @@ -216,6 +226,12 @@ def create_dockerfile_base(config: OfrakImageConfig) -> str: "", ] + # Read pinned pip/setuptools versions + pip_reqs_path = "requirements-pip.txt" + pip_reqs = [] + if os.path.exists(pip_reqs_path): + pip_reqs = read_requirements(pip_reqs_path) + requirement_suffixes = ["", "-non-pypi"] if config.install_target is InstallTarget.DEVELOP: requirement_suffixes += ["-docs", "-test"] @@ -235,19 +251,42 @@ def create_dockerfile_base(config: OfrakImageConfig) -> str: requirements_path = os.path.join(package_path, f"requirements{suff}.txt") if not os.path.exists(requirements_path): continue - with open(requirements_path) as requirements_handle: - for line in requirements_handle: - line = line.split("#")[0].strip() - if line: - python_reqs.append(line) + python_reqs += read_requirements(requirements_path) if python_reqs: + if pip_reqs: + # Install pinned pip/setuptools versions first for consistent builds + dockerfile_base_parts += [ + f"### Python build tools from {pip_reqs_path}", + "RUN python3 -m pip install '" + "' '".join(pip_reqs) + "'", + "", + ] + pip_reqs = [] dockerfile_base_parts += [ f"### Python dependencies from the {package_path} requirements file[s]", - "RUN python3 -m pip install --upgrade pip &&\\", - " python3 -m pip install '" + "' '".join(python_reqs) + "'", + "RUN python3 -m pip install '" + "' '".join(python_reqs) + "'", "", ] + # For develop builds, also install root-level requirements-dev.txt + if config.install_target is InstallTarget.DEVELOP: + dev_reqs_path = "requirements-dev.txt" + if os.path.exists(dev_reqs_path): + dev_reqs = read_requirements(dev_reqs_path) + if dev_reqs: + if pip_reqs: + # Install pinned pip/setuptools versions first for consistent builds + dockerfile_base_parts += [ + f"### Python build tools from {pip_reqs_path}", + "RUN python3 -m pip install '" + "' '".join(pip_reqs) + "'", + "", + ] + pip_reqs = [] + dockerfile_base_parts += [ + f"### Python dependencies from {dev_reqs_path}", + "RUN python3 -m pip install '" + "' '".join(dev_reqs) + "'", + "", + ] + return "\n".join(dockerfile_base_parts) @@ -255,6 +294,10 @@ def create_dockerfile_finish(config: OfrakImageConfig) -> str: full_base_image_name = "/".join((config.registry, config.base_image_name)) dockerfile_finish_parts = [ f"FROM {full_base_image_name}:{config.image_revision}\n\n", + # Download build tools for pip build isolation (PEP 517 default: setuptools + wheel). + # Used with --find-links=/pip-wheels --no-index to enforce that all runtime deps + # were pre-installed in base.Dockerfile, while still allowing build isolation. + "RUN python3 -m pip download -d /pip-wheels setuptools wheel\n\n", f"ARG OFRAK_SRC_DIR=/\n", ] @@ -276,12 +319,18 @@ def create_dockerfile_finish(config: OfrakImageConfig) -> str: dockerfile_finish_parts.append("\nWORKDIR /\n") dockerfile_finish_parts.append("ARG INSTALL_TARGET\n") if config.install_target is InstallTarget.DEVELOP: - dockerfile_finish_parts.append(f"ADD {ofrak_dir_prefix}requirements-dev.txt /\n") - dockerfile_finish_parts.append("RUN python3 -m pip install -r requirements-dev.txt\n") dockerfile_finish_parts.append( f"ADD '{ofrak_dir_prefix}pytest_ofrak' $OFRAK_SRC_DIR/pytest_ofrak\n" ) - dockerfile_finish_parts.append(f"RUN make -C $OFRAK_SRC_DIR/pytest_ofrak develop\n") + # PIP_NO_INDEX + PIP_FIND_LINKS ensures all runtime deps were pre-installed + # in base.Dockerfile, while allowing build isolation to find setuptools/wheel. + dockerfile_finish_parts.append( + "RUN PIP_NO_INDEX=1 PIP_FIND_LINKS=/pip-wheels " + f"make -C $OFRAK_SRC_DIR/pytest_ofrak develop || " + '(echo "ERROR: pip install of an OFRAK package failed when prohibited from downloading from PyPI. ' + "A dependency may be missing from base.Dockerfile or several incompatible requirements are present. " + 'Add it to the appropriate requirements.txt file and make sure all requirements agree." && exit 1)\n' + ) develop_makefile = "\\n\\\n".join( [ "$INSTALL_TARGET:", @@ -292,12 +341,23 @@ def create_dockerfile_finish(config: OfrakImageConfig) -> str: ] ) dockerfile_finish_parts.append(f'RUN printf "{develop_makefile}" >> Makefile\n') - dockerfile_finish_parts.append("RUN make $INSTALL_TARGET\n\n") + # PIP_NO_INDEX + PIP_FIND_LINKS ensures all runtime deps were pre-installed + # in base.Dockerfile, while allowing build isolation to find setuptools/wheel. + dockerfile_finish_parts.append( + "RUN PIP_NO_INDEX=1 PIP_FIND_LINKS=/pip-wheels make $INSTALL_TARGET || " + '(echo "ERROR: pip install of an OFRAK package failed when prohibited from downloading from PyPI. ' + "A dependency may be missing from base.Dockerfile or several incompatible requirements are present. " + 'Add it to the appropriate requirements.txt file and make sure all requirements agree." && exit 1)\n' + ) + # Verify all dependencies are consistent + dockerfile_finish_parts.append("RUN python3 -m pip check\n\n") test_names = " ".join([f"test_{package_name}" for package_name in package_names]) finish_makefile = "\\n\\\n".join( [ - ".PHONY: test " + test_names, - "test: " + test_names, + ".PHONY: test inspect " + test_names, + "inspect:", + "\tpython3 -m pip check", + "test: inspect " + test_names, ] + [ f"test_{package_name}:\\n\\\n\t\\$(MAKE) -C {package_name} test"