From 9a3dba1e91180eaf0a397b3432b4500ed5b60c3e Mon Sep 17 00:00:00 2001 From: Brian Rosenberg Date: Mon, 16 Dec 2024 08:55:18 -0500 Subject: [PATCH 1/6] Add docker-bake.hcl --- README.md | 22 ++++++ components/cpp_component_build/Dockerfile | 2 +- components/cpp_executor/Dockerfile | 2 +- components/java_component_build/Dockerfile | 2 +- components/java_executor/Dockerfile | 2 +- components/python/Dockerfile | 2 +- docker-bake.hcl | 88 ++++++++++++++++++++++ 7 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 docker-bake.hcl diff --git a/README.md b/README.md index ca93f67b..b47abac5 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,25 @@ Clone the [openmpf-projects repository](https://github.com/openmpf/openmpf-proje If you plan to develop and integrate your own component into OpenMPF, then please refer to the [Contribution Guide](CONTRIBUTING.md). + +#### (Optional) Create the Base Images Using Bake + +[Bake](https://docs.docker.com/build/bake/) is an experimental feature available in recent Docker +releases. Since this is an experimental feature, you may run in to issues trying to use it. In +that case, skip this step and follow the instructions in the +["Create the OpenMPF Build Image"](#docker-build-command) and +["Build the OpenMPF Component Executor Docker Images"](#component-executors-build-commands) +sections. + +Run the command below from within the `openmpf-projects/openmpf-docker` directory to build all of +the OpenMPF base images. + +- `docker buildx bake --allow=fs.read=.. -f docker-bake.hcl` + +If you were able to run the command successfully, skip to the +["Generate docker-compose.yml"](#generate-compose-file) section. + + #### Create the OpenMPF Build Image @@ -149,6 +168,9 @@ Run the following command to create the OpenMPF Python component executor single - `DOCKER_BUILDKIT=1 docker build . -f python/Dockerfile --target ssb -t openmpf_python_executor_ssb` + + + #### Generate docker-compose.yml From within the `openmpf-docker` directory, copy `env.tpl` to `.env` and set environment variables in `.env`. Make sure diff --git a/components/cpp_component_build/Dockerfile b/components/cpp_component_build/Dockerfile index 1342fc53..76995f79 100644 --- a/components/cpp_component_build/Dockerfile +++ b/components/cpp_component_build/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.2 +# syntax=docker/dockerfile:1.4 ############################################################################# # NOTICE # diff --git a/components/cpp_executor/Dockerfile b/components/cpp_executor/Dockerfile index 4ff80fc4..e1fc94a3 100644 --- a/components/cpp_executor/Dockerfile +++ b/components/cpp_executor/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.2 +# syntax=docker/dockerfile:1.4 ############################################################################# # NOTICE # diff --git a/components/java_component_build/Dockerfile b/components/java_component_build/Dockerfile index a6ad53ac..2f5308c4 100644 --- a/components/java_component_build/Dockerfile +++ b/components/java_component_build/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.2 +# syntax=docker/dockerfile:1.4 ############################################################################# # NOTICE # diff --git a/components/java_executor/Dockerfile b/components/java_executor/Dockerfile index 09abd01a..d6a743c4 100644 --- a/components/java_executor/Dockerfile +++ b/components/java_executor/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.2 +# syntax=docker/dockerfile:1.4 ############################################################################# # NOTICE # diff --git a/components/python/Dockerfile b/components/python/Dockerfile index 85041796..070b3731 100644 --- a/components/python/Dockerfile +++ b/components/python/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.2 +# syntax=docker/dockerfile:1.4 ############################################################################# # NOTICE # diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 00000000..8ae639b3 --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,88 @@ +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2024 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2024 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# + +variable "TAG" { + default = "latest" +} + +variable "REGISTRY" { + default = "" +} + +group "default" { + targets = [ + "openmpf_build", + "openmpf_cpp_component_build", "openmpf_cpp_executor", + "openmpf_java_component_build", "openmpf_java_executor", + "openmpf_python_component_build", "openmpf_python_executor", "openmpf_python_executor_ssb" + ] +} + + +target "openmpf_build" { + context = ".." + dockerfile = "openmpf-docker/openmpf_build/Dockerfile" + tags = image_name("openmpf_build") +} + + +target "openmpf_cpp_and_java_components" { + name = "openmpf_${lang}_${type}" + tags = image_name("openmpf_${lang}_${type}") + dockerfile = "${lang}_${type}/Dockerfile" + context = "components" + contexts = { + openmpf_build = "target:openmpf_build" + } + matrix = { + type = ["component_build", "executor"] + lang = ["cpp", "java"] + } +} + + +target "openmpf_python_components" { + name = "openmpf_python_${type.name}" + tags = image_name("openmpf_python_${type.name}") + dockerfile = "python/Dockerfile" + target = type.target + context = "components" + contexts = { + openmpf_build = "target:openmpf_build" + } + matrix = { + type = [ + { name = "component_build", target = "build" }, + { name = "executor" , target = "executor" }, + { name = "executor_ssb", target = "ssb" } + ] + } +} + +function "image_name" { + params = [base_name] + result = ["${REGISTRY}${base_name}:${TAG}"] +} From c6e9daaea3fefec824f6c9041b6e0f8df57b9adf Mon Sep 17 00:00:00 2001 From: Brian Rosenberg Date: Thu, 20 Feb 2025 10:23:49 -0500 Subject: [PATCH 2/6] Add subject base images to docker-bake.hcl --- docker-bake.hcl | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 8ae639b3..b0cbd575 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -37,7 +37,8 @@ group "default" { "openmpf_build", "openmpf_cpp_component_build", "openmpf_cpp_executor", "openmpf_java_component_build", "openmpf_java_executor", - "openmpf_python_component_build", "openmpf_python_executor", "openmpf_python_executor_ssb" + "openmpf_python_component_build", "openmpf_python_executor", "openmpf_python_executor_ssb", + "openmpf_python_subject_build", "openmpf_python_subject_executor" ] } @@ -82,6 +83,20 @@ target "openmpf_python_components" { } } +target "openmpf_subject_components" { + name = "openmpf_python_subject_${type}" + tags = image_name("openmpf_python_subject_${type}") + target = type + context = "subject-components/python" + contexts = { + openmpf_build = "target:openmpf_build" + } + matrix = { + type = ["build", "executor"] + } +} + + function "image_name" { params = [base_name] result = ["${REGISTRY}${base_name}:${TAG}"] From 7feccf1adb6f4908dd094d6d5956b8189ce22679 Mon Sep 17 00:00:00 2001 From: Brian Rosenberg Date: Thu, 6 Mar 2025 08:50:39 -0500 Subject: [PATCH 3/6] Change for Python3.12 --- components/cpp_executor/Dockerfile | 15 +++++++++------ components/java_executor/Dockerfile | 10 ++++++++-- components/python/Dockerfile | 15 ++++++++++++--- openmpf_build/Dockerfile | 15 +++++++++++---- subject-components/python/Dockerfile | 18 +++++++++++++----- workflow_manager/Dockerfile | 13 +++++++++++-- 6 files changed, 64 insertions(+), 22 deletions(-) diff --git a/components/cpp_executor/Dockerfile b/components/cpp_executor/Dockerfile index e1fc94a3..b94928ee 100644 --- a/components/cpp_executor/Dockerfile +++ b/components/cpp_executor/Dockerfile @@ -43,16 +43,19 @@ RUN --mount=type=tmpfs,target=/var/cache/apt \ --mount=type=tmpfs,target=/tmp \ apt-get update; \ apt-get upgrade -y; \ - apt-get install --no-install-recommends -y \ - python3.8 libpython3.8 wget ca-certificates gnupg2; \ + apt-get install --no-install-recommends -y wget ca-certificates gnupg2; \ # Can't set up cuda repo at the beginning because it requires wget ca-certificates gnupg2 wget -O- https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub \ | apt-key add -; \ echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64 /" \ > /etc/apt/sources.list.d/cuda.list; \ + wget -O- 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xba6932366a755776' \ + | apt-key add -; \ + echo 'deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main' \ + > /etc/apt/sources.list.d/deadsnakes.list; \ apt-get update; \ - apt-get install -y --no-install-recommends cuda-cudart-11-4; \ - ln --symbolic /usr/bin/python3.8 /usr/bin/python3 + apt-get install -y --no-install-recommends cuda-cudart-11-4 python3.12 libpython3.12; \ + ln --symbolic /usr/bin/python3.12 /usr/local/bin/python3 COPY --from=openmpf_build /usr/local/bin/ffmpeg /usr/local/bin/ffprobe /usr/local/bin/ @@ -85,8 +88,8 @@ COPY --from=openmpf_build /build-artifacts/install/bin/amq_detection_component \ # component executor log config COPY --from=openmpf_build /build-artifacts/install/config/Log4cxxConfig.xml $MPF_HOME/config/ -COPY --from=openmpf_build /build-artifacts/install/lib/mpf_cpp_sdk.cpython-38-x86_64-linux-gnu.so \ - /usr/lib/python3.8/lib-dynload +COPY --from=openmpf_build /build-artifacts/install/lib/mpf_cpp_sdk.cpython-312-x86_64-linux-gnu.so \ + /usr/lib/python3.12/lib-dynload COPY --from=openmpf_build /scripts/* /scripts/ diff --git a/components/java_executor/Dockerfile b/components/java_executor/Dockerfile index d6a743c4..0b299366 100644 --- a/components/java_executor/Dockerfile +++ b/components/java_executor/Dockerfile @@ -43,8 +43,14 @@ RUN --mount=type=tmpfs,target=/var/cache/apt \ --mount=type=tmpfs,target=/tmp \ apt-get update; \ apt-get upgrade -y; \ - apt-get install --no-install-recommends -y openjdk-17-jre-headless python3.8; \ - ln --symbolic /usr/bin/python3.8 /usr/bin/python3; + apt-get install --no-install-recommends -y openjdk-17-jre-headless wget ca-certificates gnupg2; \ + wget -O- 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xba6932366a755776' \ + | apt-key add -; \ + echo 'deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main' \ + > /etc/apt/sources.list.d/deadsnakes.list; \ + apt-get update; \ + apt-get install -y --no-install-recommends python3.12; \ + ln --symbolic /usr/bin/python3.12 /usr/local/bin/python3; COPY --from=openmpf_build /usr/local/bin/ffmpeg /usr/local/bin/ffprobe /usr/local/bin/ diff --git a/components/python/Dockerfile b/components/python/Dockerfile index 070b3731..823076c4 100644 --- a/components/python/Dockerfile +++ b/components/python/Dockerfile @@ -38,15 +38,24 @@ SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"] ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 + RUN --mount=type=tmpfs,target=/var/cache/apt \ --mount=type=tmpfs,target=/var/lib/apt/lists \ --mount=type=tmpfs,target=/tmp \ apt-get update; \ apt-get upgrade -y; \ + apt-get install --no-install-recommends -y wget ca-certificates gnupg2; \ + wget -O- 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xba6932366a755776' \ + | apt-key add -; \ + echo 'deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main' \ + > /etc/apt/sources.list.d/deadsnakes.list; \ + apt-get update; \ apt-get install --no-install-recommends -y \ - python3.8 python3.8-venv libpython3.8 \ + python3.12 python3.12-venv libpython3.12 \ # Required for opencv-python - libglib2.0-0 libgl1-mesa-glx; + libglib2.0-0 libgl1-mesa-glx; \ + python3.12 -m ensurepip --upgrade --default-pip; \ + ln --symbolic /usr/bin/python3.12 /usr/local/bin/python3; COPY --from=openmpf_build /usr/local/bin/ffmpeg /usr/local/bin/ffprobe /usr/local/bin/ @@ -67,7 +76,7 @@ RUN --mount=type=tmpfs,target=/var/cache/apt \ apt-get update; \ # Some pip packages will try to compile C/C++ extensions during install. To compile the C/C++ # extensions, a C++ compiler and the Python development headers must be installed. - apt-get install --no-install-recommends -y g++ python3.8-dev + apt-get install --no-install-recommends -y g++ python3.12-dev RUN python3 -m venv "$COMPONENT_VIRTUALENV"; \ pip config --site set global.no-cache-dir true; \ diff --git a/openmpf_build/Dockerfile b/openmpf_build/Dockerfile index 963c7f98..70bb1911 100644 --- a/openmpf_build/Dockerfile +++ b/openmpf_build/Dockerfile @@ -48,7 +48,7 @@ RUN --mount=type=tmpfs,target=/var/cache/apt \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ wget ca-certificates gnupg2 unzip xz-utils cmake make g++ libgtest-dev mediainfo libssl-dev \ liblog4cxx-dev libboost-dev file openjdk-17-jdk-headless \ - python3.8-dev python3-pip libgl1-mesa-glx \ + libgl1-mesa-glx \ # libheif build deps libde265-dev \ # OpenCV build deps \ @@ -56,18 +56,25 @@ RUN --mount=type=tmpfs,target=/var/cache/apt \ libavutil-dev libswscale-dev libavresample-dev libharfbuzz-dev libfreetype-dev; \ # Can't set up cuda repo at the beginning because it requires wget ca-certificates gnupg2 wget -O- https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub \ - | apt-key add -; \ + | apt-key add -; \ echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64 /" \ > /etc/apt/sources.list.d/cuda.list; \ + wget -O- 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xba6932366a755776' \ + | apt-key add -; \ + echo 'deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main' \ + > /etc/apt/sources.list.d/deadsnakes.list; \ apt-get update; \ apt-get install -y --no-install-recommends cuda-minimal-build-11-4 libcufft-dev-11-4 \ libnpp-dev-11-4 libcudnn8-dev=8.2.4.15-1+cuda11.4 libcudnn8=8.2.4.15-1+cuda11.4 \ - libcublas-dev-11-4; \ + libcublas-dev-11-4 \ + python3.12-dev python3.12-venv; \ # OpenCV doesn't use the statically compiled CUDA libraries except for libcudart and they are relatively large. find /usr/local/cuda/lib64/ -name '*.a' -not -name 'libcudart_static.a' -delete; \ rm /usr/lib/x86_64-linux-gnu/libcudnn*.a; \ ln --symbolic /usr/include/x86_64-linux-gnu/openblas-pthread/cblas.h /usr/include/cblas.h; \ - ln --symbolic /usr/bin/cmake /usr/bin/cmake3; + ln --symbolic /usr/bin/cmake /usr/bin/cmake3; \ + python3.12 -m ensurepip --upgrade --default-pip; \ + ln --symbolic /usr/bin/python3.12 /usr/local/bin/python3; FROM apt_install AS ocv_install diff --git a/subject-components/python/Dockerfile b/subject-components/python/Dockerfile index a3484f48..c77ee131 100644 --- a/subject-components/python/Dockerfile +++ b/subject-components/python/Dockerfile @@ -44,8 +44,15 @@ RUN --mount=type=tmpfs,target=/var/cache/apt \ < /etc/apt/sources.list.d/deadsnakes.list + apt-get update + apt-get install --no-install-recommends -y python3.12 python3.12-venv libpython3.12 + ln --symbolic /usr/bin/python3.12 /usr/local/bin/python3 eot ENV MPF_HOME /opt/mpf @@ -65,19 +72,20 @@ RUN --mount=type=tmpfs,target=/var/cache/apt \ --mount=type=tmpfs,target=/tmp \ < /etc/apt/sources.list.d/deadsnakes.list; \ + apt-get update; \ + apt-get install --no-install-recommends -y python3.12; \ + python3.12 -m ensurepip --upgrade --default-pip; \ + ln --symbolic /usr/bin/python3.12 /usr/local/bin/python3; + RUN wget --directory-prefix /scripts \ From 7b0a5a7de12ae2f1a858cd58cb63e35afdbe395b Mon Sep 17 00:00:00 2001 From: Brian Rosenberg Date: Thu, 6 Mar 2025 09:56:04 -0500 Subject: [PATCH 4/6] Fix wfm dockerfile --- workflow_manager/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workflow_manager/Dockerfile b/workflow_manager/Dockerfile index 3a2bd620..e17f2a38 100644 --- a/workflow_manager/Dockerfile +++ b/workflow_manager/Dockerfile @@ -51,12 +51,11 @@ RUN --mount=type=tmpfs,target=/var/cache/apt \ echo 'deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main' \ > /etc/apt/sources.list.d/deadsnakes.list; \ apt-get update; \ - apt-get install --no-install-recommends -y python3.12; \ + apt-get install --no-install-recommends -y python3.12-venv; \ python3.12 -m ensurepip --upgrade --default-pip; \ ln --symbolic /usr/bin/python3.12 /usr/local/bin/python3; - RUN wget --directory-prefix /scripts \ https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh; \ chmod +x /scripts/wait-for-it.sh; From 0ee43c80f2b169edca12ca5a77a3552de0983e1d Mon Sep 17 00:00:00 2001 From: Brian Rosenberg Date: Fri, 15 Aug 2025 09:30:07 -0400 Subject: [PATCH 5/6] Update cli runner test to use Python3.12 --- components/cli_runner/tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/cli_runner/tests/Dockerfile b/components/cli_runner/tests/Dockerfile index 48b2146f..22b9d8bf 100644 --- a/components/cli_runner/tests/Dockerfile +++ b/components/cli_runner/tests/Dockerfile @@ -24,7 +24,7 @@ # limitations under the License. # ############################################################################# -FROM python:3.8-alpine +FROM python:3.12-alpine RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-20.10.3.tgz -O- \ | tar --extract --gzip --directory /bin --strip-components 1 docker/docker From 0d608379a7ab23cd38e686e27e9004ce282837ce Mon Sep 17 00:00:00 2001 From: Brian Rosenberg Date: Thu, 28 Aug 2025 08:55:34 -0400 Subject: [PATCH 6/6] Remove pkg_resources --- components/cli_runner/mpf_python_runner.py | 23 +++++++++++----------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/components/cli_runner/mpf_python_runner.py b/components/cli_runner/mpf_python_runner.py index ba6f9d13..0771e8c5 100644 --- a/components/cli_runner/mpf_python_runner.py +++ b/components/cli_runner/mpf_python_runner.py @@ -24,13 +24,12 @@ # limitations under the License. # ############################################################################# -import pkg_resources -from typing import Any, Mapping, Union, Iterable +import importlib.metadata +from typing import Any, Iterable, Mapping, Union import mpf_cli_runner_util as util import mpf_component_api -from mpf_component_api import ImageLocation, VideoTrack, AudioTrack, GenericTrack - +from mpf_component_api import AudioTrack, GenericTrack, ImageLocation, VideoTrack class PythonComponentHandle(util.ComponentHandle): @@ -42,26 +41,26 @@ def __init__(self, descriptor: Mapping[str, Any]): self.track_type = descriptor['algorithm']['trackType'] - @classmethod - def _load_component(cls, descriptor) -> type: + @staticmethod + def _load_component(descriptor) -> type: distribution_name = descriptor['batchLibrary'] - entry_points = pkg_resources.get_entry_map(distribution_name, 'mpf.exported_component') - + dist = importlib.metadata.distribution(distribution_name) + entry_points = dist.entry_points.select(group='mpf.exported_component') if not entry_points: raise RuntimeError(f'The "{distribution_name}" component did not declare a ' '"mpf.exported_component" entrypoint') # The left hand side of the '=' in entry point declaration should be "component". # For example: 'mpf.exported_component': 'component = my_module:MyComponentClass' - component_entry_point = entry_points.get('component') - if component_entry_point is None: + if component_entry_points := entry_points.select(name='component'): + return next(iter(component_entry_points)).load() + else: # An entry point in the "mpf.exported_component" group was found, # but the left hand side of the '=' was something else. # For example: 'mpf.exported_component': 'MyComponentClass = my_module:MyComponentClass' # We really only care about the entry point group, since we don't do anything with # entry point name. - component_entry_point = next(iter(entry_points.values())) - return component_entry_point.load() + return next(iter(entry_points)).load() def supports(self, media_type: util.MediaType) -> bool: