diff --git a/.env.tpl b/.env.tpl index adb90764..5e9175d3 100644 --- a/.env.tpl +++ b/.env.tpl @@ -37,9 +37,6 @@ REGISTRY= TAG=latest -WFM_USER=admin -WFM_PASSWORD=mpfadm - # Set this if using "docker-compose.users.yml". USER_PROPERTIES_PATH= diff --git a/components/component-executor.py b/components/component-executor.py index 5f0f1f50..68a9abcd 100644 --- a/components/component-executor.py +++ b/components/component-executor.py @@ -39,7 +39,6 @@ from pathlib import Path from typing import Any, Dict, NamedTuple, Optional, Tuple -import component_registration Descriptor = Dict[str, Any] @@ -54,12 +53,6 @@ def main(): class EnvConfig(NamedTuple): - wfm_user: str - wfm_password: str - wfm_base_url: str - oidc_issuer_uri: Optional[str] - oidc_client_id: Optional[str] - oidc_client_secret: Optional[str] activemq_broker_uri: str component_log_name: Optional[str] disable_component_registration: bool @@ -69,14 +62,6 @@ class EnvConfig(NamedTuple): @staticmethod def create(): - oidc_issuer_uri = os.getenv('OIDC_JWT_ISSUER_URI', os.getenv('OIDC_ISSUER_URI')) - oidc_client_id = os.getenv('OIDC_CLIENT_ID') - oidc_client_secret = os.getenv('OIDC_CLIENT_SECRET') - if oidc_issuer_uri and (not oidc_client_id or not oidc_client_secret): - raise RuntimeError( - 'The OIDC_CLIENT_ID and OIDC_CLIENT_SECRET environment variables must both ' - 'be set.') - activemq_broker_uri = os.getenv('ACTIVE_MQ_BROKER_URI') if not activemq_broker_uri: activemq_host = os.getenv('ACTIVE_MQ_HOST', 'workflow-manager') @@ -90,12 +75,6 @@ def create(): else: log_path = mpf_home / 'share/logs' return EnvConfig( - os.getenv('WFM_USER', 'admin'), - os.getenv('WFM_PASSWORD', 'mpfadm'), - os.getenv('WFM_BASE_URL', 'http://workflow-manager:8080'), - oidc_issuer_uri, - oidc_client_id, - oidc_client_secret, activemq_broker_uri, os.getenv('COMPONENT_LOG_NAME'), bool(os.getenv('DISABLE_COMPONENT_REGISTRATION')), @@ -109,16 +88,9 @@ def init() -> Tuple[subprocess.Popen[str], Optional[subprocess.Popen[bytes]]]: descriptor_path = find_descriptor(env_config.mpf_home) print('Loading descriptor from', descriptor_path) - with open(descriptor_path, 'rb') as descriptor_file: - unparsed_descriptor = descriptor_file.read() - - if env_config.disable_component_registration: - print('Component registration disabled because the ' - '"DISABLE_COMPONENT_REGISTRATION" environment variable was set.') - else: - component_registration.register_component(env_config, unparsed_descriptor) + with open(descriptor_path) as descriptor_file: + descriptor = json.load(descriptor_file) - descriptor = json.loads(unparsed_descriptor) if env_config.node_name: node_name = env_config.node_name else: @@ -127,7 +99,11 @@ def init() -> Tuple[subprocess.Popen[str], Optional[subprocess.Popen[bytes]]]: log_dir = env_config.base_log_path / node_name / 'log' executor_proc = start_executor( - descriptor, env_config.mpf_home, env_config.activemq_broker_uri, node_name) + env_config.mpf_home, + descriptor, + descriptor_path, + env_config.activemq_broker_uri, + node_name) tail_proc = tail_log_if_needed(log_dir, env_config.component_log_name, executor_proc.pid) return executor_proc, tail_proc @@ -152,13 +128,17 @@ def find_descriptor(mpf_home: Path) -> Path: f'descriptors were found: {glob_matches}') -def start_executor(descriptor: Descriptor, mpf_home: Path, activemq_broker_uri: str, node_name: str - ) -> subprocess.Popen[str]: +def start_executor( + mpf_home: Path, + descriptor: Descriptor, + descriptor_path: Path, + activemq_broker_uri: str, + node_name: str) -> subprocess.Popen[str]: algorithm_name = descriptor['algorithm']['name'].upper() queue_name = f'MPF.DETECTION_{algorithm_name}_REQUEST' language = descriptor['sourceLanguage'].lower() - executor_env = get_executor_env_vars(mpf_home, descriptor, node_name) + executor_env = get_executor_env_vars(mpf_home, descriptor, descriptor_path, node_name) if language in ('c++', 'python'): amq_detection_component_path = str(mpf_home / 'bin/amq_detection_component') batch_lib = expand_env_vars(descriptor['batchLibrary'], executor_env) @@ -187,9 +167,9 @@ def start_executor(descriptor: Descriptor, mpf_home: Path, activemq_broker_uri: text=True) # Handle ctrl-c - signal.signal(signal.SIGINT, lambda sig, frame: forward_signal(sig, executor_proc)) + signal.signal(signal.SIGINT, lambda sig, frame: handle_sig_int(executor_proc)) # Handle docker stop - signal.signal(signal.SIGTERM, lambda sig, frame: forward_signal(sig, executor_proc)) + signal.signal(signal.SIGTERM, lambda sig, frame: handle_sig_term(executor_proc)) return executor_proc @@ -213,10 +193,22 @@ def find_java_executor_jar(descriptor: Descriptor, mpf_home: Path) -> Path: return expanded_executor_path -def forward_signal(sig: int, executor_proc: subprocess.Popen[str]) -> None: - signal_entry = signal.Signals(sig) - print(f'Sending {signal_entry.name}({sig}) to component executor.') - executor_proc.send_signal(sig) +def handle_sig_term(executor_proc: subprocess.Popen[str]): + print(f'Sending SIGTERM({signal.SIGTERM}) to component executor.') + executor_proc.terminate() + try: + executor_proc.wait(1) + except subprocess.TimeoutExpired: + executor_proc.kill() + + +def handle_sig_int(executor_proc: subprocess.Popen[str]): + print(f'Sending SIGINT({signal.SIGINT}) to component executor.') + executor_proc.send_signal(signal.SIGINT) + try: + executor_proc.wait(1) + except subprocess.TimeoutExpired: + handle_sig_term(executor_proc) def tail_log_if_needed(log_dir: Path, component_log_name: Optional[str], executor_pid: int @@ -246,11 +238,16 @@ def tail_log_if_needed(log_dir: Path, component_log_name: Optional[str], executo -def get_executor_env_vars(mpf_home: Path, descriptor: Descriptor, node_name: str) -> Dict[str, str]: - executor_env = os.environ.copy() - executor_env['THIS_MPF_NODE'] = node_name - executor_env['SERVICE_NAME'] = descriptor['componentName'] - executor_env['COMPONENT_NAME'] = descriptor['componentName'] +def get_executor_env_vars( + mpf_home: Path, + descriptor: Descriptor, + descriptor_path: Path, + node_name: str) -> Dict[str, str]: + executor_env = {**os.environ, + 'THIS_MPF_NODE': node_name, + 'SERVICE_NAME': descriptor['componentName'], + 'COMPONENT_NAME': descriptor['componentName'], + 'DESCRIPTOR_PATH': str(descriptor_path)} for json_env_var in descriptor.get('environmentVariables', ()): var_name = json_env_var['name'] diff --git a/components/component_registration.py b/components/component_registration.py deleted file mode 100644 index d4f92500..00000000 --- a/components/component_registration.py +++ /dev/null @@ -1,249 +0,0 @@ -############################################################################# -# NOTICE # -# # -# This software (or technical data) was produced for the U.S. Government # -# under contract, and is subject to the Rights in Data-General Clause # -# 52.227-14, Alt. IV (DEC 2007). # -# # -# Copyright 2024 The MITRE Corporation. All Rights Reserved. # -############################################################################# - -############################################################################# -# Copyright 2024 The MITRE Corporation # -# # -# Licensed under the Apache License, Version 2.0 (the "License"); # -# you may not use this file except in compliance with the License. # -# You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -# Unless required by applicable law or agreed to in writing, software # -# distributed under the License is distributed on an "AS IS" BASIS, # -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -# See the License for the specific language governing permissions and # -# limitations under the License. # -############################################################################# - -from __future__ import annotations - -import base64 -import errno -import http.client -import json -import socket -import ssl -import time -import urllib.error -import urllib.request -import urllib.response -from typing import Callable, Dict, NamedTuple, NoReturn, Tuple - - -def register_component(env_config, descriptor_bytes: bytes) -> None: - if env_config.oidc_issuer_uri: - OidcRegistration(env_config, descriptor_bytes - ).post_descriptor() - else: - BasicAuthRegistration(env_config, descriptor_bytes - ).post_descriptor() - - -def execute_http_request_with_retry(request_context: RequestContext) -> urllib.response.addinfourl: - url = request_context.url - while True: - request = request_context.request_builder(url) - try: - return _OPENER.open(request) - except http.client.BadStatusLine as err: - url = handle_bad_status(url, err) - except urllib.error.HTTPError as err: - url = handle_http_error(url, err, request_context) - except urllib.error.URLError as err: - url, should_wait = handle_url_error(url, err) - if should_wait: - time.sleep(10) - - -def handle_bad_status(url: str, error: http.client.BadStatusLine) -> str: - if url.startswith('https'): - raise error - new_url = url.replace('http://', 'https://') - print(f'HTTP request to {url} failed due to an invalid status line in the HTTP ' - 'response. This usually means that the server is using HTTPS, but an "http://" ' - 'URL was used. Trying again with:', new_url) - return new_url - - -def handle_http_error( - url: str, - error: urllib.error.HTTPError, - request_context: RequestContext) -> str: - if error.url != url: - print(f'Sending HTTP request to {url} resulted in a redirect to {error.url}.') - return error.url - - response_content = error.read() - try: - server_message = json.loads(response_content)['message'] - except (ValueError, KeyError): - server_message = response_content.decode() - - error_msg = f'The following error occurred while sending HTTP request to {url}: {error}' - if server_message: - error_msg += f' {server_message}' - if error.code == 401: - error_msg += f'\n{request_context.get_401_error_msg(error)}' - raise RuntimeError(error_msg) from error - - -_RETRYABLE_ERR_NOS = (socket.EAI_NONAME, socket.EAI_AGAIN, errno.ECONNREFUSED) - -def handle_url_error(url: str, error: urllib.error.URLError) -> Tuple[str, bool]: - reason = error.reason - is_unknown_protocol = isinstance(reason, ssl.SSLError) and reason.reason == 'UNKNOWN_PROTOCOL' - if is_unknown_protocol: - if url.startswith('http'): - raise error - new_url = url.replace('https://', 'http://') - print(f'HTTP request to {url} failed due to an "UNKNOWN_PROTOCOL" SSL ' - 'error. This usually means that the server is using HTTP on the specified ' - 'port, but an "https://" URL was used. Trying again with:', new_url) - return new_url, False - - if isinstance(reason, OSError) and reason.errno in _RETRYABLE_ERR_NOS: - print(f'HTTP request to {url} failed due to "{reason.strerror}". This is either ' - 'because the service is still starting up or the wrong URL was used. The ' - 'request will be re-attempted in 10 seconds.') - return url, True - raise error - - - -# The default urllib.request.HTTPRedirectHandler converts POST requests to GET requests. -# This subclass just throws an exception so we can post to the new URL ourselves. -class ThrowingRedirectHandler(urllib.request.HTTPRedirectHandler): - def http_error_302(self, req, fp, code, msg, headers) -> NoReturn: - new_url = headers.get('location') or headers.get('uri') - if new_url: - raise urllib.error.HTTPError(new_url, code, msg, headers, fp) - else: - raise RuntimeError('Received HTTP redirect response with no location header.') - - -def create_opener() -> urllib.request.OpenerDirector: - ssl_ctx = ssl.SSLContext() - return urllib.request.build_opener( - ThrowingRedirectHandler(), - urllib.request.HTTPSHandler(context=ssl_ctx)) - -_OPENER = create_opener() - - -class BasicAuthRegistration: - def __init__(self, env_config, descriptor_bytes: bytes): - self._wfm_user: str = env_config.wfm_user - self._wfm_password: str = env_config.wfm_password - self._wfm_base_url: str = env_config.wfm_base_url - self._descriptor_bytes: bytes = descriptor_bytes - - def post_descriptor(self) -> None: - url = self._wfm_base_url + '/rest/components/registerUnmanaged' - headers = create_basic_auth_header(self._wfm_user, self._wfm_password) - headers['Content-Type'] = 'application/json' - - print('Registering component by posting descriptor to', url) - request_context = RequestContext( - url, - lambda u: urllib.request.Request(u, self._descriptor_bytes, headers), - self.get_401_error_msg) - - with execute_http_request_with_retry(request_context): - # We don't need to do anything with the response. - pass - - @staticmethod - def get_401_error_msg(error: urllib.error.HTTPError): - if error.headers.get('WWW-Authenticate') == 'Bearer': - return ('Workflow Manager is configured to use OIDC, so the component must also be' - ' configured to use OIDC.') - else: - return 'The WFM_USER and WFM_PASSWORD environment variables need to be changed.' - - -class OidcRegistration: - def __init__(self, env_config, descriptor_bytes: bytes): - self._token_url: str = self._request_token_url(env_config.oidc_issuer_uri) - self._token: str = '' - self._reuse_token_until: float = 0.0 - self._wfm_base_url: str = env_config.wfm_base_url - self._client_id: str = env_config.oidc_client_id - self._client_secret: str = env_config.oidc_client_secret - self._descriptor_bytes: bytes = descriptor_bytes - self._request_auth_token() - - @classmethod - def _request_token_url(cls, oidc_issuer_uri: str) -> str: - config_url = oidc_issuer_uri + '/.well-known/openid-configuration' - print('Getting OIDC configuration metadata from', config_url) - request_context = RequestContext(config_url, urllib.request.Request, cls.get_401_error_msg) - with execute_http_request_with_retry(request_context) as resp: - return json.load(resp)['token_endpoint'] - - def _request_auth_token(self) -> None: - headers = create_basic_auth_header(self._client_id, self._client_secret) - - def create_request(url): - # Update token url in case there was a redirect. - self._token_url = url - return urllib.request.Request(url, b'grant_type=client_credentials', headers) - - print(f'Requesting token from {self._token_url}') - request_context = RequestContext(self._token_url, create_request, self.get_401_error_msg) - with execute_http_request_with_retry(request_context) as resp: - resp_content = json.load(resp) - - self._token = resp_content['access_token'] - expires_in = resp_content['expires_in'] - self._reuse_token_until = time.time() + expires_in - if expires_in > 60: - # Do not re-use token for full duration to account for clock drift and network latency. - self._reuse_token_until -= 60 - print(f'Received token that expires in {expires_in} seconds.') - - def post_descriptor(self) -> None: - url = self._wfm_base_url + '/rest/components/registerUnmanaged' - print('Registering component by posting descriptor to', url) - request_context = RequestContext( - url, self._create_post_descriptor_request, self.get_401_error_msg) - with execute_http_request_with_retry(request_context): - # We don't need to do anything with the response. - pass - - def _create_post_descriptor_request(self, url: str) -> urllib.request.Request: - if time.time() > self._reuse_token_until: - self._request_auth_token() - headers = { - 'Authorization': f'Bearer {self._token}', - 'Content-Type': 'application/json' - } - return urllib.request.Request(url, self._descriptor_bytes, headers) - - @staticmethod - def get_401_error_msg(error: urllib.error.HTTPError): - base_message = 'The OIDC environment variables need to be changed.' - if auth_header := error.headers.get('WWW-Authenticate'): - return f'The WWW-Authenticate header contained: {auth_header}\n{base_message}' - else: - return base_message - - -def create_basic_auth_header(user: str, password: str) -> Dict[str, str]: - auth_info_bytes = f'{user}:{password}'.encode() - base64_auth_info = base64.b64encode(auth_info_bytes).decode() - return {'Authorization': f'Basic {base64_auth_info}'} - - -class RequestContext(NamedTuple): - url: str - request_builder: Callable[[str], urllib.request.Request] - get_401_error_msg: Callable[[urllib.error.HTTPError], str] diff --git a/components/cpp_executor/Dockerfile b/components/cpp_executor/Dockerfile index e1fc94a3..9c3f8ed2 100644 --- a/components/cpp_executor/Dockerfile +++ b/components/cpp_executor/Dockerfile @@ -99,7 +99,7 @@ ENV BUILD_DIR /home/mpf/component_build ENV PLUGINS_DIR $MPF_HOME/plugins -COPY docker-entrypoint.sh component-executor.py component_registration.py /scripts/ +COPY docker-entrypoint.sh component-executor.py /scripts/ COPY cli_runner/*.py cli_runner/Log4cxxConfig.xml /scripts/cli_runner/ diff --git a/components/cpp_executor/README.md b/components/cpp_executor/README.md index cd5bb319..3c028329 100644 --- a/components/cpp_executor/README.md +++ b/components/cpp_executor/README.md @@ -62,27 +62,27 @@ prefix on the `FROM` lines. FROM openmpf/openmpf_cpp_component_build:latest as build_component # If your component has external dependencies, you would add the commands necessary to download or -# build the dependencies here. Adding the dependencies prior the copying in your source code -# allows you to take advantage of the Docker build cache to avoid re-installing the dependencies +# build the dependencies here. Adding the dependencies prior the copying in your source code +# allows you to take advantage of the Docker build cache to avoid re-installing the dependencies # every time your source code changes. # e.g. RUN yum install -y mydependency # Copy in your source code COPY . . -# Build your component. The [build-component.sh](../cpp_component_build/scripts/build-component.sh) +# Build your component. The [build-component.sh](../cpp_component_build/scripts/build-component.sh) # script is provided by the openmpf_cpp_component_build base image. RUN build-component.sh -# You optionally may want to run unit tests here, or wherever is appropriate for your Dockerfile. -# The [OcvFaceDetection component's Dockerfile](https://github.com/openmpf/openmpf-components/blob/master/cpp/OcvFaceDetection/Dockerfile) -# shows one way of setting up unit tests, but you can do it in whatever way you see fit. +# You optionally may want to run unit tests here, or wherever is appropriate for your Dockerfile. +# The [OcvFaceDetection component's Dockerfile](https://github.com/openmpf/openmpf-components/blob/master/cpp/OcvFaceDetection/Dockerfile) +# shows one way of setting up unit tests, but you can do it in whatever way you see fit. -# In the second stage of the build we extend the openmpf_cpp_executor base image. +# In the second stage of the build we extend the openmpf_cpp_executor base image. FROM openmpf/openmpf_cpp_executor:latest -# If your component has runtime dependencies other than the shared libraries required at compile -# time you should install them here. Adding the dependencies prior to copying your component's +# If your component has runtime dependencies other than the shared libraries required at compile +# time you should install them here. Adding the dependencies prior to copying your component's # build artifacts allows you to take advantage of the Docker build cache to avoid re-installing # the dependencies every time your source code changes. @@ -90,8 +90,8 @@ FROM openmpf/openmpf_cpp_executor:latest # Set LD_LIBRARY_PATH so this component works with the [CLI runner](../../CLI_RUNNER.md). ENV LD_LIBRARY_PATH $PLUGINS_DIR/MyFaceDetection/lib -# Copy only the files the component will need at runtime from the build stage. -# This line also copies over the libraries that your component links to. +# Copy only the files the component will need at runtime from the build stage. +# This line also copies over the libraries that your component links to. # Note that running build-component.sh in the first stage collected those libraries for you. COPY --from=build_component $BUILD_DIR/plugin/MyFaceDetection \ $PLUGINS_DIR/MyFaceDetection @@ -118,14 +118,11 @@ For example: `docker build -t MyFaceDetection /path/to/MyFaceDetection`. ### Run your component 1. Start OpenMPF -2. Run the following command replacing `` with the value provided in the build step. If your OpenMPF - deployment uses non-default credentials the `WFM_USER` and `WFM_PASSWORD` values will need to be modified. +2. Run the following command replacing `` with the value provided in the build step. ```bash docker run \ --network openmpf_default \ -v openmpf_shared_data:/opt/mpf/share \ - -e WFM_USER=admin \ - -e WFM_PASSWORD=mpfadm \ ``` diff --git a/components/java_executor/Dockerfile b/components/java_executor/Dockerfile index d6a743c4..a12fdf56 100644 --- a/components/java_executor/Dockerfile +++ b/components/java_executor/Dockerfile @@ -58,7 +58,7 @@ ENV PLUGINS_DIR $MPF_HOME/plugins COPY --from=openmpf_build /build-artifacts/java-executor/*.jar $MPF_HOME/jars/ -COPY component-executor.py component_registration.py /scripts/ +COPY component-executor.py /scripts/ ENTRYPOINT ["python3", "-u", "/scripts/component-executor.py"] diff --git a/components/java_executor/README.md b/components/java_executor/README.md index e82925f8..0c7d0ce0 100644 --- a/components/java_executor/README.md +++ b/components/java_executor/README.md @@ -71,9 +71,9 @@ prefix on the `FROM` lines. # In first stage of the build we extend the openmpf_java_component_build base image. FROM openmpf/openmpf_java_component_build:latest as build_component -# If your component has external dependencies, you would add the commands necessary to download -# or build the dependencies here. Adding the dependencies prior the copying in your source code -# allows you to take advantage of the Docker build cache to avoid re-installing the dependencies +# If your component has external dependencies, you would add the commands necessary to download +# or build the dependencies here. Adding the dependencies prior the copying in your source code +# allows you to take advantage of the Docker build cache to avoid re-installing the dependencies # every time your source code changes. # e.g. RUN yum install -y mydependency @@ -81,7 +81,7 @@ FROM openmpf/openmpf_java_component_build:latest as build_component # Start by just copying your pom.xml file. COPY pom.xml pom.xml -# Download Maven dependencies before copying in the rest of the source code so that +# Download Maven dependencies before copying in the rest of the source code so that # Maven doesn't need to re-download dependencies every time the source code changes. RUN mvn org.apache.maven.plugins:maven-dependency-plugin:3.1.1:go-offline; @@ -89,21 +89,21 @@ RUN mvn org.apache.maven.plugins:maven-dependency-plugin:3.1.1:go-offline; COPY . . # Build and package your component. -RUN mvn package -Dmpf.assembly.format=dir +RUN mvn package -Dmpf.assembly.format=dir -# In the second stage of the build we extend the openmpf_java_executor base image. +# In the second stage of the build we extend the openmpf_java_executor base image. FROM openmpf/openmpf_java_executor:latest -# If your component has runtime dependencies other than the Maven libraries required at -# compile time you should install them here. Adding the dependencies prior to copying your -# component's build artifacts allows you to take advantage of the Docker build cache to avoid +# If your component has runtime dependencies other than the Maven libraries required at +# compile time you should install them here. Adding the dependencies prior to copying your +# component's build artifacts allows you to take advantage of the Docker build cache to avoid # re-installing the dependencies every time your source code changes. -# Copy only the files the component will need at runtime from the build stage. -# This line also copies over the jar dependencies. +# Copy only the files the component will need at runtime from the build stage. +# This line also copies over the jar dependencies. # One of the things that the `mvn package` command does is collect the jar dependencies. COPY --from=build_component \ /home/mpf/component_src/target/plugin-packages/MyFaceDetection/MyFaceDetection \ @@ -125,14 +125,11 @@ docker build -t ### Run your component 1. Start OpenMPF -2. Run the following command replacing `` with the value provided in the build step. If your OpenMPF - deployment uses non-default credentials the `WFM_USER` and `WFM_PASSWORD` values will need to be modified. +2. Run the following command replacing `` with the value provided in the build step. ```bash docker run \ --network openmpf_default \ -v openmpf_shared_data:/opt/mpf/share \ - -e WFM_USER=admin \ - -e WFM_PASSWORD=mpfadm \ ``` diff --git a/components/python/Dockerfile b/components/python/Dockerfile index 070b3731..a029f104 100644 --- a/components/python/Dockerfile +++ b/components/python/Dockerfile @@ -156,7 +156,7 @@ ENV MPF_LOG_PATH $MPF_HOME/share/logs ENV PLUGINS_DIR $MPF_HOME/plugins -COPY docker-entrypoint.sh component-executor.py component_registration.py /scripts/ +COPY docker-entrypoint.sh component-executor.py /scripts/ COPY cli_runner/*.py /scripts/cli_runner/ diff --git a/components/python/README.md b/components/python/README.md index 01fc8f8c..f54e838d 100644 --- a/components/python/README.md +++ b/components/python/README.md @@ -73,19 +73,19 @@ base images then you should omit the `openmpf/` prefix on the `FROM` line. ```dockerfile FROM openmpf/openmpf_python_executor_ssb:latest as build_component -# If your component has external dependencies, you would add the commands necessary to download -# or install the dependencies here. Adding the dependencies prior the copying in your source code -# allows you to take advantage of the Docker build cache to avoid re-installing the dependencies +# If your component has external dependencies, you would add the commands necessary to download +# or install the dependencies here. Adding the dependencies prior the copying in your source code +# allows you to take advantage of the Docker build cache to avoid re-installing the dependencies # every time your source code changes. # e.g. RUN pip3 install --no-cache-dir 'opencv-python>=4.4.0' 'tensorflow>=2.1.0' -# `--mount=target=.,readwrite` will bind-mount the root of the build context on to the current +# `--mount=target=.,readwrite` will bind-mount the root of the build context on to the current # working directory, which is set to $SRC_DIR in the base image. Written data will be discarded. -# The [install-component.sh](./install-component.sh) script will install your component in to your -# component's virtualenv (located at $COMPONENT_VIRTUALENV). It is provided by the +# The [install-component.sh](./install-component.sh) script will install your component in to your +# component's virtualenv (located at $COMPONENT_VIRTUALENV). It is provided by the # openmpf_python_executor_ssb base image. -# You also may want run unit tests in this step. -# The [EastTextDetection component's Dockerfile](https://github.com/openmpf/openmpf-components/blob/master/python/EastTextDetection/Dockerfile) +# You also may want run unit tests in this step. +# The [EastTextDetection component's Dockerfile](https://github.com/openmpf/openmpf-components/blob/master/python/EastTextDetection/Dockerfile) # shows one way of setting up unit tests. RUN --mount=target=.,readwrite install-component.sh ``` @@ -99,9 +99,9 @@ images then you should omit the `openmpf/` prefix on the `FROM` lines. # In first stage of the build we extend the openmpf_python_component_build base image. FROM openmpf/openmpf_python_component_build:latest as build_component -# If your component has external dependencies, you would add the commands necessary to download -# or build the dependencies here. Adding the dependencies prior the copying in your source code -# allows you to take advantage of the Docker build cache to avoid re-installing the dependencies +# If your component has external dependencies, you would add the commands necessary to download +# or build the dependencies here. Adding the dependencies prior the copying in your source code +# allows you to take advantage of the Docker build cache to avoid re-installing the dependencies # every time your source code changes. # e.g. RUN apt-get update && apt-get install -y gcc && rm -rf /var/lib/apt/lists/* # e.g. RUN pip3 install --no-cache-dir 'opencv-python>=4.4.0' 'tensorflow>=2.1.0' @@ -110,27 +110,27 @@ FROM openmpf/openmpf_python_component_build:latest as build_component COPY . . # Install your component in to your component's virtualenv (located at $COMPONENT_VIRTUALENV). -# The [install-component.sh](../python_component_build/scripts/install-component.sh) +# The [install-component.sh](../python_component_build/scripts/install-component.sh) # script is provided by the openmpf_python_component_build base image. RUN install-component.sh -# You optionally may want to run unit tests here, or wherever is appropriate for your Dockerfile. -# The [EastTextDetection component's Dockerfile](https://github.com/openmpf/openmpf-components/blob/7145929319ff18c2b5957a3b7f88e4a04fcf3670/python/EastTextDetection/Dockerfile) -# shows one way of setting up unit tests, but you can do it in whatever way you see fit. +# You optionally may want to run unit tests here, or wherever is appropriate for your Dockerfile. +# The [EastTextDetection component's Dockerfile](https://github.com/openmpf/openmpf-components/blob/7145929319ff18c2b5957a3b7f88e4a04fcf3670/python/EastTextDetection/Dockerfile) +# shows one way of setting up unit tests, but you can do it in whatever way you see fit. # In the second stage of the build we extend the openmpf_python_executor base image FROM openmpf/openmpf_python_executor:latest -# If your component has runtime dependencies that are not pip packages, -# you should install them here. Adding the dependencies prior to copying your component's +# If your component has runtime dependencies that are not pip packages, +# you should install them here. Adding the dependencies prior to copying your component's # build artifacts allows you to take advantage of the Docker build cache to avoid re-installing # the dependencies every time your source code changes. # Copy your component's virtualenv from the build stage. -# The install-component.sh script from the build stage installed -# your plugin code and its pip dependencies in the +# The install-component.sh script from the build stage installed +# your plugin code and its pip dependencies in the # virtualenv located at $COMPONENT_VIRTUALENV. COPY --from=build_component $COMPONENT_VIRTUALENV $COMPONENT_VIRTUALENV @@ -156,14 +156,11 @@ For example: `docker build -t MyFaceDetection /path/to/MyFaceDetection`. ### Run your component 1. Start OpenMPF -2. Run the following command replacing `` with the value provided in the build step. If your OpenMPF - deployment uses non-default credentials the `WFM_USER` and `WFM_PASSWORD` values will need to be modified. +2. Run the following command replacing `` with the value provided in the build step. ```bash docker run \ --network openmpf_default \ -v openmpf_shared_data:/opt/mpf/share \ - -e WFM_USER=admin \ - -e WFM_PASSWORD=mpfadm \ ``` diff --git a/docker-compose.components.yml b/docker-compose.components.yml index fd0102d0..05bd665c 100644 --- a/docker-compose.components.yml +++ b/docker-compose.components.yml @@ -41,22 +41,15 @@ x-component-base: &component-base mode: replicated replicas: 2 -x-detection-component-base: &detection-component-base - <<: *component-base - environment: - <<: *common-env-vars - WFM_USER: - WFM_PASSWORD: - services: argos-translation: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_argos_translation:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/python/ArgosTranslation azure-form-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_azure_form_detection:${TAG} build: context: ${OPENMPF_PROJECTS_PATH}/openmpf-components/python/AzureFormDetection @@ -66,7 +59,7 @@ services: MPF_PROP_ACS_SUBSCRIPTION_KEY: ${ACS_FORM_DETECTION_SUBSCRIPTION_KEY} azure_read_text_detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_azure_read_text_detection:${TAG} build: context: ${OPENMPF_PROJECTS_PATH}/openmpf-components/python/AzureReadTextDetection @@ -76,7 +69,7 @@ services: MPF_PROP_ACS_SUBSCRIPTION_KEY: ${ACS_READ_SUBSCRIPTION_KEY} azure-speech-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_azure_speech_detection:${TAG} build: context: ${OPENMPF_PROJECTS_PATH}/openmpf-components/python/AzureSpeechDetection @@ -88,7 +81,7 @@ services: MPF_PROP_ACS_BLOB_SERVICE_KEY: ${ACS_SPEECH_BLOB_SERVICE_KEY} azure-translation: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_azure_translation:${TAG} build: context: ${OPENMPF_PROJECTS_PATH}/openmpf-components/python/AzureTranslation @@ -101,7 +94,7 @@ services: MPF_PROP_ACS_SUBSCRIPTION_KEY: ${ACS_TRANSLATION_SUBSCRIPTION_KEY} clip-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_clip_detection:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/python/ClipDetection @@ -131,7 +124,7 @@ services: --grpc-infer-allocation-pool-size=16 ] east-text-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_east_text_detection:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/python/EastTextDetection deploy: @@ -141,49 +134,49 @@ services: cpus: "2.0" fasttext-language-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_fasttext_language_detection:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/python/FastTextLanguageDetection keyword-tagging: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_keyword_tagging:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/cpp/KeywordTagging llama-video-summarization: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_llama_video_summarization:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/python/LlamaVideoSummarization mog-motion-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_mog_motion_detection:${TAG} build: context: ${OPENMPF_PROJECTS_PATH}/openmpf-contrib-components/cpp/motion dockerfile: MogMotionDetection/Dockerfile nlp-text-correction: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_nlp_text_correction:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/python/NlpTextCorrection oalpr-license-plate-text-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_oalpr_license_plate_text_detection:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/cpp/OalprLicensePlateTextDetection ocv-dnn-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_ocv_dnn_detection:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/cpp/OcvDnnDetection ocv-face-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_ocv_face_detection:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/cpp/OcvFaceDetection ocv-yolo-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_ocv_yolo_detection:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/cpp/OcvYoloDetection @@ -224,12 +217,12 @@ services: build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/python/OrToolsSubjectComponent scene-change-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_scene_change_detection:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/cpp/SceneChangeDetection sphinx-speech-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_sphinx_speech_detection:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/java/SphinxSpeechDetection environment: @@ -237,34 +230,34 @@ services: JAVA_TOOL_OPTIONS: -Xmx1g # limit Java heap size to 1GB subsense-motion-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_subsense_motion_detection:${TAG} build: context: ${OPENMPF_PROJECTS_PATH}/openmpf-contrib-components/cpp/motion dockerfile: SubsenseMotionDetection/Dockerfile tesseract-ocr-text-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_tesseract_ocr_text_detection:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/cpp/TesseractOCRTextDetection tika-image-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_tika_image_detection:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/java/TikaImageDetection tika-text-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_tika_text_detection:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/java/TikaTextDetection transformer-tagging: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_transformer_tagging:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/python/TransformerTagging trtis-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_trtis_detection:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/cpp/TrtisDetection environment: @@ -276,7 +269,7 @@ services: - trtis-detection-server trtis-detection-server: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_trtis_detection_server:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/cpp/TrtisDetection/trtserver_dockerfile deploy: @@ -294,7 +287,7 @@ services: command: [trtserver,--model-store=/models] whisper-speech-detection: - <<: *detection-component-base + <<: *component-base image: ${REGISTRY}openmpf_whisper_speech_detection:${TAG} build: ${OPENMPF_PROJECTS_PATH}/openmpf-components/python/WhisperSpeechDetection diff --git a/integration_tests/Dockerfile b/integration_tests/Dockerfile index 2cd1604c..78d36aba 100644 --- a/integration_tests/Dockerfile +++ b/integration_tests/Dockerfile @@ -93,7 +93,6 @@ ENV WFM_BASE_URL=http://localhost:8181 ENV JDBC_URL=jdbc:postgresql://db:5432/mpf ENV PYTHONUNBUFFERED 1 -COPY descriptor-receiver.py /scripts/descriptor-receiver.py COPY check-test-reports.py /scripts/check-test-reports.py COPY docker-entrypoint.sh /scripts/docker-entrypoint.sh diff --git a/integration_tests/descriptor-receiver.py b/integration_tests/descriptor-receiver.py deleted file mode 100644 index 7340ac0e..00000000 --- a/integration_tests/descriptor-receiver.py +++ /dev/null @@ -1,75 +0,0 @@ -#! /usr/bin/env python3 - -############################################################################# -# NOTICE # -# # -# This software (or technical data) was produced for the U.S. Government # -# under contract, and is subject to the Rights in Data-General Clause # -# 52.227-14, Alt. IV (DEC 2007). # -# # -# Copyright 2024 The MITRE Corporation. All Rights Reserved. # -############################################################################# - -############################################################################# -# Copyright 2024 The MITRE Corporation # -# # -# Licensed under the Apache License, Version 2.0 (the "License"); # -# you may not use this file except in compliance with the License. # -# You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -# Unless required by applicable law or agreed to in writing, software # -# distributed under the License is distributed on an "AS IS" BASIS, # -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -# See the License for the specific language governing permissions and # -# limitations under the License. # -############################################################################# - -import http.server -import json -import os - -# Creates an endpoint where Docker components can register their descriptors for use in integration tests. -def main(): - http.server.HTTPServer.request_queue_size = 100 - server = http.server.HTTPServer(('', 8080), RequestHandler) - server.serve_forever() - -class RequestHandler(http.server.BaseHTTPRequestHandler): - error_content_type = 'application/json' - error_message_format = '{"message": "%(message)s"}' - - def do_POST(self): - try: - content_len = int(self.headers['Content-Length']) - post_body = self.rfile.read(content_len) - descriptor = json.loads(post_body) - component_name = descriptor['componentName'] - except (ValueError, KeyError): - self.send_error(400, 'Failed to parse component descriptor.') - raise - - print('Received descriptor for', component_name) - - descriptor_dir = os.path.join(os.getenv('MPF_HOME', '/opt/mpf'), 'plugins', component_name, 'descriptor') - - if not os.path.exists(descriptor_dir): - os.makedirs(descriptor_dir) - - descriptor_path = os.path.join(descriptor_dir, 'descriptor.json') - if os.path.exists(descriptor_path): - print('Replacing descriptor at:', descriptor_path) - else: - print('Saving descriptor at:', descriptor_path) - - with open(descriptor_path, 'w') as f: - f.write(post_body.decode()) - - self.send_response(200) - self.end_headers() - self.wfile.write('{"message": "Descriptor stored."}'.encode()) - - -if __name__ == '__main__': - main() diff --git a/integration_tests/docker-entrypoint.sh b/integration_tests/docker-entrypoint.sh index f7a3e2a1..a6ad08f7 100755 --- a/integration_tests/docker-entrypoint.sh +++ b/integration_tests/docker-entrypoint.sh @@ -37,10 +37,6 @@ fi # Remove old test reports since /test-reports gets bind mounted in compose file. rm --recursive --force /test-reports/* - -python3 -u /scripts/descriptor-receiver.py & -descriptor_receiver_pid=$! - cd /home/mpf/openmpf-projects/openmpf # Move test sample data into a location that's accessible by all of the nodes. @@ -94,8 +90,6 @@ mvn verify -Pjenkins \ maven_exit_code=$? -kill "$descriptor_receiver_pid" - cd ../.. mkdir --parents /test-reports/surefire-reports find . -path '*/surefire-reports/*.xml' -exec cp {} /test-reports/surefire-reports \;