From 532af61a859b63d2e6a4c815a2fdb605dcb07dc0 Mon Sep 17 00:00:00 2001 From: Seth Hollandsworth Date: Tue, 28 Feb 2023 17:01:52 -0500 Subject: [PATCH 1/5] bug fixes and adding a new flag for approving wildcards for headless mode --- src/confcom/HISTORY.rst | 5 + src/confcom/README.md | 8 + src/confcom/azext_confcom/_help.py | 4 + src/confcom/azext_confcom/_params.py | 6 + src/confcom/azext_confcom/config.py | 2 + src/confcom/azext_confcom/container.py | 25 +- src/confcom/azext_confcom/custom.py | 2 + .../azext_confcom/data/internal_config.json | 2 +- src/confcom/azext_confcom/os_util.py | 8 +- src/confcom/azext_confcom/rootfs_proxy.py | 10 +- src/confcom/azext_confcom/security_policy.py | 35 ++- src/confcom/azext_confcom/template_util.py | 175 +++++++---- .../azext_confcom/tests/latest/README.md | 3 +- .../tests/latest/test_confcom_arm.py | 276 ++++++++++-------- .../tests/latest/test_confcom_tar.py | 224 ++++++++++++++ src/confcom/setup.py | 2 +- 16 files changed, 582 insertions(+), 205 deletions(-) diff --git a/src/confcom/HISTORY.rst b/src/confcom/HISTORY.rst index 72dee2e0ce3..a4a1b5b5ba1 100644 --- a/src/confcom/HISTORY.rst +++ b/src/confcom/HISTORY.rst @@ -2,6 +2,11 @@ Release History =============== +0.2.12 +* adding ability for mixed-mode OCI image pulling, e.g. using tar files and remote registries in the same template +* adding option to use allow-all regex for environment variables +* tar file bug fixes + 0.2.11 * bug fix for clean room scenario where non-existent docker client connection attempted to be closed * adding ability for ARM Template workflows to use regex for environment variables diff --git a/src/confcom/README.md b/src/confcom/README.md index 18fde73c884..ec9fca2a49a 100644 --- a/src/confcom/README.md +++ b/src/confcom/README.md @@ -4,6 +4,7 @@ - [Repository](#repository) - [Prerequisites](#prerequisites) - [Installation Instructions (End User)](#installation-instructions-end-user) + - [Current Limitations](#current-limitations) - [Trademarks](#trademarks) ## Repository @@ -48,6 +49,13 @@ az extension add -n confcom ``` +## Current Limitations + +The `confcom` extension does not currently support: + +- [ARM Template functions](https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/template-functions) other than `variables` and `parameters`. +- Variables and Parameters with non-primitive data types e.g. objects and arrays + ## Trademarks This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft diff --git a/src/confcom/azext_confcom/_help.py b/src/confcom/azext_confcom/_help.py index b89072a42a4..c195a985727 100644 --- a/src/confcom/azext_confcom/_help.py +++ b/src/confcom/azext_confcom/_help.py @@ -49,6 +49,10 @@ type: boolean short-summary: 'When enabled, the generated security policy adds the ability to use /bin/sh or /bin/bash to debug the container. It also enabled stdio access, ability to dump stack traces, and enables runtime logging. It is recommended to only use this option for debugging purposes.' + - name: --approve-wildcards -y + type: boolean + short-summary: 'When enabled, all prompts for using wildcards in environment variables are automatically approved.' + - name: --disable-stdio type: boolean short-summary: 'When enabled, the containers in the container group do not have access to stdio.' diff --git a/src/confcom/azext_confcom/_params.py b/src/confcom/azext_confcom/_params.py index 3b9b9bc1c1e..db08cf5e228 100644 --- a/src/confcom/azext_confcom/_params.py +++ b/src/confcom/azext_confcom/_params.py @@ -64,6 +64,12 @@ def load_arguments(self, _): required=False, help="Debug mode will enable processes in a container group that are helpful for debugging", ) + c.argument( + "approve_wildcards", + options_list=("--approve-wildcards", "-y"), + required=False, + help="Approving wildcards by default will get rid of the prompts during the wildcard environment variable use case and auto-approve the use of wildcards", + ) c.argument( "disable_stdio", options_list=("--disable-stdio",), diff --git a/src/confcom/azext_confcom/config.py b/src/confcom/azext_confcom/config.py index d9012f77c2c..c6a1b8be4d1 100644 --- a/src/confcom/azext_confcom/config.py +++ b/src/confcom/azext_confcom/config.py @@ -70,6 +70,8 @@ POLICY_FIELD_CONTAINERS_ELEMENTS_COMMANDS = "command" POLICY_FIELD_CONTAINERS_ELEMENTS_ENVS = "env_rules" POLICY_FIELD_CONTAINERS_ELEMENTS_ENVS_STRATEGY = "strategy" +POLICY_FIELD_CONTAINERS_ELEMENTS_ENVS_VALUE = "value" +POLICY_FIELD_CONTAINERS_ELEMENTS_ENVS_NAME = "name" POLICY_FIELD_CONTAINERS_ELEMENTS_ENVS_RULE = "pattern" POLICY_FIELD_CONTAINERS_ELEMENTS_REQUIRED = "required" POLICY_FIELD_CONTAINERS_ELEMENTS_LAYERS = "layers" diff --git a/src/confcom/azext_confcom/container.py b/src/confcom/azext_confcom/container.py index 96e1e687f7b..b52be58638e 100644 --- a/src/confcom/azext_confcom/container.py +++ b/src/confcom/azext_confcom/container.py @@ -7,7 +7,7 @@ import json import os from typing import Any, List, Dict -from azext_confcom.template_util import case_insensitive_dict_get +from azext_confcom.template_util import case_insensitive_dict_get, replace_params_and_vars from azext_confcom import config from azext_confcom.errors import eprint @@ -368,6 +368,27 @@ def get_mounts(self) -> List: def set_extra_environment_rules(self, rules: Dict) -> None: self._extraEnvironmentRules = rules + def parse_all_parameters_and_variables(self, params, vars_dict) -> None: + field_names = [ + "containerImage", + "_environmentRules", + "_command", + "_workingDir", + "_mounts", + "_identifier", + "_exec_processes", + "_extraEnvironmentRules", + ] + for field_name in field_names: + attribute = getattr(self, field_name) + out = replace_params_and_vars(params, vars_dict, attribute) + setattr(self, field_name, out) + # set these at the end since they're derived from containerImage, which could have been altered + if ":" in self.containerImage: + self.base, self.tag = self.containerImage.split(":", 1) + else: + self.base, self.tag = self.containerImage, "latest" + def _get_environment_rules(self) -> List[Dict[str, Any]]: out_rules = copy.deepcopy(self._environmentRules) env_var_names = [ @@ -469,7 +490,7 @@ def from_json( def __init__( self, containerImage: str, - environmentRules: Dict, + environmentRules: List[Dict], command: List[str], mounts: List[Dict], workingDir: str, diff --git a/src/confcom/azext_confcom/custom.py b/src/confcom/azext_confcom/custom.py index 768a0cff405..b7016452baa 100644 --- a/src/confcom/azext_confcom/custom.py +++ b/src/confcom/azext_confcom/custom.py @@ -26,6 +26,7 @@ def acipolicygen_confcom( image_name: str, infrastructure_svn: str, tar_mapping_location: str, + approve_wildcards: str = False, use_json: bool = False, outraw: bool = False, outraw_pretty_print: bool = False, @@ -97,6 +98,7 @@ def acipolicygen_confcom( arm_template_parameters, debug_mode=debug_mode, disable_stdio=disable_stdio, + approve_wildcards=approve_wildcards, ) elif image_name: container_group_policies = security_policy.load_policy_from_image_name( diff --git a/src/confcom/azext_confcom/data/internal_config.json b/src/confcom/azext_confcom/data/internal_config.json index 401ea690d6d..962a05252af 100644 --- a/src/confcom/azext_confcom/data/internal_config.json +++ b/src/confcom/azext_confcom/data/internal_config.json @@ -1,5 +1,5 @@ { - "version": "0.2.11", + "version": "0.2.12", "hcsshim_config": { "maxVersion": "1.0.0", "minVersion": "0.0.1" diff --git a/src/confcom/azext_confcom/os_util.py b/src/confcom/azext_confcom/os_util.py index 325af174d99..8a56172a4be 100644 --- a/src/confcom/azext_confcom/os_util.py +++ b/src/confcom/azext_confcom/os_util.py @@ -98,7 +98,7 @@ def map_image_from_tar(image_name: str, tar: TarFile, tar_location: str): ] info_file = None # if there's more than one image in the tarball, we need to do some more logic - if len(info_file_name) > 1: + if len(info_file_name) > 0: # extract just the manifest file and see if any of the RepoTags match the image_name we're searching for # the manifest.json should have a list of all the image tags # and what json files they map to to get env vars, startup cmd, etc. @@ -114,13 +114,11 @@ def map_image_from_tar(image_name: str, tar: TarFile, tar_location: str): break # remove the extracted manifest file to clean up os.remove(manifest_path) - elif len(info_file_name) == 0: - eprint(f"Tarball at {tar_location} contains no images") else: - info_file = info_file_name[0] + eprint(f"Tarball at {tar_location} contains no images") if not info_file: - eprint(f"Image {image_name} is not found in tarball at {tar_location}") + return None tar.extract(info_file.name, path=tar_dir) # get the path of the json file and read it in diff --git a/src/confcom/azext_confcom/rootfs_proxy.py b/src/confcom/azext_confcom/rootfs_proxy.py index 5e5ef0d26f1..3482704b66d 100644 --- a/src/confcom/azext_confcom/rootfs_proxy.py +++ b/src/confcom/azext_confcom/rootfs_proxy.py @@ -85,11 +85,13 @@ def get_policy_image_layers( output = [output[j * 2 + 1] for j in range(len(output) // 2)] output = [i.rstrip("\n").split(": ", 1)[1] for i in output] else: - eprint( - "Cannot get layer hashes. Please check whether the image exists in local repository/daemon." - ) + output = [] + # eprint( + # "Cannot get layer hashes. Please check whether the image exists in local repository/daemon." + # ) if err.decode("utf8") != "": - eprint(err.decode("utf8")) + output = [] + # eprint(err.decode("utf8")) return output diff --git a/src/confcom/azext_confcom/security_policy.py b/src/confcom/azext_confcom/security_policy.py index a5a55c0cccf..9ebfce05c2a 100644 --- a/src/confcom/azext_confcom/security_policy.py +++ b/src/confcom/azext_confcom/security_policy.py @@ -20,7 +20,6 @@ from azext_confcom.template_util import ( extract_confidential_properties, is_sidecar, - parse_template, pretty_print_func, print_func, readable_diff, @@ -31,7 +30,8 @@ process_mounts, extract_probe, process_env_vars_from_template, - get_image_info + get_image_info, + get_tar_location_from_mapping ) from azext_confcom.rootfs_proxy import SecurityPolicyProxy @@ -45,6 +45,9 @@ class OutputType(Enum): class AciPolicy: # pylint: disable=too-many-instance-attributes + all_params = {} + all_vars = {} + def __init__( self, deserialized_config: Any, @@ -393,12 +396,9 @@ def populate_policy_content_for_all_images( action="ignore", message="unclosed", category=ResourceWarning ) - client = None tar_location = "" layer_cache = {} - if not tar_mapping: - client = self._get_docker_client() - elif isinstance(tar_mapping, str): + if isinstance(tar_mapping, str): tar_location = tar_mapping proxy = self._get_rootfs_proxy() container_images = self.get_images() @@ -419,9 +419,9 @@ def populate_policy_content_for_all_images( message_queue = [] # populate regular container images(s) for image in container_images: - + image.parse_all_parameters_and_variables(AciPolicy.all_params, AciPolicy.all_vars) image_name = f"{image.base}:{image.tag}" - image_info = get_image_info(progress, message_queue, client, tar_mapping, image) + image_info, tar = get_image_info(progress, message_queue, tar_mapping, image) # verify and populate the working directory property if not image.get_working_dir() and image_info: @@ -471,12 +471,15 @@ def populate_policy_content_for_all_images( } ) + # populate tar location + if isinstance(tar_mapping, dict): + tar_location = get_tar_location_from_mapping(tar_mapping, image_name) # populate layer info if layer_cache.get(image_name): image.set_layers(layer_cache.get(image_name)) else: image.set_layers(proxy.get_policy_image_layers( - image.base, image.tag, tar_location=tar_location + image.base, image.tag, tar_location=tar_location if tar else "" )) layer_cache[image_name] = image.get_layers() progress.update() @@ -487,7 +490,7 @@ def populate_policy_content_for_all_images( for message in message_queue: logger.warning(message) - def get_images(self) -> List[Any]: + def get_images(self) -> List[ContainerImage]: return self._images def pull_image(self, image: ContainerImage) -> Any: @@ -501,6 +504,7 @@ def load_policy_from_arm_template_str( infrastructure_svn: str = None, debug_mode: bool = False, disable_stdio: bool = False, + approve_wildcards: bool = False, ) -> List[AciPolicy]: """Function that converts ARM template string to an ACI Policy""" input_arm_json = os_util.load_json_from_str(template_data) @@ -537,9 +541,8 @@ def load_policy_from_arm_template_str( get_values_for_params(input_parameter_json, all_params) - input_arm_json = parse_template(all_params, - case_insensitive_dict_get(input_arm_json, config.ACI_FIELD_TEMPLATE_VARIABLES) - or {}, input_arm_json) + AciPolicy.all_params = all_params + AciPolicy.all_vars = case_insensitive_dict_get(input_arm_json, config.ACI_FIELD_TEMPLATE_VARIABLES) or {} container_groups = [] @@ -612,7 +615,8 @@ def load_policy_from_arm_template_str( { config.ACI_FIELD_CONTAINERS_ID: image_name, config.ACI_FIELD_CONTAINERS_CONTAINERIMAGE: image_name, - config.ACI_FIELD_CONTAINERS_ENVS: process_env_vars_from_template(image_properties), + config.ACI_FIELD_CONTAINERS_ENVS: process_env_vars_from_template( + AciPolicy.all_params, AciPolicy.all_vars, image_properties, approve_wildcards), config.ACI_FIELD_CONTAINERS_COMMAND: case_insensitive_dict_get( image_properties, config.ACI_FIELD_TEMPLATE_COMMAND ) @@ -651,6 +655,7 @@ def load_policy_from_arm_template_file( parameter_path: str, debug_mode: bool = False, disable_stdio: bool = False, + approve_wildcards: bool = False, ) -> List[AciPolicy]: """Utility function: generate policy object from given arm template and parameter file paths""" input_arm_json = os_util.load_str_from_file(template_path) @@ -659,7 +664,7 @@ def load_policy_from_arm_template_file( input_parameter_json = os_util.load_str_from_file(parameter_path) return load_policy_from_arm_template_str( input_arm_json, input_parameter_json, infrastructure_svn, - debug_mode=debug_mode, disable_stdio=disable_stdio + debug_mode=debug_mode, disable_stdio=disable_stdio, approve_wildcards=approve_wildcards, ) diff --git a/src/confcom/azext_confcom/template_util.py b/src/confcom/azext_confcom/template_util.py index 4e9fd8ab68a..cb7f7c9f199 100644 --- a/src/confcom/azext_confcom/template_util.py +++ b/src/confcom/azext_confcom/template_util.py @@ -18,6 +18,26 @@ from azext_confcom import config +# TODO: these can be optimized to not have so many groups in the single match +# make this global so it can be used in multiple functions +PARAMETER_AND_VARIABLE_REGEX = r"\[(?:parameters|variables)\(\s*'([^\.\/]+?)'\s*\)\]" +WHOLE_PARAMETER_AND_VARIABLE = r"(\s*\[\s*(parameters|variables))(\(\s*'([^\.\/]+?)'\s*\)\])" + + +class DockerClient: + def __init__(self) -> None: + self._client = None + + def get_client(self) -> docker.DockerClient: + if not self._client: + self._client = docker.from_env() + return self._client + + def __exit__(self, exc_type, exc_value, traceback) -> None: + if self._client: + self._client.close() + + def case_insensitive_dict_get(dictionary, search_key) -> Any: if not isinstance(dictionary, dict): return None @@ -32,9 +52,12 @@ def case_insensitive_dict_get(dictionary, search_key) -> Any: return None -def get_image_info(progress, message_queue, client, tar_mapping, image): +def get_image_info(progress, message_queue, tar_mapping, image): image_info = None raw_image = None + tar = False + if not image.base: + eprint("Image name cannot be empty") image_name = f"{image.base}:{image.tag}" if len(image.tag.split(":")) > 1: eprint( @@ -44,45 +67,55 @@ def get_image_info(progress, message_queue, client, tar_mapping, image): # we want to do if tar_mapping: tar_location = get_tar_location_from_mapping(tar_mapping, image_name) - with tarfile.open(tar_location) as tar: - # get all the info out of the tarfile - image_info = os_util.map_image_from_tar( - image_name, tar, tar_location - ) - message_queue.append("read from local tar file") - else: - # see if we have the image locally so we can have a - # 'clean-room' - if not image_info: - try: - raw_image = client.images.get(image_name) - image_info = raw_image.attrs.get("Config") - message_queue.append( - f"Using local version of {image_name}. It may differ from the remote image" - ) - except docker.errors.ImageNotFound: - message_queue.append( - f"{image_name} is not found locally. Attempting to pull from remote..." + # if we have a tar location, we can try to get the image info + if tar_location: + with tarfile.open(tar_location) as tar: + # get all the info out of the tarfile + image_info = os_util.map_image_from_tar( + image_name, tar, tar_location ) - - if not image_info: - try: - # pull image to local daemon (if not in local - # daemon) - if not raw_image: - raw_image = client.images.pull(image.base, image.tag) - image_info = raw_image.attrs.get("Config") - except (docker.errors.ImageNotFound, docker.errors.NotFound): - progress.close() - eprint( - f"{image_name} is not found remotely. " - + "Please check to make sure the image and repository exist" - ) - # warn if the image is the "latest" - if image.tag == "latest": + if image_info is not None: + tar = True + message_queue.append(f"{image_name} read from local tar file") + + # see if we have the image locally so we can have a + # 'clean-room' + if not image_info: + try: + client = DockerClient().get_client() + raw_image = client.images.get(image_name) + image_info = raw_image.attrs.get("Config") + message_queue.append( + f"Using local version of {image_name}. It may differ from the remote image" + ) + except docker.errors.ImageNotFound: message_queue.append( - 'Using image tag "latest" is not recommended' + f"{image_name} is not found locally. Attempting to pull from remote..." ) + except docker.errors.DockerException: + progress.close() + eprint( + f"{image_name} is not found in tar file and Docker is not running." + ) + + if not image_info: + try: + # pull image to local daemon (if not in local + # daemon) + if not raw_image: + raw_image = client.images.pull(image.base, image.tag) + image_info = raw_image.attrs.get("Config") + except (docker.errors.ImageNotFound, docker.errors.NotFound): + progress.close() + eprint( + f"{image_name} is not found remotely. " + + "Please check to make sure the image and repository exist" + ) + # warn if the image is the "latest" + if image.tag == "latest": + message_queue.append( + 'Using image tag "latest" is not recommended' + ) progress.update() @@ -105,7 +138,7 @@ def get_image_info(progress, message_queue, client, tar_mapping, image): + f"Only {config.ACI_FIELD_CONTAINERS_ARCHITECTURE_VALUE} is supported by Confidential ACI" ) - return image_info + return image_info, tar def get_tar_location_from_mapping(tar_mapping: Any, image_name: str) -> str: @@ -122,15 +155,15 @@ def get_tar_location_from_mapping(tar_mapping: Any, image_name: str) -> str: ) else: tar_location = tar_mapping - # this needs to exist to continue - if not tar_location: - eprint( - f"The image {image_name} is not present in the tarball mapping file" - ) + # for mixed mode, the image doesn't have to be in the tarfile + # so this can return None return tar_location -def process_env_vars_from_template(image_properties: dict) -> List[Dict[str, str]]: +def process_env_vars_from_template(params: dict, + vars_dict: dict, + image_properties: dict, + approve_wildcards: bool) -> List[Dict[str, str]]: env_vars = [] # add in the env vars from the template template_env_vars = case_insensitive_dict_get( @@ -147,10 +180,15 @@ def process_env_vars_from_template(image_properties: dict) -> List[Dict[str, str f"Environment variable with value: {value} is missing a name" ) - if value: - if config.ACI_FIELD_TEMPLATE_PARAMETERS in value: - response = input(f'Create a wildcard policy for the environment variable {name} (y/n): ') - if response.lower() == 'y': + if value is not None: + param_check = find_value_in_params_and_vars( + params, vars_dict, value, ignore_undefined_parameters=True) + param_name = re.findall(PARAMETER_AND_VARIABLE_REGEX, value) + + if param_name and param_check == value: + response = approve_wildcards or input( + f'Create a wildcard policy for the environment variable {name} (y/n): ') + if approve_wildcards or response.lower() == 'y': env_vars.append({ config.ACI_FIELD_CONTAINERS_ENVS_NAME: name, config.ACI_FIELD_CONTAINERS_ENVS_VALUE: ".*", @@ -341,14 +379,35 @@ def change_key_names(dictionary) -> Dict: return dictionary +def replace_params_and_vars(params: dict, vars_dict: dict, attribute): + out = None + if isinstance(attribute, (int, float, bool)): + out = attribute + elif isinstance(attribute, str): + out = find_value_in_params_and_vars(params, vars_dict, attribute) + param_name = re.finditer(WHOLE_PARAMETER_AND_VARIABLE, attribute) + + # there should only be one match + full_param_name = next(param_name, None) + if full_param_name: + full_param_name = full_param_name.group(0) + out = attribute.replace(full_param_name, find_value_in_params_and_vars(params, vars_dict, attribute)) + elif isinstance(attribute, list): + out = [] + for item in attribute: + out.append(replace_params_and_vars(params, vars_dict, item)) + elif isinstance(attribute, dict): + out = {} + for key, value in attribute.items(): + out[key] = replace_params_and_vars(params, vars_dict, value) + return out + + def find_value_in_params_and_vars(params: dict, vars_dict: dict, search: str, ignore_undefined_parameters=False) -> str: """Utility function: either returns the input search value, or replaces it with the defined value in either params or vars of the ARM template""" # this pattern might need to be updated for more naming options in the future - # pattern = "(parameters|variables)\('([\w\-\_0-9]+)'\)" - pattern = r"(?:parameters|variables)\(\s*'([^\.\/]+?)'\s*\)" - param_name = re.findall(pattern, search) - + param_name = re.findall(PARAMETER_AND_VARIABLE_REGEX, search) if not param_name: return search @@ -357,12 +416,12 @@ def find_value_in_params_and_vars(params: dict, vars_dict: dict, search: str, ig # figure out if we need to search in variables or parameters - match = "" + match = None if config.ACI_FIELD_TEMPLATE_PARAMETERS in search: param_value = case_insensitive_dict_get(params, param_name) - if not param_value: + if param_value is None: eprint( f"""Field "{param_name}" not found in ["{config.ACI_FIELD_TEMPLATE_PARAMETERS}"] or ["{config.ACI_FIELD_TEMPLATE_VARIABLES}"]""" @@ -370,17 +429,17 @@ def find_value_in_params_and_vars(params: dict, vars_dict: dict, search: str, ig # fallback to default value match = case_insensitive_dict_get( param_value, "value" - ) or case_insensitive_dict_get(param_value, "defaultValue") + ) if "value" in param_value else case_insensitive_dict_get(param_value, "defaultValue") else: match = case_insensitive_dict_get(vars_dict, param_name) - if not match and not ignore_undefined_parameters: + if match is None and not ignore_undefined_parameters: eprint( f"""Field "{param_name}"'s value not found in ["{config.ACI_FIELD_TEMPLATE_PARAMETERS}"] or ["{config.ACI_FIELD_TEMPLATE_VARIABLES}"]""" ) - return match or search + return match if match is not None else search def parse_template(params: dict, vars_dict: dict, template, ignore_undefined_parameters=False) -> Any: @@ -695,7 +754,7 @@ def get_container_group_name( eprint( f'Field ["{config.ACI_FIELD_TEMPLATE_PARAMETERS}"] is empty or cannot be found in Parameter file' ) - + # TODO: replace this with doing param replacement as-needed arm_json = parse_template(all_params, all_vars, arm_json) # find the image names and extract them from the template arm_resources = case_insensitive_dict_get(arm_json, config.ACI_FIELD_RESOURCES) diff --git a/src/confcom/azext_confcom/tests/latest/README.md b/src/confcom/azext_confcom/tests/latest/README.md index dfc916e0574..41ea57a2208 100644 --- a/src/confcom/azext_confcom/tests/latest/README.md +++ b/src/confcom/azext_confcom/tests/latest/README.md @@ -25,7 +25,6 @@ test_arm_template_missing_image_name | N/A | Error condition if an image isn't s test_arm_template_missing_resources | N/A | Error condition where no resources are specified to deploy test_arm_template_missing_aci | N/A | Error condition where ACI is not specified in resources test_arm_template_missing_containers | N/A | Error condition where there are no containers in the ACI resource -test_arm_template_missing_default_value | N/A | Error condition where there aren't default values or defined values for parameters in the ARM Template test_arm_template_missing_definition | python:3.6.14-slim-buster | Error condition where image is specified in template.parameters.json but not in template.json test_arm_template_with_parameter_file | mcr.microsoft.com/azure-functions/python:4-python3.8 | Condition where image in template.parameters.json overwrites image name in template.json test_arm_template_with_parameter_file_injected_env_vars | mcr.microsoft.com/azure-functions/python:4-python3.8 | See if env vars from the image are injected into the policy. Also make sure the `concat` function in ARM template won't break the CLI if it's not in a required spot like image name @@ -40,6 +39,7 @@ test_arm_template_without_stdio_access | rust:1.52.1 | See if disabling containe test_arm_template_policy_regex | python:3.6.14-slim-buster | Make sure the regex generated from the ARM Template workflow matches that of the policy.json workflow test_wildcard_env_var | python:3.6.14-slim-buster | Check that an "allow all" regex is created when a value for env var is not provided via a parameter value test_wildcard_env_var_invalid | N/A | Make sure the process errors out if a value is not given for an env var or an undefined parameter is used for the name of an env var +test_arm_template_with_env_var | rust:1.52.1 | Make sure that a value that looks similar to but is not an ARM parameter is treated as a string ## policy.json [test file](test_confcom_scenario.py) @@ -104,5 +104,6 @@ This is a way to generate a CCE policy without the use of the docker daemon. The Test Name | Image Used | Purpose ---|---|--- test_arm_template_with_parameter_file_clean_room_tar | nginx:1.23 | Create a policy from a tar file and compare it to a policy generated from an ARM template +test_arm_template_mixed_mode_tar | python:3.9 & nginx:1.22 | Create a policy with one image from a tar file and one image that must be downloaded or used locally from the daemon test_arm_template_with_parameter_file_clean_room_tar_invalid | N/A | Fail out if searching for an image in a tar file that does not include it test_clean_room_fake_tar_invalid | N/A | Fail out if the path to the tar file doesn't exist diff --git a/src/confcom/azext_confcom/tests/latest/test_confcom_arm.py b/src/confcom/azext_confcom/tests/latest/test_confcom_arm.py index da0526623e2..bc54a58f5cd 100644 --- a/src/confcom/azext_confcom/tests/latest/test_confcom_arm.py +++ b/src/confcom/azext_confcom/tests/latest/test_confcom_arm.py @@ -286,7 +286,9 @@ def test_arm_template_missing_image_name(self): { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", - + "variables": { + "image": "" + }, "parameters": { "containergroupname": { @@ -386,7 +388,10 @@ def test_arm_template_missing_image_name(self): """ with self.assertRaises(SystemExit) as exc_info: - load_policy_from_arm_template_str(custom_arm_json_missing_image_name, "") + out_policies = load_policy_from_arm_template_str(custom_arm_json_missing_image_name, "") + for policy in out_policies: + policy.populate_policy_content_for_all_images() + self.assertEqual(exc_info.exception.code, 1) def test_arm_template_missing_resources(self): @@ -610,121 +615,6 @@ def test_arm_template_missing_containers(self): # @unittest.skip("not in use") @pytest.mark.run(order=3) class PolicyGeneratingArmParametersIncorrect(unittest.TestCase): - def test_arm_template_missing_default_value(self): - custom_arm_json_missing_default_value = """ - { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - - - "parameters": { - "containergroupname": { - "type": "string", - "metadata": { - "description": "Name for the container group" - } - }, - - "containername": { - "type": "string", - "metadata": { - "description": "Name for the container" - } - }, - "image": { - "type": "string", - "metadata": { - "description": "Name for the image" - } - }, - - "port": { - "type": "string", - "metadata": { - "description": "Port to open on the container and the public IP address." - }, - "defaultValue": "8080" - }, - "cpuCores": { - "type": "string", - "metadata": { - "description": "The number of CPU cores to allocate to the container." - }, - "defaultValue": "1.0" - }, - "memoryInGb": { - "type": "string", - "metadata": { - "description": "The amount of memory to allocate to the container in gigabytes." - }, - "defaultValue": "1.5" - }, - "location": { - "type": "string", - "defaultValue": "[resourceGroup().location]", - "metadata": { - "description": "Location for all resources." - } - } - }, - "resources": [ - { - "name": "[parameters('containergroupname')]", - "type": "Microsoft.ContainerInstance/containerGroups", - "apiVersion": "2022-04-01-preview", - "location": "[parameters('location')]", - - "properties": { - "containers": [ - { - "name": "[parameters('containername')]", - "properties": { - "image": "[parameters('image')]", - "ports": [ - { - "port": "[parameters('port')]" - } - ], - "resources": { - "requests": { - "cpu": "[parameters('cpuCores')]", - "memoryInGb": "[parameters('memoryInGb')]" - } - } - } - } - ], - - "osType": "Linux", - "restartPolicy": "OnFailure", - "confidentialComputeProperties": { - "IsolationType": "SevSnp" - }, - "ipAddress": { - "type": "Public", - "ports": [ - { - "protocol": "Tcp", - "port": "[parameters( 'port' )]" - } - ] - } - } - } - ], - "outputs": { - "containerIPv4Address": { - "type": "string", - "value": "[reference(resourceId('Microsoft.ContainerInstance/containerGroups/', parameters('containergroupname'))).ipAddress.ip]" - } - } - } - """ - - with self.assertRaises(SystemExit) as exc_info: - load_policy_from_arm_template_str(custom_arm_json_missing_default_value, "") - self.assertEqual(exc_info.exception.code, 1) - def test_arm_template_missing_definition(self): custom_arm_json_missing_definition = """ { @@ -3418,7 +3308,157 @@ def test_wildcard_env_var_invalid(self): self.assertEqual(wrapped_exit.exception.code, 1) with self.assertRaises(SystemExit) as wrapped_exit: - load_policy_from_arm_template_str(self.custom_arm_json_error2, "") + out = load_policy_from_arm_template_str(self.custom_arm_json_error2, "") + for policy in out: + policy.populate_policy_content_for_all_images() + self.assertEqual(wrapped_exit.exception.code, 1) +# @unittest.skip("not in use") +@pytest.mark.run(order=15) +class PolicyGeneratingEdgeCases(unittest.TestCase): + + custom_arm_json_default_value = """ + { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + + + "parameters": { + "containergroupname": { + "type": "string", + "metadata": { + "description": "Name for the container group" + }, + "defaultValue":"simple-container-group" + }, + "image": { + "type": "string", + "metadata": { + "description": "Name for the container group" + }, + "defaultValue":"rust:1.52.1" + }, + "containername": { + "type": "string", + "metadata": { + "description": "Name for the container" + }, + "defaultValue":"simple-container" + }, + + "port": { + "type": "string", + "metadata": { + "description": "Port to open on the container and the public IP address." + }, + "defaultValue": "8080" + }, + "cpuCores": { + "type": "string", + "metadata": { + "description": "The number of CPU cores to allocate to the container." + }, + "defaultValue": "1.0" + }, + "memoryInGb": { + "type": "string", + "metadata": { + "description": "The amount of memory to allocate to the container in gigabytes." + }, + "defaultValue": "1.5" + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Location for all resources." + } + } + }, + "resources": [ + { + "name": "[parameters('containergroupname')]", + "type": "Microsoft.ContainerInstance/containerGroups", + "apiVersion": "2022-04-01-preview", + "location": "[parameters('location')]", + + "properties": { + "containers": [ + { + "name": "[parameters('containername')]", + "properties": { + "image": "[parameters('image')]", + "environmentVariables": [ + { + "name": "PORT", + "value": "parameters('abc')" + } + ], + + "ports": [ + { + "port": "[parameters('port')]" + } + ], + "command": [ + "/bin/bash", + "-c", + "while sleep 5; do cat /mnt/input/access.log; done" + ], + "resources": { + "requests": { + "cpu": "[parameters('cpuCores')]", + "memoryInGb": "[parameters('memoryInGb')]" + } + } + } + } + ], + + "osType": "Linux", + "restartPolicy": "OnFailure", + "confidentialComputeProperties": { + "IsolationType": "SevSnp" + }, + "ipAddress": { + "type": "Public", + "ports": [ + { + "protocol": "Tcp", + "port": "[parameters( 'port' )]" + } + ] + } + } + } + ], + "outputs": { + "containerIPv4Address": { + "type": "string", + "value": "[reference(resourceId('Microsoft.ContainerInstance/containerGroups/', parameters('containergroupname'))).ipAddress.ip]" + } + } + } + """ + + @classmethod + def setUpClass(cls): + cls.aci_arm_policy = load_policy_from_arm_template_str( + cls.custom_arm_json_default_value, "" + )[0] + cls.aci_arm_policy.populate_policy_content_for_all_images() + + def test_arm_template_with_env_var(self): + regular_image_json = json.loads( + self.aci_arm_policy.get_serialized_output( + output_type=OutputType.RAW, rego_boilerplate=False + ) + ) + + env_var = regular_image_json[0][config.POLICY_FIELD_CONTAINERS_ELEMENTS_ENVS][0][config.POLICY_FIELD_CONTAINERS_ELEMENTS_ENVS_RULE] + + # see if the remote image and the local one produce the same output + self.assertEquals(env_var, "PORT=parameters('abc')") + self.assertEquals(regular_image_json[0][config.POLICY_FIELD_CONTAINERS_ID], "rust:1.52.1") \ No newline at end of file diff --git a/src/confcom/azext_confcom/tests/latest/test_confcom_tar.py b/src/confcom/azext_confcom/tests/latest/test_confcom_tar.py index 6aa7524a586..af75d819790 100644 --- a/src/confcom/azext_confcom/tests/latest/test_confcom_tar.py +++ b/src/confcom/azext_confcom/tests/latest/test_confcom_tar.py @@ -213,6 +213,230 @@ def test_arm_template_with_parameter_file_clean_room_tar(self): {}, ) + def test_arm_template_mixed_mode_tar(self): + custom_arm_json_default_value = """ + { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + + + "parameters": { + "containergroupname": { + "type": "string", + "metadata": { + "description": "Name for the container group" + }, + "defaultValue":"simple-container-group" + }, + "image": { + "type": "string", + "metadata": { + "description": "Name for the container group" + }, + "defaultValue":"nginx:1.22" + }, + "containername": { + "type": "string", + "metadata": { + "description": "Name for the container" + }, + "defaultValue":"simple-container" + }, + "image2": { + "type": "string", + "metadata": { + "description": "Name for the container group" + }, + "defaultValue":"python:3.9" + }, + "containername2": { + "type": "string", + "metadata": { + "description": "Name for the container" + }, + "defaultValue":"simple-container2" + }, + + "port": { + "type": "string", + "metadata": { + "description": "Port to open on the container and the public IP address." + }, + "defaultValue": "8080" + }, + "cpuCores": { + "type": "string", + "metadata": { + "description": "The number of CPU cores to allocate to the container." + }, + "defaultValue": "1.0" + }, + "memoryInGb": { + "type": "string", + "metadata": { + "description": "The amount of memory to allocate to the container in gigabytes." + }, + "defaultValue": "1.5" + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Location for all resources." + } + } + }, + "resources": [ + { + "name": "[parameters('containergroupname')]", + "type": "Microsoft.ContainerInstance/containerGroups", + "apiVersion": "2022-04-01-preview", + "location": "[parameters('location')]", + + "properties": { + "containers": [ + { + "name": "[parameters('containername')]", + "properties": { + "image": "[parameters('image')]", + "environmentVariables": [ + { + "name": "PORT", + "value": "80" + } + ], + + "ports": [ + { + "port": "[parameters('port')]" + } + ], + "command": [ + "/bin/bash", + "-c", + "while sleep 5; do cat /mnt/input/access.log; done" + ], + "resources": { + "requests": { + "cpu": "[parameters('cpuCores')]", + "memoryInGb": "[parameters('memoryInGb')]" + } + } + } + }, + { + "name": "[parameters('containername2')]", + + "properties": { + "image": "[parameters('image2')]", + "command": [ + "python3" + ], + "ports": [ + { + "port": "[parameters('port')]" + } + ], + "resources": { + "requests": { + "cpu": "[parameters('cpuCores')]", + "memoryInGb": "[parameters('memoryInGb')]" + } + }, + "environmentVariables": [ + { + "name": "PATH", + "value": "/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + } + ] + } + } + ], + + "osType": "Linux", + "restartPolicy": "OnFailure", + "confidentialComputeProperties": { + "IsolationType": "SevSnp" + }, + "ipAddress": { + "type": "Public", + "ports": [ + { + "protocol": "Tcp", + "port": "[parameters( 'port' )]" + } + ] + } + } + } + ], + "outputs": { + "containerIPv4Address": { + "type": "string", + "value": "[reference(resourceId('Microsoft.ContainerInstance/containerGroups/', parameters('containergroupname'))).ipAddress.ip]" + } + } + } + """ + + regular_image = load_policy_from_arm_template_str( + custom_arm_json_default_value, "" + )[0] + + regular_image.populate_policy_content_for_all_images() + + clean_room_image = load_policy_from_arm_template_str( + custom_arm_json_default_value, "" + )[0] + + # save the tar file for the image in the testing directory + client = docker.from_env() + image = client.images.get("nginx:1.22") + + # Note: Class setup and teardown shouldn't have side effects, and reading from the tar file fails when all the tests are running in parallel, so we want to save and delete this tar file as a part of the test. Not as a part of the testing class. + f = open(self.image_path, "wb") + for chunk in image.save(named=True): + f.write(chunk) + f.close() + client.close() + + try: + clean_room_image.populate_policy_content_for_all_images( + tar_mapping=self.tar_mapping_file + ) + finally: + # delete the tar file + if os.path.isfile(self.image_path): + os.remove(self.image_path) + + regular_image_json = json.loads( + regular_image.get_serialized_output(output_type=OutputType.RAW, use_json=True) + ) + + clean_room_json = json.loads( + clean_room_image.get_serialized_output(output_type=OutputType.RAW, use_json=True) + ) + + regular_image_json[config.POLICY_FIELD_CONTAINERS][ + config.POLICY_FIELD_CONTAINERS_ELEMENTS + ]["0"].pop(config.POLICY_FIELD_CONTAINERS_ID) + clean_room_json[config.POLICY_FIELD_CONTAINERS][ + config.POLICY_FIELD_CONTAINERS_ELEMENTS + ]["0"].pop(config.POLICY_FIELD_CONTAINERS_ID) + regular_image_json[config.POLICY_FIELD_CONTAINERS][ + config.POLICY_FIELD_CONTAINERS_ELEMENTS + ]["1"].pop(config.POLICY_FIELD_CONTAINERS_ID) + clean_room_json[config.POLICY_FIELD_CONTAINERS][ + config.POLICY_FIELD_CONTAINERS_ELEMENTS + ]["1"].pop(config.POLICY_FIELD_CONTAINERS_ID) + + # see if the remote image and the local one produce the same output + self.assertEqual( + deepdiff.DeepDiff(regular_image_json, clean_room_json, ignore_order=True), + {}, + ) + + def test_arm_template_with_parameter_file_clean_room_tar_invalid(self): custom_arm_json_default_value = """ { diff --git a/src/confcom/setup.py b/src/confcom/setup.py index 2982235c626..5d6211b6250 100644 --- a/src/confcom/setup.py +++ b/src/confcom/setup.py @@ -21,7 +21,7 @@ # TODO: Confirm this is the right version number you want and it matches your # HISTORY.rst entry. -VERSION = "0.2.11" +VERSION = "0.2.12" # The full list of classifiers is available at # https://pypi.python.org/pypi?%3Aaction=list_classifiers From f3d16461eb3c90066190a4d7cdae52db87fb210d Mon Sep 17 00:00:00 2001 From: Seth Hollandsworth Date: Wed, 8 Mar 2023 15:12:17 -0500 Subject: [PATCH 2/5] making layer caching work across multiple container groups --- src/confcom/azext_confcom/rootfs_proxy.py | 15 +++++++++++---- src/confcom/azext_confcom/security_policy.py | 12 ++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/confcom/azext_confcom/rootfs_proxy.py b/src/confcom/azext_confcom/rootfs_proxy.py index 3482704b66d..88ab6025303 100644 --- a/src/confcom/azext_confcom/rootfs_proxy.py +++ b/src/confcom/azext_confcom/rootfs_proxy.py @@ -17,6 +17,9 @@ class SecurityPolicyProxy: # pylint: disable=too-few-public-methods + # static variable to cache layer hashes between container groups + layer_cache = {} + def __init__(self): script_directory = os.path.dirname(os.path.realpath(__file__)) DEFAULT_LIB = "./bin/dmverity-vhd" @@ -49,9 +52,12 @@ def __init__(self): def get_policy_image_layers( self, image: str, tag: str, tar_location: str = "" ) -> List[str]: - policy_bin_str = str(self.policy_bin) + image_name = f"{image}:{tag}" + # populate layer info + if self.layer_cache.get(image_name): + return self.layer_cache.get(image_name) - img = image + ":" + tag + policy_bin_str = str(self.policy_bin) arg_list = [ f"{policy_bin_str}", @@ -64,7 +70,7 @@ def get_policy_image_layers( arg_list += ["-d"] # add the image to the end of the parameter list - arg_list += ["roothash", "-i", f"{img}"] + arg_list += ["roothash", "-i", f"{image_name}"] outputlines = None err = None @@ -93,5 +99,6 @@ def get_policy_image_layers( if err.decode("utf8") != "": output = [] # eprint(err.decode("utf8")) - + # cache output layers + self.layer_cache[image_name] = output return output diff --git a/src/confcom/azext_confcom/security_policy.py b/src/confcom/azext_confcom/security_policy.py index 9ebfce05c2a..f928382ac71 100644 --- a/src/confcom/azext_confcom/security_policy.py +++ b/src/confcom/azext_confcom/security_policy.py @@ -397,7 +397,6 @@ def populate_policy_content_for_all_images( ) tar_location = "" - layer_cache = {} if isinstance(tar_mapping, str): tar_location = tar_mapping proxy = self._get_rootfs_proxy() @@ -475,13 +474,10 @@ def populate_policy_content_for_all_images( if isinstance(tar_mapping, dict): tar_location = get_tar_location_from_mapping(tar_mapping, image_name) # populate layer info - if layer_cache.get(image_name): - image.set_layers(layer_cache.get(image_name)) - else: - image.set_layers(proxy.get_policy_image_layers( - image.base, image.tag, tar_location=tar_location if tar else "" - )) - layer_cache[image_name] = image.get_layers() + image.set_layers(proxy.get_policy_image_layers( + image.base, image.tag, tar_location=tar_location if tar else "" + )) + progress.update() progress.close() self.close() From e4465e92d3c62b04270edfd92071814f3026e518 Mon Sep 17 00:00:00 2001 From: Seth Hollandsworth Date: Mon, 13 Mar 2023 10:57:38 -0400 Subject: [PATCH 3/5] adding a couple bug fixes for using sha hashes --- src/confcom/azext_confcom/template_util.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/confcom/azext_confcom/template_util.py b/src/confcom/azext_confcom/template_util.py index cb7f7c9f199..275c0d982d0 100644 --- a/src/confcom/azext_confcom/template_util.py +++ b/src/confcom/azext_confcom/template_util.py @@ -52,6 +52,10 @@ def case_insensitive_dict_get(dictionary, search_key) -> Any: return None +def image_has_hash(image: str) -> bool: + return "@sha256:" in image + + def get_image_info(progress, message_queue, tar_mapping, image): image_info = None raw_image = None @@ -59,13 +63,15 @@ def get_image_info(progress, message_queue, tar_mapping, image): if not image.base: eprint("Image name cannot be empty") image_name = f"{image.base}:{image.tag}" - if len(image.tag.split(":")) > 1: - eprint( - f"The image name: {image.tag} cannot have the digest present to use a tarball as the image source" - ) + # only try to grab the info locally if that's absolutely what # we want to do if tar_mapping: + if image_has_hash(image_name): + progress.close() + eprint( + f"The image name: {image_name} cannot have the digest present to use a tarball as the image source" + ) tar_location = get_tar_location_from_mapping(tar_mapping, image_name) # if we have a tar location, we can try to get the image info if tar_location: @@ -103,7 +109,7 @@ def get_image_info(progress, message_queue, tar_mapping, image): # pull image to local daemon (if not in local # daemon) if not raw_image: - raw_image = client.images.pull(image.base, image.tag) + raw_image = client.images.pull(image_name) image_info = raw_image.attrs.get("Config") except (docker.errors.ImageNotFound, docker.errors.NotFound): progress.close() From 5b9f8a798bf5392cf14bb7c05d1e6ac9eefbdde9 Mon Sep 17 00:00:00 2001 From: Seth Hollandsworth Date: Tue, 14 Mar 2023 11:14:25 -0400 Subject: [PATCH 4/5] updating version number --- src/confcom/HISTORY.rst | 4 ++++ src/confcom/azext_confcom/data/internal_config.json | 2 +- src/confcom/setup.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/confcom/HISTORY.rst b/src/confcom/HISTORY.rst index a4a1b5b5ba1..3921f17b670 100644 --- a/src/confcom/HISTORY.rst +++ b/src/confcom/HISTORY.rst @@ -2,6 +2,10 @@ Release History =============== +0.2.13 +* fixing bug where you could not pull by sha value if a tag was not specified +* fixing error message when attempting to use sha value with tar files + 0.2.12 * adding ability for mixed-mode OCI image pulling, e.g. using tar files and remote registries in the same template * adding option to use allow-all regex for environment variables diff --git a/src/confcom/azext_confcom/data/internal_config.json b/src/confcom/azext_confcom/data/internal_config.json index 962a05252af..5c29558af2f 100644 --- a/src/confcom/azext_confcom/data/internal_config.json +++ b/src/confcom/azext_confcom/data/internal_config.json @@ -1,5 +1,5 @@ { - "version": "0.2.12", + "version": "0.2.13", "hcsshim_config": { "maxVersion": "1.0.0", "minVersion": "0.0.1" diff --git a/src/confcom/setup.py b/src/confcom/setup.py index 5d6211b6250..364ebd90b65 100644 --- a/src/confcom/setup.py +++ b/src/confcom/setup.py @@ -21,7 +21,7 @@ # TODO: Confirm this is the right version number you want and it matches your # HISTORY.rst entry. -VERSION = "0.2.12" +VERSION = "0.2.13" # The full list of classifiers is available at # https://pypi.python.org/pypi?%3Aaction=list_classifiers From 86980bb7c7f026de1dc0f0e00804f6a65e2a03fa Mon Sep 17 00:00:00 2001 From: Seth Hollandsworth Date: Tue, 14 Mar 2023 11:20:59 -0400 Subject: [PATCH 5/5] updating feature update list --- src/confcom/HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/src/confcom/HISTORY.rst b/src/confcom/HISTORY.rst index 3921f17b670..770f05f1382 100644 --- a/src/confcom/HISTORY.rst +++ b/src/confcom/HISTORY.rst @@ -5,6 +5,7 @@ Release History 0.2.13 * fixing bug where you could not pull by sha value if a tag was not specified * fixing error message when attempting to use sha value with tar files +* making image caching template-wide instead of container group-wide 0.2.12 * adding ability for mixed-mode OCI image pulling, e.g. using tar files and remote registries in the same template