From 27627aa4b3bb8764445da4ad6e430e3cf08d8df9 Mon Sep 17 00:00:00 2001 From: Stephen Shao Date: Wed, 23 Jul 2025 11:44:38 -0400 Subject: [PATCH 01/10] Update the flow use per-model registry settings for both build and run phase --- src/madengine/mad_cli.py | 11 ++++-- .../tools/distributed_orchestrator.py | 38 ++++--------------- src/madengine/tools/docker_builder.py | 30 +++++++++------ 3 files changed, 34 insertions(+), 45 deletions(-) diff --git a/src/madengine/mad_cli.py b/src/madengine/mad_cli.py index 6f238276..3a578908 100644 --- a/src/madengine/mad_cli.py +++ b/src/madengine/mad_cli.py @@ -283,14 +283,15 @@ def _process_batch_manifest_entries(batch_data: Dict, manifest_output: str, regi if os.path.exists(manifest_output): with open(manifest_output, 'r') as f: build_manifest = json.load(f) + # Remove top-level registry if present + build_manifest.pop("registry", None) else: # Create a minimal manifest structure build_manifest = { "built_images": {}, "built_models": {}, "context": {}, - "credentials_required": [], - "registry": registry or "" + "credentials_required": [] } # Process each model in the batch manifest @@ -341,7 +342,8 @@ def _process_batch_manifest_entries(batch_data: Dict, manifest_output: str, regi "build_duration": 0, "build_command": f"# Skipped build for {model_name} (build_new=false)", "log_file": f"{model_name}_{model_name}.ubuntu.amd.build.skipped.log", - "registry_image": model_registry_image or f"{model_registry or registry or 'dockerhub'}/{synthetic_image_name}" if model_registry_image or model_registry or registry else "" + "registry_image": model_registry_image or f"{model_registry or registry or 'dockerhub'}/{synthetic_image_name}" if model_registry_image or model_registry or registry else "", + "registry": model_registry or registry or "dockerhub" } # Add to built_models @@ -370,7 +372,8 @@ def _process_batch_manifest_entries(batch_data: Dict, manifest_output: str, regi "build_duration": 0, "build_command": f"# Skipped build for {model_name} (build_new=false)", "log_file": f"{model_name}_{model_name}.ubuntu.amd.build.skipped.log", - "registry_image": model_registry_image or "" + "registry_image": model_registry_image or "", + "registry": model_registry or registry or "dockerhub" } build_manifest["built_models"][synthetic_image_name] = { "name": model_name, diff --git a/src/madengine/tools/distributed_orchestrator.py b/src/madengine/tools/distributed_orchestrator.py index 9234de9c..ffafbd8f 100644 --- a/src/madengine/tools/distributed_orchestrator.py +++ b/src/madengine/tools/distributed_orchestrator.py @@ -209,18 +209,11 @@ def run_phase(self, manifest_file: str = "build_manifest.json", print(f"Loaded manifest with {len(manifest['built_images'])} images") - # Auto-detect registry from manifest if not provided via CLI - if not registry and "registry" in manifest: - manifest_registry = manifest["registry"] - if manifest_registry and manifest_registry.strip(): # Check for non-empty string - registry = manifest_registry - print(f"Auto-detected registry from manifest: {registry}") - else: - print("Manifest registry is empty, will use local images only") - elif registry: + # Registry is now per-image; CLI registry is fallback + if registry: print(f"Using registry from CLI: {registry}") else: - print("No registry specified, will use local images only") + print("No registry specified, will use per-image registry or local images only") # Copy scripts for running self._copy_scripts() @@ -262,31 +255,17 @@ def run_phase(self, manifest_file: str = "build_manifest.json", model_info = manifest["built_models"][image_name] try: print(f"\nRunning model {model_info['name']} with image {image_name}") - - # Handle registry image pulling and tagging according to manifest - if "registry_image" in build_info: - # Registry image exists - pull it and tag as docker_image, then run with docker_image - registry_image = build_info["registry_image"] - docker_image = build_info["docker_image"] - - # Extract registry from the registry_image format - effective_registry = registry - if not effective_registry and registry_image: - registry_parts = registry_image.split('/') - if len(registry_parts) > 1 and '.' in registry_parts[0]: - effective_registry = registry_parts[0] - elif registry_image.startswith('docker.io/') or '/' in registry_image: - effective_registry = "docker.io" - + # Use per-image registry if present, else CLI registry + effective_registry = build_info.get("registry", registry) + registry_image = build_info.get("registry_image") + docker_image = build_info.get("docker_image") + if registry_image: if effective_registry: print(f"Pulling image from registry: {registry_image}") try: - # Ensure all parameters are strings and credentials is properly formatted registry_image_str = str(registry_image) if registry_image else "" docker_image_str = str(docker_image) if docker_image else "" effective_registry_str = str(effective_registry) if effective_registry else "" - - # Pull registry image and tag it as docker_image runner.pull_image(registry_image_str, docker_image_str, effective_registry_str, self.credentials) actual_image = docker_image_str print(f"Successfully pulled and tagged as: {docker_image_str}") @@ -294,7 +273,6 @@ def run_phase(self, manifest_file: str = "build_manifest.json", print(f"Failed to pull from registry, falling back to local image: {e}") actual_image = docker_image else: - # Registry image exists but no valid registry found, try to pull as-is and tag print(f"Attempting to pull registry image as-is: {registry_image}") try: registry_image_str = str(registry_image) if registry_image else "" diff --git a/src/madengine/tools/docker_builder.py b/src/madengine/tools/docker_builder.py index 6c4f22d6..ef3f3471 100644 --- a/src/madengine/tools/docker_builder.py +++ b/src/madengine/tools/docker_builder.py @@ -322,14 +322,26 @@ def export_build_manifest(self, output_file: str = "build_manifest.json", regist Args: output_file: Path to output manifest file - registry: Registry used for building (added to manifest metadata) + registry: Registry used for building (added to each image entry) """ # Extract credentials from models credentials_required = list(set([ model.get("cred", "") for model in self.built_models.values() if model.get("cred", "") != "" ])) - + + # Set registry for each built image + for image_name, build_info in self.built_images.items(): + # If registry is not set in build_info, set it from argument + if registry: + build_info["registry"] = registry + # If registry_image is present, try to parse registry from it if not set + elif "registry_image" in build_info and "registry" not in build_info: + reg_img = build_info["registry_image"] + if reg_img and "/" in reg_img: + reg_part = reg_img.split('/')[0] + build_info["registry"] = reg_part + manifest = { "built_images": self.built_images, "built_models": self.built_models, @@ -342,15 +354,11 @@ def export_build_manifest(self, output_file: str = "build_manifest.json", regist }, "credentials_required": credentials_required } - + # Add multi-node args to context if present if "build_multi_node_args" in self.context.ctx: manifest["context"]["multi_node_args"] = self.context.ctx["build_multi_node_args"] - - # Add registry information to manifest metadata if provided - if registry: - manifest["registry"] = registry - + # Add push failure summary if any pushes failed push_failures = [] for image_name, build_info in self.built_images.items(): @@ -360,13 +368,13 @@ def export_build_manifest(self, output_file: str = "build_manifest.json", regist "intended_registry_image": build_info.get("registry_image"), "error": build_info.get("push_error") }) - + if push_failures: manifest["push_failures"] = push_failures - + with open(output_file, 'w') as f: json.dump(manifest, f, indent=2) - + print(f"Build manifest exported to: {output_file}") if push_failures: print(f"Warning: {len(push_failures)} image(s) failed to push to registry") From c7c6d37a699ea9f96211c2ccbf1a94bee0be5e50 Mon Sep 17 00:00:00 2001 From: Stephen Shao Date: Wed, 23 Jul 2025 12:14:08 -0400 Subject: [PATCH 02/10] correct registry image will be used for each model as intended --- src/madengine/tools/docker_builder.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/madengine/tools/docker_builder.py b/src/madengine/tools/docker_builder.py index ef3f3471..a1035c60 100644 --- a/src/madengine/tools/docker_builder.py +++ b/src/madengine/tools/docker_builder.py @@ -446,7 +446,7 @@ def build_all_models(self, models: typing.List[typing.Dict], model_info, dockerfile, credentials, clean_cache, phase_suffix ) - # Determine registry image name and add to manifest before push operations + # Determine registry image name for push/tag registry_image = None if model_registry_image: registry_image = model_registry_image @@ -454,6 +454,11 @@ def build_all_models(self, models: typing.List[typing.Dict], registry_image = self._determine_registry_image_name( build_info["docker_image"], model_registry, credentials ) + # Always use registry_image from batch_build_metadata if present + if batch_build_metadata and model_info["name"] in batch_build_metadata: + meta = batch_build_metadata[model_info["name"]] + if meta.get("registry_image"): + registry_image = meta["registry_image"] if registry_image: build_info["registry_image"] = registry_image if build_info["docker_image"] in self.built_images: @@ -462,6 +467,7 @@ def build_all_models(self, models: typing.List[typing.Dict], # Now attempt to push to registry if registry is set if model_registry and registry_image: try: + # Use registry_image from batch_build_metadata for push/tag if present actual_registry_image = self.push_image( build_info["docker_image"], model_registry, credentials ) From 74494936be3258bec47af76c2ed373c68869829b Mon Sep 17 00:00:00 2001 From: Stephen Shao Date: Wed, 23 Jul 2025 13:36:15 -0400 Subject: [PATCH 03/10] The push_image function now accepts and uses the explicit registry_image from batch.json for each model. --- src/madengine/tools/docker_builder.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/madengine/tools/docker_builder.py b/src/madengine/tools/docker_builder.py index a1035c60..7fffd4e2 100644 --- a/src/madengine/tools/docker_builder.py +++ b/src/madengine/tools/docker_builder.py @@ -270,7 +270,7 @@ def login_to_registry(self, registry: str, credentials: typing.Dict = None) -> N print(f"Failed to login to registry {registry}: {e}") raise - def push_image(self, docker_image: str, registry: str = None, credentials: typing.Dict = None) -> str: + def push_image(self, docker_image: str, registry: str = None, credentials: typing.Dict = None, explicit_registry_image: str = None) -> str: """Push the built image to a registry. Args: @@ -290,26 +290,33 @@ def push_image(self, docker_image: str, registry: str = None, credentials: typin self.login_to_registry(registry, credentials) # Determine registry image name (this should match what was already determined) - registry_image = self._determine_registry_image_name(docker_image, registry, credentials) + if explicit_registry_image: + registry_image = explicit_registry_image + else: + registry_image = self._determine_registry_image_name(docker_image, registry, credentials) + print(f"[DEBUG] push_image: docker_image='{docker_image}', registry='{registry}', registry_image='{registry_image}'") try: # Tag the image if different from local name if registry_image != docker_image: + print(f"[DEBUG] Tagging image: docker tag {docker_image} {registry_image}") tag_command = f"docker tag {docker_image} {registry_image}" - print(f"šŸ·ļø Tagging image: {tag_command}") self.console.sh(tag_command) - + else: + print(f"[DEBUG] No tag needed, docker_image and registry_image are the same: {docker_image}") + # Push the image + print(f"[DEBUG] Pushing image: docker push {registry_image}") push_command = f"docker push {registry_image}" print(f"\nšŸš€ Starting docker push to registry...") print(f"šŸ“¤ Registry: {registry}") print(f"šŸ·ļø Image: {registry_image}") self.console.sh(push_command) - + print(f"āœ… Successfully pushed image to registry: {registry_image}") print(f"{'='*80}") return registry_image - + except Exception as e: print(f"Failed to push image {docker_image} to registry {registry}: {e}") raise From 7f2c63b9c969a93ac50ff37f93f0c5a3a8ffa012 Mon Sep 17 00:00:00 2001 From: Stephen Shao Date: Wed, 23 Jul 2025 14:14:25 -0400 Subject: [PATCH 04/10] Updated the explicit_registry_image assignment --- src/madengine/tools/docker_builder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/madengine/tools/docker_builder.py b/src/madengine/tools/docker_builder.py index 7fffd4e2..670fd761 100644 --- a/src/madengine/tools/docker_builder.py +++ b/src/madengine/tools/docker_builder.py @@ -473,10 +473,11 @@ def build_all_models(self, models: typing.List[typing.Dict], # Now attempt to push to registry if registry is set if model_registry and registry_image: + explicit_registry_image = registry_image try: # Use registry_image from batch_build_metadata for push/tag if present actual_registry_image = self.push_image( - build_info["docker_image"], model_registry, credentials + build_info["docker_image"], model_registry, credentials, explicit_registry_image ) if actual_registry_image != registry_image: print(f"Warning: Pushed image name {actual_registry_image} differs from intended {registry_image}") From 9f50d043aba43877f86a3eb035ce65a6508deb72 Mon Sep 17 00:00:00 2001 From: Stephen Shao Date: Wed, 23 Jul 2025 14:27:35 -0400 Subject: [PATCH 05/10] Debug the registry info setting --- src/madengine/tools/docker_builder.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/madengine/tools/docker_builder.py b/src/madengine/tools/docker_builder.py index 670fd761..ee7ffc4d 100644 --- a/src/madengine/tools/docker_builder.py +++ b/src/madengine/tools/docker_builder.py @@ -342,12 +342,6 @@ def export_build_manifest(self, output_file: str = "build_manifest.json", regist # If registry is not set in build_info, set it from argument if registry: build_info["registry"] = registry - # If registry_image is present, try to parse registry from it if not set - elif "registry_image" in build_info and "registry" not in build_info: - reg_img = build_info["registry_image"] - if reg_img and "/" in reg_img: - reg_part = reg_img.split('/')[0] - build_info["registry"] = reg_part manifest = { "built_images": self.built_images, From 05f8a26a20be80573dc62829b1b7184db9cd8646 Mon Sep 17 00:00:00 2001 From: Stephen Shao Date: Wed, 23 Jul 2025 16:12:32 -0400 Subject: [PATCH 06/10] Updated the function of export build manifest --- src/madengine/tools/distributed_orchestrator.py | 2 +- src/madengine/tools/docker_builder.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/madengine/tools/distributed_orchestrator.py b/src/madengine/tools/distributed_orchestrator.py index ffafbd8f..e7a62ffa 100644 --- a/src/madengine/tools/distributed_orchestrator.py +++ b/src/madengine/tools/distributed_orchestrator.py @@ -144,7 +144,7 @@ def build_phase(self, registry: str = None, clean_cache: bool = False, ) # Export build manifest with registry information - builder.export_build_manifest(manifest_output, registry) + builder.export_build_manifest(manifest_output, registry, batch_build_metadata) print("=" * 60) print("BUILD PHASE COMPLETED") diff --git a/src/madengine/tools/docker_builder.py b/src/madengine/tools/docker_builder.py index ee7ffc4d..6ea0c39f 100644 --- a/src/madengine/tools/docker_builder.py +++ b/src/madengine/tools/docker_builder.py @@ -320,16 +320,17 @@ def push_image(self, docker_image: str, registry: str = None, credentials: typin except Exception as e: print(f"Failed to push image {docker_image} to registry {registry}: {e}") raise - - def export_build_manifest(self, output_file: str = "build_manifest.json", registry: str = None) -> None: + + def export_build_manifest(self, output_file: str = "build_manifest.json", registry: str = None, batch_build_metadata: typing.Optional[dict] = None) -> None: """Export enhanced build information to a manifest file. This creates a comprehensive build manifest that includes all necessary information for deployment, reducing the need for separate execution configs. - + Args: output_file: Path to output manifest file registry: Registry used for building (added to each image entry) + batch_build_metadata: Optional metadata for batch builds """ # Extract credentials from models credentials_required = list(set([ @@ -343,6 +344,9 @@ def export_build_manifest(self, output_file: str = "build_manifest.json", regist if registry: build_info["registry"] = registry + if batch_build_metadata and image_name in batch_build_metadata: + build_info["registry"] = batch_build_metadata[image_name].get("registry") + manifest = { "built_images": self.built_images, "built_models": self.built_models, From 8f8dc880f8a0bad1178946643c5f6f759ef12534 Mon Sep 17 00:00:00 2001 From: Stephen Shao Date: Wed, 23 Jul 2025 22:08:34 -0400 Subject: [PATCH 07/10] Add verbose for debugging --- src/madengine/mad_cli.py | 13 ++++++++++++- src/madengine/tools/distributed_orchestrator.py | 12 ++++++++++-- src/madengine/tools/docker_builder.py | 3 +++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/madengine/mad_cli.py b/src/madengine/mad_cli.py index 3a578908..577da662 100644 --- a/src/madengine/mad_cli.py +++ b/src/madengine/mad_cli.py @@ -473,9 +473,17 @@ def build( batch_data = None effective_tags = tags batch_build_metadata = None + + # There are 2 scenarios for batch builds and single builds + # - Batch builds: Use the batch manifest to determine which models to build + # - Single builds: Use the tags directly if batch_manifest: + # Process the batch manifest + if verbose: console.print(f"[DEBUG] Processing batch manifest: {batch_manifest}") try: batch_data = process_batch_manifest(batch_manifest) + if verbose: console.print(f"[DEBUG] batch_data: {batch_data}") + effective_tags = batch_data["build_tags"] # Build a mapping of model_name -> registry_image/registry for build_new models batch_build_metadata = {} @@ -485,6 +493,8 @@ def build( "registry_image": model.get("registry_image"), "registry": model.get("registry") } + if verbose: console.print(f"[DEBUG] batch_build_metadata: {batch_build_metadata}") + console.print(Panel( f"ļæ½ [bold cyan]Batch Build Mode[/bold cyan]\n" f"Input manifest: [yellow]{batch_manifest}[/yellow]\n" @@ -541,12 +551,13 @@ def build( orchestrator = DistributedOrchestrator(args, build_only_mode=True) progress.update(task, description="Building models...") - # Pass batch_build_metadata to build_phase if present + # Prepare build phase arguments build_phase_kwargs = dict( registry=registry, clean_cache=clean_docker_cache, manifest_output=manifest_output ) + # Pass batch_build_metadata to build_phase if present if batch_build_metadata: build_phase_kwargs["batch_build_metadata"] = batch_build_metadata diff --git a/src/madengine/tools/distributed_orchestrator.py b/src/madengine/tools/distributed_orchestrator.py index e7a62ffa..c7b86ed5 100644 --- a/src/madengine/tools/distributed_orchestrator.py +++ b/src/madengine/tools/distributed_orchestrator.py @@ -5,6 +5,8 @@ This module provides orchestration capabilities for distributed execution scenarios like Ansible or Kubernetes, where Docker image building and container execution are separated across different nodes. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. """ import os @@ -87,7 +89,8 @@ def __init__(self, args, build_only_mode: bool = False): print(f"Docker Hub credentials: {self.credentials['dockerhub']}") def build_phase(self, registry: str = None, clean_cache: bool = False, - manifest_output: str = "build_manifest.json", batch_build_metadata: typing.Optional[dict] = None) -> typing.Dict: + manifest_output: str = "build_manifest.json", + batch_build_metadata: typing.Optional[dict] = None) -> typing.Dict: """Execute the build phase - build all Docker images. This method supports both build-only mode (for dedicated build nodes) @@ -109,15 +112,20 @@ def build_phase(self, registry: str = None, clean_cache: bool = False, print("(Build-only mode - no GPU detection)") print("=" * 60) - print(f"Building models with args {self.args}") + # Print the arguments as a dictionary for better readability + print(f"Building models with args: {vars(self.args) if hasattr(self.args, '__dict__') else self.args}") # Discover models + print("=" * 60) + print("DISCOVERING MODELS") discover_models = DiscoverModels(args=self.args) models = discover_models.run() print(f"Discovered {len(models)} models to build") # Copy scripts for building + print("=" * 60) + print("COPYING SCRIPTS") self._copy_scripts() # Validate build context for build-only mode diff --git a/src/madengine/tools/docker_builder.py b/src/madengine/tools/docker_builder.py index 6ea0c39f..5c2ed641 100644 --- a/src/madengine/tools/docker_builder.py +++ b/src/madengine/tools/docker_builder.py @@ -338,6 +338,8 @@ def export_build_manifest(self, output_file: str = "build_manifest.json", regist if model.get("cred", "") != "" ])) + print(f"[DEBUG] batch_build_metadata: {batch_build_metadata}") + # Set registry for each built image for image_name, build_info in self.built_images.items(): # If registry is not set in build_info, set it from argument @@ -345,6 +347,7 @@ def export_build_manifest(self, output_file: str = "build_manifest.json", regist build_info["registry"] = registry if batch_build_metadata and image_name in batch_build_metadata: + print(f"[DEBUG] Overriding registry for {image_name} from batch_build_metadata") build_info["registry"] = batch_build_metadata[image_name].get("registry") manifest = { From de6b49c1aa0e5834a4821bd9af979557b7f9e41a Mon Sep 17 00:00:00 2001 From: Stephen Shao Date: Wed, 23 Jul 2025 22:55:12 -0400 Subject: [PATCH 08/10] Debug the export build manifest --- src/madengine/tools/docker_builder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/madengine/tools/docker_builder.py b/src/madengine/tools/docker_builder.py index 5c2ed641..2bda0966 100644 --- a/src/madengine/tools/docker_builder.py +++ b/src/madengine/tools/docker_builder.py @@ -339,6 +339,7 @@ def export_build_manifest(self, output_file: str = "build_manifest.json", regist ])) print(f"[DEBUG] batch_build_metadata: {batch_build_metadata}") + print(f"[DEBUG] built_images: {self.built_images}") # Set registry for each built image for image_name, build_info in self.built_images.items(): From f1a39058f9b3ebf16945158533e4d6c4c3c8f595 Mon Sep 17 00:00:00 2001 From: Stephen Shao Date: Wed, 23 Jul 2025 23:20:40 -0400 Subject: [PATCH 09/10] Debug the registry extract from batch build metadata --- src/madengine/tools/docker_builder.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/madengine/tools/docker_builder.py b/src/madengine/tools/docker_builder.py index 2bda0966..1df9cba1 100644 --- a/src/madengine/tools/docker_builder.py +++ b/src/madengine/tools/docker_builder.py @@ -347,9 +347,12 @@ def export_build_manifest(self, output_file: str = "build_manifest.json", regist if registry: build_info["registry"] = registry - if batch_build_metadata and image_name in batch_build_metadata: - print(f"[DEBUG] Overriding registry for {image_name} from batch_build_metadata") - build_info["registry"] = batch_build_metadata[image_name].get("registry") + docker_file = build_info.get("docker_file", "") + truncated_docker_file = docker_file.split("/")[-1].split(".Dockerfile")[0] + model_name = image_name.split("ci-")[1].split(truncated_docker_file)[0] + if batch_build_metadata and model_name in batch_build_metadata: + print(f"[DEBUG] Overriding registry for {model_name} from batch_build_metadata") + build_info["registry"] = batch_build_metadata[model_name].get("registry") manifest = { "built_images": self.built_images, From d412956520d74118684c4b4733a7ded3f9ad2a55 Mon Sep 17 00:00:00 2001 From: Stephen Shao Date: Wed, 23 Jul 2025 23:33:17 -0400 Subject: [PATCH 10/10] Debug the exaction --- src/madengine/tools/docker_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/madengine/tools/docker_builder.py b/src/madengine/tools/docker_builder.py index 1df9cba1..2945036c 100644 --- a/src/madengine/tools/docker_builder.py +++ b/src/madengine/tools/docker_builder.py @@ -347,9 +347,9 @@ def export_build_manifest(self, output_file: str = "build_manifest.json", regist if registry: build_info["registry"] = registry - docker_file = build_info.get("docker_file", "") + docker_file = build_info.get("dockerfile", "") truncated_docker_file = docker_file.split("/")[-1].split(".Dockerfile")[0] - model_name = image_name.split("ci-")[1].split(truncated_docker_file)[0] + model_name = image_name.split("ci-")[1].split(truncated_docker_file)[0].rstrip("_") if batch_build_metadata and model_name in batch_build_metadata: print(f"[DEBUG] Overriding registry for {model_name} from batch_build_metadata") build_info["registry"] = batch_build_metadata[model_name].get("registry")