From cac117e6c7856b132c06eab926508e36df64e80b Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Fri, 18 Feb 2022 11:43:33 +0800 Subject: [PATCH 1/8] [DLMED] add ReferenceResolver Signed-off-by: Nic Ma --- docs/source/apps.rst | 3 + monai/apps/__init__.py | 2 +- monai/apps/manifest/__init__.py | 1 + monai/apps/manifest/reference_resolver.py | 244 ++++++++++++++++++++++ tests/test_reference_resolver.py | 126 +++++++++++ 5 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 monai/apps/manifest/reference_resolver.py create mode 100644 tests/test_reference_resolver.py diff --git a/docs/source/apps.rst b/docs/source/apps.rst index 6535ad82b7..4b1cdc6f43 100644 --- a/docs/source/apps.rst +++ b/docs/source/apps.rst @@ -44,6 +44,9 @@ Model Manifest .. autoclass:: ConfigItem :members: +.. autoclass:: ReferenceResolver + :members: + `Utilities` ----------- diff --git a/monai/apps/__init__.py b/monai/apps/__init__.py index df085bddea..0f233bc3ef 100644 --- a/monai/apps/__init__.py +++ b/monai/apps/__init__.py @@ -10,6 +10,6 @@ # limitations under the License. from .datasets import CrossValidation, DecathlonDataset, MedNISTDataset -from .manifest import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem +from .manifest import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, ReferenceResolver from .mmars import MODEL_DESC, RemoteMMARKeys, download_mmar, get_model_spec, load_from_mmar from .utils import SUPPORTED_HASH_TYPES, check_hash, download_and_extract, download_url, extractall, get_logger, logger diff --git a/monai/apps/manifest/__init__.py b/monai/apps/manifest/__init__.py index d8919a5249..79c4376d5c 100644 --- a/monai/apps/manifest/__init__.py +++ b/monai/apps/manifest/__init__.py @@ -10,3 +10,4 @@ # limitations under the License. from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem +from .reference_resolver import ReferenceResolver diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py new file mode 100644 index 0000000000..6e20d0dc7a --- /dev/null +++ b/monai/apps/manifest/reference_resolver.py @@ -0,0 +1,244 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, List, Optional, Union +import warnings + +from monai.apps.manifest.config_item import ConfigComponent, ConfigExpression, ConfigItem + + +class ReferenceResolver: + """ + Utility class to resolve the references between config items. + + Args: + components: config components to resolve, if None, can also `add()` component in runtime. + + """ + + def __init__(self, items: Optional[Dict[str, ConfigItem]] = None): + self.items = {} if items is None else items + self.resolved_content = {} + + def add(self, item: ConfigItem): + """ + Add a config item to the resolution graph. + + Args: + item: a config item to resolve. + + """ + id = item.get_id() + if id in self.items: + warnings.warn(f"id '{id}' is already added.") + return + self.items[id] = item + + def resolve_one_item(self, id: str, waiting_list: Optional[List[str]] = None): + """ + Resolve one item with specified id name. + If has unresolved references, recursively resolve the references first. + + Args: + id: id name of expected item to resolve. + waiting_list: list of items wait to resolve references. it's used to detect circular references. + when resolving references like: `{"name": "A", "dep": "@B"}` and `{"name": "B", "dep": "@A"}`. + + """ + if waiting_list is None: + waiting_list = [] + waiting_list.append(id) + item = self.items.get(id) + item_config = item.get_config() + ref_ids = self.find_refs_in_config(config=item_config, id=id) + + # if current item has reference already in the waiting list, that's circular references + for d in ref_ids: + if d in waiting_list: + raise ValueError(f"detected circular references for id='{d}' in the config content.") + + if len(ref_ids) > 0: + # # check whether the component has any unresolved deps + for ref_id in ref_ids: + if ref_id not in self.resolved_content: + # this reffring component is not resolved + if ref_id not in self.items: + raise RuntimeError(f"the referring item `{ref_id}` is not defined in config.") + # resolve the reference first + self.resolve_one_item(id=ref_id, waiting_list=waiting_list) + + # all references are resolved + new_config = self.resolve_config_with_refs(config=item_config, id=id, refs=self.resolved_content) + item.update_config(config=new_config) + if isinstance(item, ConfigComponent): + self.resolved_content[id] = item.instantiate() + elif isinstance(item, ConfigExpression): + self.resolved_content[id] = item.evaluate(locals={"refs": self.resolved_content}) + else: + self.resolved_content[id] = new_config + + def resolve_all(self): + """ + Resolve the references for all the config items. + + """ + for k in self.items.keys(): + self.resolve_one_item(id=k) + + def get_resolved_content(self, id: str): + """ + Get the resolved content with specified id name. + If not resolved, try to resolve it first. + + Args: + id: id name of the expected item. + + """ + if id not in self.resolved_content: + self.resolve_one_item(id=id) + return self.resolved_content.get(id) + + def get_item(self, id: str, resolve: bool = False): + """ + Get the config item with specified id name, then can be used for lazy instantiation. + If `resolve=True`, try to resolve it first. + + Args: + id: id name of the expected config item. + + """ + if resolve and id not in self.resolved_content: + self.resolve_one_item(id=id) + return self.items.get(id) + + @staticmethod + def match_refs_pattern(value: str) -> List[str]: + """ + Match regular expression for the input string to find the references. + The reference part starts with "@", like: "@XXX#YYY#ZZZ". + + Args: + value: input value to match regular expression. + + """ + refs: List[str] = [] + # regular expression pattern to match "@XXX" or "@XXX#YYY" + result = re.compile(r"@\w*[\#\w]*").findall(value) + for item in result: + if ConfigExpression.is_expression(value) or value == item: + # only check when string starts with "$" or the whole content is "@XXX" + ref_obj_id = item[1:] + if ref_obj_id not in refs: + refs.append(ref_obj_id) + return refs + + @staticmethod + def resolve_refs_pattern(value: str, refs: Dict) -> str: + """ + Match regular expression for the input string to update content with the references. + The reference part starts with "@", like: "@XXX#YYY#ZZZ". + References dictionary must contain the referring IDs as keys. + + Args: + value: input value to match regular expression. + refs: all the referring components with ids as keys, default to `None`. + + """ + # regular expression pattern to match "@XXX" or "@XXX#YYY" + result = re.compile(r"@\w*[\#\w]*").findall(value) + for item in result: + ref_id = item[1:] + if ConfigExpression.is_expression(value): + # replace with local code and execute later + value = value.replace(item, f"refs['{ref_id}']") + elif value == item: + if ref_id not in refs: + raise KeyError(f"can not find expected ID '{ref_id}' in the references.") + value = refs[ref_id] + return value + + @staticmethod + def find_refs_in_config( + config: Union[Dict, List, str], + id: Optional[str] = None, + refs: Optional[List[str]] = None, + ) -> List[str]: + """ + Recursively search all the content of input config item to get the ids of references. + References mean (1) referring to the ID of other item, can be extracted by `match_fn`, for example: + `{"lr": "$@epoch / 100"}` with "@" mark, the referring IDs: `["epoch"]`. (2) if sub-item in the config + is instantiable, treat it as reference because must instantiate it before resolving current config. + For `dict` and `list`, recursively check the sub-items. + + Args: + config: input config content to search. + id: ID name for the input config, default to `None`. + refs: list of the ID name of existing references, default to `None`. + + """ + refs_: List[str] = [] if refs is None else refs + if isinstance(config, str): + refs_ += ReferenceResolver.match_refs_pattern(value=config) + + if isinstance(config, list): + for i, v in enumerate(config): + sub_id = f"{id}#{i}" if id is not None else f"{i}" + if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): + refs_.append(sub_id) + refs_ = ReferenceResolver.find_refs_in_config(v, sub_id, refs_) + if isinstance(config, dict): + for k, v in config.items(): + sub_id = f"{id}#{k}" if id is not None else f"{k}" + if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): + refs_.append(sub_id) + refs_ = ReferenceResolver.find_refs_in_config(v, sub_id, refs_) + return refs_ + + @staticmethod + def resolve_config_with_refs( + config: Union[Dict, List, str], + id: Optional[str] = None, + refs: Optional[Dict] = None, + ): + """ + With all the references in `refs`, resolve the config content with them and return new config. + + Args: + config: input config content to resolve. + id: ID name for the input config, default to `None`. + refs: all the referring components with ids, default to `None`. + + """ + refs_: Dict = {} if refs is None else refs + if isinstance(config, str): + config = ReferenceResolver.resolve_refs_pattern(config, refs) + if isinstance(config, list): + # all the items in the list should be replaced with the references + ret_list: List = [] + for i, v in enumerate(config): + sub_id = f"{id}#{i}" if id is not None else f"{i}" + if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): + ret_list.append(refs_[sub_id]) + else: + ret_list.append(ReferenceResolver.resolve_config_with_refs(v, sub_id, refs_)) + return ret_list + if isinstance(config, dict): + # all the items in the dict should be replaced with the references + ret_dict: Dict = {} + for k, v in config.items(): + sub_id = f"{id}#{k}" if id is not None else f"{k}" + if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): + ret_dict[k] = refs_[sub_id] + else: + ret_dict[k] = ReferenceResolver.resolve_config_with_refs(v, sub_id, refs_) + return ret_dict + return config diff --git a/tests/test_reference_resolver.py b/tests/test_reference_resolver.py new file mode 100644 index 0000000000..f6a8dc416c --- /dev/null +++ b/tests/test_reference_resolver.py @@ -0,0 +1,126 @@ +# Copyright 2020 - 2021 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from monai.apps.manifest.config_item import ComponentLocator, ConfigExpression, ConfigItem + +import torch +from parameterized import parameterized + +import monai +from monai.apps import ConfigComponent, ReferenceResolver +from monai.data import DataLoader +from monai.transforms import LoadImaged, RandTorchVisiond +from monai.utils import optional_import + +_, has_tv = optional_import("torchvision") + +# test instance with no dependencies +TEST_CASE_1 = [ + { + # all the recursively parsed config items + "transform#1": {"": "LoadImaged", "": {"keys": ["image"]}}, + "transform#1#": "LoadImaged", + "transform#1#": {"keys": ["image"]}, + "transform#1##keys": ["image"], + "transform#1##keys#0": "image", + }, + "transform#1", + LoadImaged, +] +# test depends on other component and executable code +TEST_CASE_2 = [ + { + # some the recursively parsed config items + "dataloader": { + "": "DataLoader", + "": {"dataset": "@dataset", "collate_fn": "$monai.data.list_data_collate"}, + }, + "dataset": {"": "Dataset", "": {"data": [1, 2]}}, + "dataloader#": "DataLoader", + "dataloader#": {"dataset": "@dataset", "collate_fn": "$monai.data.list_data_collate"}, + "dataloader##dataset": "@dataset", + "dataloader##collate_fn": "$monai.data.list_data_collate", + "dataset#": "Dataset", + "dataset#": {"data": [1, 2]}, + "dataset##data": [1, 2], + "dataset##data#0": 1, + "dataset##data#1": 2, + }, + "dataloader", + DataLoader, +] +# test config has key `name` +TEST_CASE_3 = [ + { + # all the recursively parsed config items + "transform#1": { + "": "RandTorchVisiond", + "": {"keys": "image", "name": "ColorJitter", "brightness": 0.25}, + }, + "transform#1#": "RandTorchVisiond", + "transform#1#": {"keys": "image", "name": "ColorJitter", "brightness": 0.25}, + "transform#1##keys": "image", + "transform#1##name": "ColorJitter", + "transform#1##brightness": 0.25, + }, + "transform#1", + RandTorchVisiond, +] + + +class TestReferenceResolver(unittest.TestCase): + @parameterized.expand([TEST_CASE_1, TEST_CASE_2] + ([TEST_CASE_3] if has_tv else [])) + def test_resolve(self, configs, expected_id, output_type): + locator = ComponentLocator() + resolver = ReferenceResolver() + # add items to resolver + for k, v in configs.items(): + if ConfigComponent.is_instantiable(v): + resolver.add(ConfigComponent(config=v, id=k, locator=locator)) + elif ConfigExpression.is_expression(v): + resolver.add(ConfigExpression(config=v, id=k, globals={"monai": monai, "torch": torch})) + else: + resolver.add(ConfigItem(config=v, id=k)) + + result = resolver.get_resolved_content(expected_id) + self.assertTrue(isinstance(result, output_type)) + + # test resolve all + resolver.resolved_content = {} + resolver.resolve_all() + result = resolver.get_resolved_content(expected_id) + self.assertTrue(isinstance(result, output_type)) + + # test lazy instantiation + item = resolver.get_item(expected_id, resolve=True) + config = item.get_config() + config[""] = False + item.update_config(config=config) + if isinstance(item, ConfigComponent): + result = item.instantiate() + else: + result = item.get_config() + self.assertTrue(isinstance(result, output_type)) + + def test_circular_references(self): + locator = ComponentLocator() + resolver = ReferenceResolver() + configs = {"A": "@B", "B": "@C", "C": "@A"} + for k, v in configs.items(): + resolver.add(ConfigComponent(config=v, id=k, locator=locator)) + for k in ["A", "B", "C"]: + with self.assertRaises(ValueError): + resolver.get_resolved_content(k) + + +if __name__ == "__main__": + unittest.main() From 9bdfc8f220e915b5f9add1d6f4d2ed7c4772fc96 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Fri, 18 Feb 2022 16:56:23 +0800 Subject: [PATCH 2/8] [DLMED] refine docs Signed-off-by: Nic Ma --- monai/apps/manifest/reference_resolver.py | 159 ++++++++++++---------- tests/test_reference_resolver.py | 12 +- 2 files changed, 92 insertions(+), 79 deletions(-) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 6e20d0dc7a..cce25fd8a6 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -10,28 +10,45 @@ # limitations under the License. import re -from typing import Dict, List, Optional, Union import warnings +from typing import Dict, List, Optional, Sequence, Set, Union from monai.apps.manifest.config_item import ConfigComponent, ConfigExpression, ConfigItem class ReferenceResolver: """ - Utility class to resolve the references between config items. + Utility class to manage config items and resolve the references between them. + + There are 3 kinds of `references` in the config content: + - The IDs of other config items used as "@XXX" in this config item, for example: + config item with ID="A" is a list `[1, 2, 3]`, another config item "B" can be `"args": {"input_list": "@A"}`. + Then it means A is one reference of B. + - If sub-item in the config is `instantiable`, treat it as reference because must instantiate the sub-item + before using this config. + - If sub-item in the config is `expression`, also treat it as reference because must evaluate the expression + before using this config. + + The typical usage of the APIs: + - Automatically search the content of specified config item and find out all the references. + - Recursively resolve the references of this config item and update them in the config content. + - If this config item is instantiable, try to instantiate it and save the instance in the `resolved_content`. + If this config item is an expression, try to evaluate it and save the result in the `resolved_content`. + Otherwise, save the updated config content in the `resolved_content`. Args: - components: config components to resolve, if None, can also `add()` component in runtime. + items: config items to resolve, if None, can also `add()` component in runtime. """ - def __init__(self, items: Optional[Dict[str, ConfigItem]] = None): - self.items = {} if items is None else items + def __init__(self, items: Optional[Sequence[ConfigItem]] = None): + # save the items in a dictionary with the `id` as key + self.items = {} if items is None else {i.get_id(): i for i in items} self.resolved_content = {} - def add(self, item: ConfigItem): + def add_item(self, item: ConfigItem): """ - Add a config item to the resolution graph. + Add a config item to the resolver. Args: item: a config item to resolve. @@ -43,22 +60,39 @@ def add(self, item: ConfigItem): return self.items[id] = item - def resolve_one_item(self, id: str, waiting_list: Optional[List[str]] = None): + def get_item(self, id: str, resolve: bool = False): """ - Resolve one item with specified id name. - If has unresolved references, recursively resolve the references first. + Get the config item with specified id name, then can be used for lazy instantiation, etc. + If `resolve=True` and the item is not resolved, try to resolve it first, then it will have + no reference in the config content. Args: - id: id name of expected item to resolve. - waiting_list: list of items wait to resolve references. it's used to detect circular references. - when resolving references like: `{"name": "A", "dep": "@B"}` and `{"name": "B", "dep": "@A"}`. + id: id name of the expected config item. """ - if waiting_list is None: - waiting_list = [] - waiting_list.append(id) - item = self.items.get(id) + if resolve and id not in self.resolved_content: + self._resolve_one_item(id=id) + return self.items.get(id) + + def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None): + """ + Resolve one config item with specified id name, save the resolved result in `resolved_content`. + If it has unresolved references, recursively resolve the referring items first. + + Args: + id: id name of expected config item to resolve. + waiting_list: list of the ids of items wait to resolve references. + it's used to detect circular references when resolving references like: + `{"name": "A", "dep": "@B"}` and `{"name": "B", "dep": "@A"}`. + + """ + item = self.items[id] # if invalid id name, raise KeyError item_config = item.get_config() + + if waiting_list is None: + waiting_list = set() + waiting_list.add(id) + ref_ids = self.find_refs_in_config(config=item_config, id=id) # if current item has reference already in the waiting list, that's circular references @@ -67,18 +101,19 @@ def resolve_one_item(self, id: str, waiting_list: Optional[List[str]] = None): raise ValueError(f"detected circular references for id='{d}' in the config content.") if len(ref_ids) > 0: - # # check whether the component has any unresolved deps + # # check whether the component has any unresolved references for ref_id in ref_ids: if ref_id not in self.resolved_content: - # this reffring component is not resolved + # this referring item is not resolved if ref_id not in self.items: - raise RuntimeError(f"the referring item `{ref_id}` is not defined in config.") - # resolve the reference first - self.resolve_one_item(id=ref_id, waiting_list=waiting_list) + raise ValueError(f"the referring item `{ref_id}` is not defined in config.") + # recursively resolve the reference first + self._resolve_one_item(id=ref_id, waiting_list=waiting_list) - # all references are resolved - new_config = self.resolve_config_with_refs(config=item_config, id=id, refs=self.resolved_content) + # all references are resolved, then try to resolve current config item + new_config = self.update_config_with_refs(config=item_config, id=id, refs=self.resolved_content) item.update_config(config=new_config) + # save the resolved result into `resolved_content` to recursively resolve others if isinstance(item, ConfigComponent): self.resolved_content[id] = item.instantiate() elif isinstance(item, ConfigExpression): @@ -91,34 +126,21 @@ def resolve_all(self): Resolve the references for all the config items. """ - for k in self.items.keys(): - self.resolve_one_item(id=k) + for k in self.items: + self._resolve_one_item(id=k) def get_resolved_content(self, id: str): """ Get the resolved content with specified id name. - If not resolved, try to resolve it first. + If not resolved yet, try to resolve it first. Args: id: id name of the expected item. """ if id not in self.resolved_content: - self.resolve_one_item(id=id) - return self.resolved_content.get(id) - - def get_item(self, id: str, resolve: bool = False): - """ - Get the config item with specified id name, then can be used for lazy instantiation. - If `resolve=True`, try to resolve it first. - - Args: - id: id name of the expected config item. - - """ - if resolve and id not in self.resolved_content: - self.resolve_one_item(id=id) - return self.items.get(id) + self._resolve_one_item(id=id) + return self.resolved_content[id] @staticmethod def match_refs_pattern(value: str) -> List[str]: @@ -142,7 +164,7 @@ def match_refs_pattern(value: str) -> List[str]: return refs @staticmethod - def resolve_refs_pattern(value: str, refs: Dict) -> str: + def update_refs_pattern(value: str, refs: Dict) -> str: """ Match regular expression for the input string to update content with the references. The reference part starts with "@", like: "@XXX#YYY#ZZZ". @@ -157,46 +179,40 @@ def resolve_refs_pattern(value: str, refs: Dict) -> str: result = re.compile(r"@\w*[\#\w]*").findall(value) for item in result: ref_id = item[1:] + if ref_id not in refs: + raise KeyError(f"can not find expected ID '{ref_id}' in the references.") + if ConfigExpression.is_expression(value): - # replace with local code and execute later + # replace with local code, will be used in the `evaluate` logic with `locals={"refs": ...}` value = value.replace(item, f"refs['{ref_id}']") elif value == item: - if ref_id not in refs: - raise KeyError(f"can not find expected ID '{ref_id}' in the references.") + # the whole content is "@XXX", it will avoid the case that regular string contains "@" value = refs[ref_id] return value @staticmethod def find_refs_in_config( - config: Union[Dict, List, str], - id: Optional[str] = None, - refs: Optional[List[str]] = None, + config: Union[Dict, List, str], id: Optional[str] = None, refs: Optional[List[str]] = None ) -> List[str]: """ Recursively search all the content of input config item to get the ids of references. - References mean (1) referring to the ID of other item, can be extracted by `match_fn`, for example: - `{"lr": "$@epoch / 100"}` with "@" mark, the referring IDs: `["epoch"]`. (2) if sub-item in the config - is instantiable, treat it as reference because must instantiate it before resolving current config. + References mean: the IDs of other config items used as "@XXX" in this config item, or the + sub-item in the config is `instantiable`, or the sub-item in the config is `expression`. For `dict` and `list`, recursively check the sub-items. Args: config: input config content to search. - id: ID name for the input config, default to `None`. - refs: list of the ID name of existing references, default to `None`. + id: ID name for the input config item, default to `None`. + refs: list of the ID name of found references, default to `None`. """ refs_: List[str] = [] if refs is None else refs if isinstance(config, str): refs_ += ReferenceResolver.match_refs_pattern(value=config) - if isinstance(config, list): - for i, v in enumerate(config): - sub_id = f"{id}#{i}" if id is not None else f"{i}" - if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): - refs_.append(sub_id) - refs_ = ReferenceResolver.find_refs_in_config(v, sub_id, refs_) - if isinstance(config, dict): - for k, v in config.items(): + if isinstance(config, (list, dict)): + subs = enumerate(config) if isinstance(config, list) else config.items() + for k, v in subs: sub_id = f"{id}#{k}" if id is not None else f"{k}" if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): refs_.append(sub_id) @@ -204,23 +220,20 @@ def find_refs_in_config( return refs_ @staticmethod - def resolve_config_with_refs( - config: Union[Dict, List, str], - id: Optional[str] = None, - refs: Optional[Dict] = None, - ): + def update_config_with_refs(config: Union[Dict, List, str], id: Optional[str] = None, refs: Optional[Dict] = None): """ - With all the references in `refs`, resolve the config content with them and return new config. + With all the references in `refs`, update the input config content with references + and return the new config. Args: - config: input config content to resolve. + config: input config content to update. id: ID name for the input config, default to `None`. - refs: all the referring components with ids, default to `None`. + refs: all the referring content with ids, default to `None`. """ refs_: Dict = {} if refs is None else refs if isinstance(config, str): - config = ReferenceResolver.resolve_refs_pattern(config, refs) + config = ReferenceResolver.update_refs_pattern(config, refs) if isinstance(config, list): # all the items in the list should be replaced with the references ret_list: List = [] @@ -229,7 +242,7 @@ def resolve_config_with_refs( if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): ret_list.append(refs_[sub_id]) else: - ret_list.append(ReferenceResolver.resolve_config_with_refs(v, sub_id, refs_)) + ret_list.append(ReferenceResolver.update_config_with_refs(v, sub_id, refs_)) return ret_list if isinstance(config, dict): # all the items in the dict should be replaced with the references @@ -239,6 +252,6 @@ def resolve_config_with_refs( if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): ret_dict[k] = refs_[sub_id] else: - ret_dict[k] = ReferenceResolver.resolve_config_with_refs(v, sub_id, refs_) + ret_dict[k] = ReferenceResolver.update_config_with_refs(v, sub_id, refs_) return ret_dict return config diff --git a/tests/test_reference_resolver.py b/tests/test_reference_resolver.py index f6a8dc416c..ac0dcec75a 100644 --- a/tests/test_reference_resolver.py +++ b/tests/test_reference_resolver.py @@ -10,13 +10,13 @@ # limitations under the License. import unittest -from monai.apps.manifest.config_item import ComponentLocator, ConfigExpression, ConfigItem import torch from parameterized import parameterized import monai from monai.apps import ConfigComponent, ReferenceResolver +from monai.apps.manifest.config_item import ComponentLocator, ConfigExpression, ConfigItem from monai.data import DataLoader from monai.transforms import LoadImaged, RandTorchVisiond from monai.utils import optional_import @@ -85,17 +85,17 @@ def test_resolve(self, configs, expected_id, output_type): # add items to resolver for k, v in configs.items(): if ConfigComponent.is_instantiable(v): - resolver.add(ConfigComponent(config=v, id=k, locator=locator)) + resolver.add_item(ConfigComponent(config=v, id=k, locator=locator)) elif ConfigExpression.is_expression(v): - resolver.add(ConfigExpression(config=v, id=k, globals={"monai": monai, "torch": torch})) + resolver.add_item(ConfigExpression(config=v, id=k, globals={"monai": monai, "torch": torch})) else: - resolver.add(ConfigItem(config=v, id=k)) + resolver.add_item(ConfigItem(config=v, id=k)) result = resolver.get_resolved_content(expected_id) self.assertTrue(isinstance(result, output_type)) # test resolve all - resolver.resolved_content = {} + resolver.resolved_content = {} # clear content resolver.resolve_all() result = resolver.get_resolved_content(expected_id) self.assertTrue(isinstance(result, output_type)) @@ -116,7 +116,7 @@ def test_circular_references(self): resolver = ReferenceResolver() configs = {"A": "@B", "B": "@C", "C": "@A"} for k, v in configs.items(): - resolver.add(ConfigComponent(config=v, id=k, locator=locator)) + resolver.add_item(ConfigComponent(config=v, id=k, locator=locator)) for k in ["A", "B", "C"]: with self.assertRaises(ValueError): resolver.get_resolved_content(k) From 068d463addb397aafc871e0cf4472b3c96ea0f31 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Fri, 18 Feb 2022 17:05:37 +0800 Subject: [PATCH 3/8] [DLMED] fix flake8 Signed-off-by: Nic Ma --- monai/apps/manifest/reference_resolver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index cce25fd8a6..55c030675d 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -11,7 +11,7 @@ import re import warnings -from typing import Dict, List, Optional, Sequence, Set, Union +from typing import Any, Dict, List, Optional, Sequence, Set, Union from monai.apps.manifest.config_item import ConfigComponent, ConfigExpression, ConfigItem @@ -44,7 +44,7 @@ class ReferenceResolver: def __init__(self, items: Optional[Sequence[ConfigItem]] = None): # save the items in a dictionary with the `id` as key self.items = {} if items is None else {i.get_id(): i for i in items} - self.resolved_content = {} + self.resolved_content: Dict[str, Any] = {} def add_item(self, item: ConfigItem): """ @@ -233,7 +233,7 @@ def update_config_with_refs(config: Union[Dict, List, str], id: Optional[str] = """ refs_: Dict = {} if refs is None else refs if isinstance(config, str): - config = ReferenceResolver.update_refs_pattern(config, refs) + config = ReferenceResolver.update_refs_pattern(config, refs_) if isinstance(config, list): # all the items in the list should be replaced with the references ret_list: List = [] From 4c04f023860d30a9ed519ca3d976eb7bfc4a039e Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Fri, 18 Feb 2022 17:10:46 +0800 Subject: [PATCH 4/8] [DLMED] simplify Instantiable interface Signed-off-by: Nic Ma --- monai/apps/manifest/config_item.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/monai/apps/manifest/config_item.py b/monai/apps/manifest/config_item.py index 1f0c06c8fc..7ad2194d0d 100644 --- a/monai/apps/manifest/config_item.py +++ b/monai/apps/manifest/config_item.py @@ -32,20 +32,6 @@ class Instantiable(ABC): """ - @abstractmethod - def resolve_module_name(self, *args: Any, **kwargs: Any): - """ - Resolve the target module name, it should return an object class (or function) to be instantiated. - """ - raise NotImplementedError(f"subclass {self.__class__.__name__} must implement this method.") - - @abstractmethod - def resolve_args(self, *args: Any, **kwargs: Any): - """ - Resolve the arguments, it should return arguments to be passed to the object when instantiating. - """ - raise NotImplementedError(f"subclass {self.__class__.__name__} must implement this method.") - @abstractmethod def is_disabled(self, *args: Any, **kwargs: Any) -> bool: """ @@ -54,9 +40,9 @@ def is_disabled(self, *args: Any, **kwargs: Any) -> bool: raise NotImplementedError(f"subclass {self.__class__.__name__} must implement this method.") @abstractmethod - def instantiate(self, *args: Any, **kwargs: Any): + def instantiate(self, *args: Any, **kwargs: Any) -> object: """ - Instantiate the target component. + Instantiate the target component and return the instance. """ raise NotImplementedError(f"subclass {self.__class__.__name__} must implement this method.") From b6e800512b8257216beaeede5909613d5b10f296 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 18 Feb 2022 14:06:11 +0000 Subject: [PATCH 5/8] update docstring (wip) Signed-off-by: Wenqi Li --- monai/apps/manifest/config_item.py | 12 ++---- monai/apps/manifest/reference_resolver.py | 47 ++++++++++++----------- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/monai/apps/manifest/config_item.py b/monai/apps/manifest/config_item.py index 7ad2194d0d..1320f4796e 100644 --- a/monai/apps/manifest/config_item.py +++ b/monai/apps/manifest/config_item.py @@ -23,13 +23,7 @@ class Instantiable(ABC): """ - Base class for instantiable object with module name and arguments. - - .. code-block:: python - - if not is_disabled(): - instantiate(module_name=resolve_module_name(), args=resolve_args()) - + Base class for an instantiable object. """ @abstractmethod @@ -316,7 +310,7 @@ def __init__(self, config: Any, id: Optional[str] = None, globals: Optional[Dict def evaluate(self, locals: Optional[Dict] = None): """ - Excute current config content and return the result if it is expression, based on python `eval()`. + Execute the current config content and return the result if it is expression, based on Python `eval()`. For more details: https://docs.python.org/3/library/functions.html#eval. Args: @@ -332,7 +326,7 @@ def evaluate(self, locals: Optional[Dict] = None): def is_expression(config: Union[Dict, List, str]) -> bool: """ Check whether the config is an executable expression string. - Currently A string starts with ``"$"`` character is interpreted as an expression. + Currently, a string starts with ``"$"`` character is interpreted as an expression. Args: config: input config content to check. diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 55c030675d..29b5570495 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -18,26 +18,27 @@ class ReferenceResolver: """ - Utility class to manage config items and resolve the references between them. - - There are 3 kinds of `references` in the config content: - - The IDs of other config items used as "@XXX" in this config item, for example: - config item with ID="A" is a list `[1, 2, 3]`, another config item "B" can be `"args": {"input_list": "@A"}`. - Then it means A is one reference of B. - - If sub-item in the config is `instantiable`, treat it as reference because must instantiate the sub-item - before using this config. - - If sub-item in the config is `expression`, also treat it as reference because must evaluate the expression - before using this config. - - The typical usage of the APIs: - - Automatically search the content of specified config item and find out all the references. - - Recursively resolve the references of this config item and update them in the config content. - - If this config item is instantiable, try to instantiate it and save the instance in the `resolved_content`. - If this config item is an expression, try to evaluate it and save the result in the `resolved_content`. - Otherwise, save the updated config content in the `resolved_content`. + Utility class to manage a set of ``ConfigItem`` and resolve the references between them. + + This class maintains a set of ``ConfigItem`` objects and their associated IDs. + The IDs must be unique within this set. A string in ``ConfigItem`` + starting with ``@`` will be treated as a reference to other ``ConfigItem`` objects by ID. + Since ``ConfigItem`` may have a nested dictionary or list structure, + the reference string may also contain a ``#`` character to refer to a substructure by + key indexing for a dictionary or integer indexing for a list. + + A typical workflow of resolving references is as follows: + + - Add multiple ``ConfigItem`` objects to the ``ReferenceResolver`` by ``add_item()``. + - Call ``resolve()`` to automatically resolve the references. This is done recursively by: + - Convert the items to objects, for those do not have references to other items. + - If this it is instantiable, instantiate it and cache the class instance in ``resolved_content``. + - If this it is an expression, evaluate it and save the value in ``resolved_content``. + - Replace the reference strings with the actual objects. + - Call ``get_resolved_content()`` to get the items with reference strings replaced by the corresponding objects. Args: - items: config items to resolve, if None, can also `add()` component in runtime. + items: ``ConfigItem``s to resolve, this could be added later with ``add_item()``. """ @@ -62,12 +63,14 @@ def add_item(self, item: ConfigItem): def get_item(self, id: str, resolve: bool = False): """ - Get the config item with specified id name, then can be used for lazy instantiation, etc. - If `resolve=True` and the item is not resolved, try to resolve it first, then it will have - no reference in the config content. + Get the ``ConfigItem`` by id. + + If `resolve=True` and the returned item will be resolved, that is, + all the reference strings are replaced by the corresponding ``ConfigItem`` objects. Args: - id: id name of the expected config item. + id: id of the expected config item. + resolve: whether to resolve the item if it is not resolved, default to False. """ if resolve and id not in self.resolved_content: From 7847855f96d3e0db514c6b4303bfe43327810104 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Fri, 18 Feb 2022 23:46:17 +0800 Subject: [PATCH 6/8] [DLMED] update id Signed-off-by: Nic Ma --- monai/apps/manifest/config_item.py | 12 +++++------ monai/apps/manifest/reference_resolver.py | 26 ++++++++--------------- tests/test_reference_resolver.py | 8 +------ 3 files changed, 16 insertions(+), 30 deletions(-) diff --git a/monai/apps/manifest/config_item.py b/monai/apps/manifest/config_item.py index 1320f4796e..075d00b961 100644 --- a/monai/apps/manifest/config_item.py +++ b/monai/apps/manifest/config_item.py @@ -120,11 +120,11 @@ class ConfigItem: Args: config: content of a config item, can be objects of any types, a configuration resolver may interpret the content to generate a configuration object. - id: optional name of the current config item, defaults to `None`. + id: name of the current config item, defaults to empty string. """ - def __init__(self, config: Any, id: Optional[str] = None) -> None: + def __init__(self, config: Any, id: str = "") -> None: self.config = config self.id = id @@ -183,7 +183,7 @@ class ConfigComponent(ConfigItem, Instantiable): Args: config: content of a config item. - id: optional name of the current config item, defaults to `None`. + id: name of the current config item, defaults to empty string. locator: a ``ComponentLocator`` to convert a module name string into the actual python module. if `None`, a ``ComponentLocator(excludes=excludes)`` will be used. excludes: if ``locator`` is None, create a new ``ComponentLocator`` with ``excludes``. @@ -194,7 +194,7 @@ class ConfigComponent(ConfigItem, Instantiable): def __init__( self, config: Any, - id: Optional[str] = None, + id: str = "", locator: Optional[ComponentLocator] = None, excludes: Optional[Union[Sequence[str], str]] = None, ) -> None: @@ -299,12 +299,12 @@ class ConfigExpression(ConfigItem): Args: config: content of a config item. - id: optional name of current config item, defaults to `None`. + id: name of current config item, defaults to empty string. globals: additional global context to evaluate the string. """ - def __init__(self, config: Any, id: Optional[str] = None, globals: Optional[Dict] = None) -> None: + def __init__(self, config: Any, id: str = "", globals: Optional[Dict] = None) -> None: super().__init__(config=config, id=id) self.globals = globals diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 29b5570495..3716f425d7 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -56,6 +56,8 @@ def add_item(self, item: ConfigItem): """ id = item.get_id() + if len(id) == 0: + raise ValueError("id should not be empty when resolving reference.") if id in self.items: warnings.warn(f"id '{id}' is already added.") return @@ -124,14 +126,6 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None): else: self.resolved_content[id] = new_config - def resolve_all(self): - """ - Resolve the references for all the config items. - - """ - for k in self.items: - self._resolve_one_item(id=k) - def get_resolved_content(self, id: str): """ Get the resolved content with specified id name. @@ -194,9 +188,7 @@ def update_refs_pattern(value: str, refs: Dict) -> str: return value @staticmethod - def find_refs_in_config( - config: Union[Dict, List, str], id: Optional[str] = None, refs: Optional[List[str]] = None - ) -> List[str]: + def find_refs_in_config(config: Union[Dict, List, str], id: str, refs: Optional[List[str]] = None) -> List[str]: """ Recursively search all the content of input config item to get the ids of references. References mean: the IDs of other config items used as "@XXX" in this config item, or the @@ -205,7 +197,7 @@ def find_refs_in_config( Args: config: input config content to search. - id: ID name for the input config item, default to `None`. + id: ID name for the input config item. refs: list of the ID name of found references, default to `None`. """ @@ -216,21 +208,21 @@ def find_refs_in_config( if isinstance(config, (list, dict)): subs = enumerate(config) if isinstance(config, list) else config.items() for k, v in subs: - sub_id = f"{id}#{k}" if id is not None else f"{k}" + sub_id = f"{id}#{k}" if len(id) > 0 else f"{k}" if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): refs_.append(sub_id) refs_ = ReferenceResolver.find_refs_in_config(v, sub_id, refs_) return refs_ @staticmethod - def update_config_with_refs(config: Union[Dict, List, str], id: Optional[str] = None, refs: Optional[Dict] = None): + def update_config_with_refs(config: Union[Dict, List, str], id: str, refs: Optional[Dict] = None): """ With all the references in `refs`, update the input config content with references and return the new config. Args: config: input config content to update. - id: ID name for the input config, default to `None`. + id: ID name for the input config. refs: all the referring content with ids, default to `None`. """ @@ -241,7 +233,7 @@ def update_config_with_refs(config: Union[Dict, List, str], id: Optional[str] = # all the items in the list should be replaced with the references ret_list: List = [] for i, v in enumerate(config): - sub_id = f"{id}#{i}" if id is not None else f"{i}" + sub_id = f"{id}#{i}" if len(id) > 0 else f"{i}" if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): ret_list.append(refs_[sub_id]) else: @@ -251,7 +243,7 @@ def update_config_with_refs(config: Union[Dict, List, str], id: Optional[str] = # all the items in the dict should be replaced with the references ret_dict: Dict = {} for k, v in config.items(): - sub_id = f"{id}#{k}" if id is not None else f"{k}" + sub_id = f"{id}#{k}" if len(id) > 0 else f"{k}" if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): ret_dict[k] = refs_[sub_id] else: diff --git a/tests/test_reference_resolver.py b/tests/test_reference_resolver.py index ac0dcec75a..a62d6befd9 100644 --- a/tests/test_reference_resolver.py +++ b/tests/test_reference_resolver.py @@ -91,13 +91,7 @@ def test_resolve(self, configs, expected_id, output_type): else: resolver.add_item(ConfigItem(config=v, id=k)) - result = resolver.get_resolved_content(expected_id) - self.assertTrue(isinstance(result, output_type)) - - # test resolve all - resolver.resolved_content = {} # clear content - resolver.resolve_all() - result = resolver.get_resolved_content(expected_id) + result = resolver.get_resolved_content(expected_id) # the root id is `expected_id` here self.assertTrue(isinstance(result, output_type)) # test lazy instantiation From bbe6428be2b6938e8e26c11e3f269014fb487ebe Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 18 Feb 2022 16:44:55 +0000 Subject: [PATCH 7/8] nonbreaking updates Signed-off-by: Wenqi Li --- monai/apps/manifest/reference_resolver.py | 115 ++++++++++------------ 1 file changed, 50 insertions(+), 65 deletions(-) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 3716f425d7..14763ba504 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -11,7 +11,7 @@ import re import warnings -from typing import Any, Dict, List, Optional, Sequence, Set, Union +from typing import Any, Dict, Optional, Sequence, Set from monai.apps.manifest.config_item import ConfigComponent, ConfigExpression, ConfigItem @@ -27,15 +27,15 @@ class ReferenceResolver: the reference string may also contain a ``#`` character to refer to a substructure by key indexing for a dictionary or integer indexing for a list. - A typical workflow of resolving references is as follows: + In this class, resolving references is essentially substitution of the reference strings with the + corresponding python objects. A typical workflow of resolving references is as follows: - Add multiple ``ConfigItem`` objects to the ``ReferenceResolver`` by ``add_item()``. - - Call ``resolve()`` to automatically resolve the references. This is done recursively by: + - Call ``get_resolved_content()`` to automatically resolve the references. This is done (recursively) by: - Convert the items to objects, for those do not have references to other items. - - If this it is instantiable, instantiate it and cache the class instance in ``resolved_content``. - - If this it is an expression, evaluate it and save the value in ``resolved_content``. - - Replace the reference strings with the actual objects. - - Call ``get_resolved_content()`` to get the items with reference strings replaced by the corresponding objects. + - If it is instantiable, instantiate it and cache the class instance in ``resolved_content``. + - If it is an expression, evaluate it and save the value in ``resolved_content``. + - Substitute the reference strings with the corresponding objects. Args: items: ``ConfigItem``s to resolve, this could be added later with ``add_item()``. @@ -43,20 +43,20 @@ class ReferenceResolver: """ def __init__(self, items: Optional[Sequence[ConfigItem]] = None): - # save the items in a dictionary with the `id` as key + # save the items in a dictionary with the `ConfigItem.id` as key self.items = {} if items is None else {i.get_id(): i for i in items} self.resolved_content: Dict[str, Any] = {} def add_item(self, item: ConfigItem): """ - Add a config item to the resolver. + Add a ``ConfigItem`` to the resolver. Args: - item: a config item to resolve. + item: a ``ConfigItem``. """ id = item.get_id() - if len(id) == 0: + if id == "": raise ValueError("id should not be empty when resolving reference.") if id in self.items: warnings.warn(f"id '{id}' is already added.") @@ -67,8 +67,8 @@ def get_item(self, id: str, resolve: bool = False): """ Get the ``ConfigItem`` by id. - If `resolve=True` and the returned item will be resolved, that is, - all the reference strings are replaced by the corresponding ``ConfigItem`` objects. + If ``resolve=True``, the returned item will be resolved, that is, + all the reference strings are substituted by the corresponding ``ConfigItem`` objects. Args: id: id of the expected config item. @@ -81,13 +81,13 @@ def get_item(self, id: str, resolve: bool = False): def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None): """ - Resolve one config item with specified id name, save the resolved result in `resolved_content`. + Resolve one ``ConfigItem`` of ``id``, cache the resolved result in ``resolved_content``. If it has unresolved references, recursively resolve the referring items first. Args: - id: id name of expected config item to resolve. - waiting_list: list of the ids of items wait to resolve references. - it's used to detect circular references when resolving references like: + id: id name of ``ConfigItem`` to be resolved. + waiting_list: set of ids pending to be resolved. + It's used to detect circular references such as: `{"name": "A", "dep": "@B"}` and `{"name": "B", "dep": "@A"}`. """ @@ -105,7 +105,7 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None): if d in waiting_list: raise ValueError(f"detected circular references for id='{d}' in the config content.") - if len(ref_ids) > 0: + if ref_ids: # # check whether the component has any unresolved references for ref_id in ref_ids: if ref_id not in self.resolved_content: @@ -128,8 +128,7 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None): def get_resolved_content(self, id: str): """ - Get the resolved content with specified id name. - If not resolved yet, try to resolve it first. + Get the resolved ``ConfigItem`` by id. If there are unresolved references, try to resolve them first. Args: id: id name of the expected item. @@ -140,31 +139,29 @@ def get_resolved_content(self, id: str): return self.resolved_content[id] @staticmethod - def match_refs_pattern(value: str) -> List[str]: + def match_refs_pattern(value: str) -> Set[str]: """ Match regular expression for the input string to find the references. - The reference part starts with "@", like: "@XXX#YYY#ZZZ". + The reference string starts with ``"@"``, like: ``"@XXX#YYY#ZZZ"``. Args: value: input value to match regular expression. """ - refs: List[str] = [] + refs: Set[str] = set() # regular expression pattern to match "@XXX" or "@XXX#YYY" result = re.compile(r"@\w*[\#\w]*").findall(value) for item in result: if ConfigExpression.is_expression(value) or value == item: # only check when string starts with "$" or the whole content is "@XXX" - ref_obj_id = item[1:] - if ref_obj_id not in refs: - refs.append(ref_obj_id) + refs.add(item[1:]) return refs @staticmethod def update_refs_pattern(value: str, refs: Dict) -> str: """ Match regular expression for the input string to update content with the references. - The reference part starts with "@", like: "@XXX#YYY#ZZZ". + The reference part starts with ``"@"``, like: ``"@XXX#YYY#ZZZ"``. References dictionary must contain the referring IDs as keys. Args: @@ -178,7 +175,6 @@ def update_refs_pattern(value: str, refs: Dict) -> str: ref_id = item[1:] if ref_id not in refs: raise KeyError(f"can not find expected ID '{ref_id}' in the references.") - if ConfigExpression.is_expression(value): # replace with local code, will be used in the `evaluate` logic with `locals={"refs": ...}` value = value.replace(item, f"refs['{ref_id}']") @@ -188,10 +184,10 @@ def update_refs_pattern(value: str, refs: Dict) -> str: return value @staticmethod - def find_refs_in_config(config: Union[Dict, List, str], id: str, refs: Optional[List[str]] = None) -> List[str]: + def find_refs_in_config(config, id: str, refs: Optional[Set[str]] = None) -> Set[str]: """ Recursively search all the content of input config item to get the ids of references. - References mean: the IDs of other config items used as "@XXX" in this config item, or the + References mean: the IDs of other config items (``"@XXX"`` in this config item), or the sub-item in the config is `instantiable`, or the sub-item in the config is `expression`. For `dict` and `list`, recursively check the sub-items. @@ -201,23 +197,22 @@ def find_refs_in_config(config: Union[Dict, List, str], id: str, refs: Optional[ refs: list of the ID name of found references, default to `None`. """ - refs_: List[str] = [] if refs is None else refs + refs_: Set[str] = refs or set() if isinstance(config, str): - refs_ += ReferenceResolver.match_refs_pattern(value=config) - - if isinstance(config, (list, dict)): - subs = enumerate(config) if isinstance(config, list) else config.items() - for k, v in subs: - sub_id = f"{id}#{k}" if len(id) > 0 else f"{k}" - if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): - refs_.append(sub_id) - refs_ = ReferenceResolver.find_refs_in_config(v, sub_id, refs_) + return refs_.union(ReferenceResolver.match_refs_pattern(value=config)) + if not isinstance(config, (list, dict)): + return refs_ + for k, v in config.items() if isinstance(config, dict) else enumerate(config): + sub_id = f"{id}#{k}" if id != "" else f"{k}" + if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): + refs_.add(sub_id) + refs_ = ReferenceResolver.find_refs_in_config(v, sub_id, refs_) return refs_ @staticmethod - def update_config_with_refs(config: Union[Dict, List, str], id: str, refs: Optional[Dict] = None): + def update_config_with_refs(config, id: str, refs: Optional[Dict] = None): """ - With all the references in `refs`, update the input config content with references + With all the references in ``refs``, update the input config content with references and return the new config. Args: @@ -226,27 +221,17 @@ def update_config_with_refs(config: Union[Dict, List, str], id: str, refs: Optio refs: all the referring content with ids, default to `None`. """ - refs_: Dict = {} if refs is None else refs + refs_: Dict = refs or {} if isinstance(config, str): - config = ReferenceResolver.update_refs_pattern(config, refs_) - if isinstance(config, list): - # all the items in the list should be replaced with the references - ret_list: List = [] - for i, v in enumerate(config): - sub_id = f"{id}#{i}" if len(id) > 0 else f"{i}" - if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): - ret_list.append(refs_[sub_id]) - else: - ret_list.append(ReferenceResolver.update_config_with_refs(v, sub_id, refs_)) - return ret_list - if isinstance(config, dict): - # all the items in the dict should be replaced with the references - ret_dict: Dict = {} - for k, v in config.items(): - sub_id = f"{id}#{k}" if len(id) > 0 else f"{k}" - if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): - ret_dict[k] = refs_[sub_id] - else: - ret_dict[k] = ReferenceResolver.update_config_with_refs(v, sub_id, refs_) - return ret_dict - return config + return ReferenceResolver.update_refs_pattern(config, refs_) + if not isinstance(config, (list, dict)): + return config + ret = type(config)() + for idx, v in config.items() if isinstance(config, dict) else enumerate(config): + sub_id = f"{id}#{idx}" if id != "" else f"{idx}" + if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): + updated = ReferenceResolver.update_config_with_refs(v, sub_id, refs_) + else: + updated = ReferenceResolver.update_config_with_refs(v, sub_id, refs_) + ret.update({idx: updated}) if isinstance(ret, dict) else ret.append(updated) + return ret From 3e67741ea1244db79e44ab50d60dcc8fee4b013a Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Sat, 19 Feb 2022 08:03:47 +0800 Subject: [PATCH 8/8] [DLMED] simplify code Signed-off-by: Nic Ma --- monai/apps/manifest/reference_resolver.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 14763ba504..32d089370a 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -105,15 +105,14 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None): if d in waiting_list: raise ValueError(f"detected circular references for id='{d}' in the config content.") - if ref_ids: - # # check whether the component has any unresolved references - for ref_id in ref_ids: - if ref_id not in self.resolved_content: - # this referring item is not resolved - if ref_id not in self.items: - raise ValueError(f"the referring item `{ref_id}` is not defined in config.") - # recursively resolve the reference first - self._resolve_one_item(id=ref_id, waiting_list=waiting_list) + # # check whether the component has any unresolved references + for d in ref_ids: + if d not in self.resolved_content: + # this referring item is not resolved + if d not in self.items: + raise ValueError(f"the referring item `{d}` is not defined in config.") + # recursively resolve the reference first + self._resolve_one_item(id=d, waiting_list=waiting_list) # all references are resolved, then try to resolve current config item new_config = self.update_config_with_refs(config=item_config, id=id, refs=self.resolved_content)