From a3193bae88fac8c1974fca6db129ad0478c2dc59 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Sat, 19 Feb 2022 12:28:32 +0800 Subject: [PATCH 01/21] [DLMED] add config parser Signed-off-by: Nic Ma --- docs/source/apps.rst | 3 + monai/apps/__init__.py | 2 +- monai/apps/manifest/__init__.py | 1 + monai/apps/manifest/config_parser.py | 178 +++++++++++++++++++++++++++ tests/test_config_parser.py | 76 ++++++++++++ tests/test_reference_resolver.py | 2 +- 6 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 monai/apps/manifest/config_parser.py create mode 100644 tests/test_config_parser.py diff --git a/docs/source/apps.rst b/docs/source/apps.rst index 4b1cdc6f43..d0fe131d85 100644 --- a/docs/source/apps.rst +++ b/docs/source/apps.rst @@ -44,6 +44,9 @@ Model Manifest .. autoclass:: ConfigItem :members: +.. autoclass:: ConfigParser + :members: + .. autoclass:: ReferenceResolver :members: diff --git a/monai/apps/__init__.py b/monai/apps/__init__.py index 0f233bc3ef..51c4003458 100644 --- a/monai/apps/__init__.py +++ b/monai/apps/__init__.py @@ -10,6 +10,6 @@ # limitations under the License. from .datasets import CrossValidation, DecathlonDataset, MedNISTDataset -from .manifest import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, ReferenceResolver +from .manifest import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, ConfigParser, ReferenceResolver from .mmars import MODEL_DESC, RemoteMMARKeys, download_mmar, get_model_spec, load_from_mmar from .utils import SUPPORTED_HASH_TYPES, check_hash, download_and_extract, download_url, extractall, get_logger, logger diff --git a/monai/apps/manifest/__init__.py b/monai/apps/manifest/__init__.py index 79c4376d5c..b8ddf57f93 100644 --- a/monai/apps/manifest/__init__.py +++ b/monai/apps/manifest/__init__.py @@ -10,4 +10,5 @@ # limitations under the License. from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem +from .config_parser import ConfigParser from .reference_resolver import ReferenceResolver diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py new file mode 100644 index 0000000000..f7180078cb --- /dev/null +++ b/monai/apps/manifest/config_parser.py @@ -0,0 +1,178 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +from copy import deepcopy +from typing import Any, Dict, Optional, Sequence, Union + +from monai.apps.manifest.config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem +from monai.apps.manifest.reference_resolver import ReferenceResolver + + +class ConfigParser: + """ + Parse a config content, access or update the items of config content with unique ID. + A typical usage is a config dictionary contains all the necessary information to define training workflow in JSON. + For more details of the config format, please check :py:class:`monai.apps.ConfigComponent`. + + It can recursively parse the config content, treat every item as a `ConfigItem` with unique ID, the ID is joined + by "#" mark for nested content. For example: + The config content `{"preprocessing": [{"": "LoadImage", "": {"keys": "image"}}]}` is parsed as items: + - `id="preprocessing", config=[{"": "LoadImage", "": {"keys": "image"}}]` + - `id="preprocessing#0", config={"": "LoadImage", "": {"keys": "image"}}` + - `id="preprocessing#0#", config="LoadImage"` + - `id="preprocessing#0#", config={"keys": "image"}` + - `id="preprocessing#0##keys", config="image"` + + There are 3 levels config information during the parsing: + - For the input config content, it supports to `get` and `update` the whole content or part of it specified with id, + it can be useful for lazy instantiation, etc. + - After parsing, all the config items are independent `ConfigItem`, can get it before / after resolving references. + - After resolving, the resolved output of every `ConfigItem` is python objects or instances, can be used in other + programs directly. + + Args: + config: input config content to parse. + id: specified ID name for the config content. + excludes: when importing modules to instantiate components, if any string of the `excludes` exists + in the full module name, don't import this module. + globals: pre-import packages as global variables to evaluate the python `eval` expressions. + for example, pre-import `monai`, then execute `eval("monai.data.list_data_collate")`. + default to `{"monai": "monai", "torch": "torch", "np": "numpy"}` as `numpy` and `torch` + are MONAI mininum requirements. + if the value in global is string, will import it immediately. + + """ + + def __init__( + self, + config: Any, + excludes: Optional[Union[Sequence[str], str]] = None, + globals: Optional[Dict[str, Any]] = None, + ): + self.config = None + self.update_config(config=config) + + self.globals: Dict[str, Any] = {} + globals = {"monai": "monai", "torch": "torch", "np": "numpy"} if globals is None else globals + if globals is not None: + for k, v in globals.items(): + self.globals[k] = importlib.import_module(v) if isinstance(v, str) else v + + self.locator = ComponentLocator(excludes=excludes) + self.reference_resolver: ReferenceResolver = ReferenceResolver() + # flag to identify the parsing status of current config content + self.parsed = False + + def update_config(self, config: Any, id: str = ""): + """ + Set config content for the parser, if `id` provided, `config` will replace the config item with `id`. + + Args: + config: target config content to set. + id: id name to specify the target position, joined by "#" mark for nested content, use index from 0 for list. + for example: "transforms#5", "transforms#5##keys", etc. + default to update all the config content. + + """ + if id != "" and isinstance(self.config, (dict, list)): + keys = id.split("#") + # get the last second config item and replace it + last_id = "#".join(keys[:-1]) + conf_ = self.get_config(id=last_id) + conf_[keys[-1] if isinstance(conf_, dict) else int(keys[-1])] = config + else: + self.config = config + # must totally parse again as the content is modified + self.parsed = False + + def get_config(self, id: str = ""): + """ + Get config content of current config, if `id` provided, get the config item with `id`. + + Args: + id: nested id name to specify the expected position, joined by "#" mark, use index from 0 for list. + for example: "transforms#5", "transforms#5##keys", etc. + default to get all the config content. + + """ + config = self.config + if id != "" and isinstance(config, (dict, list)): + keys = id.split("#") + for k in keys: + config = config[k] if isinstance(config, dict) else config[int(k)] + return config + + def _do_parse(self, config, id: str = ""): + """ + Recursively parse the nested config content, add every config item to the resolver. + + Args: + config: config content to parse. + id: id name of current config item, nested ids are joined by "#" mark. defaults to None. + for example: "transforms#5", "transforms#5##keys", etc. + default to empty string. + + """ + if isinstance(config, (dict, list)): + subs = enumerate(config) if isinstance(config, list) else config.items() + for k, v in subs: + sub_id = f"{id}#{k}" if id != "" else k + self._do_parse(config=v, id=sub_id) + + if id != "": + # copy every config item to make them independent and add them to the resolver + item_conf = deepcopy(config) + if ConfigComponent.is_instantiable(item_conf): + self.reference_resolver.add_item(ConfigComponent(config=item_conf, id=id, locator=self.locator)) + elif ConfigExpression.is_expression(item_conf): + self.reference_resolver.add_item(ConfigExpression(config=item_conf, id=id, globals=self.globals)) + else: + self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) + + def parse_config(self): + """ + Parse the config content, add every config item to the resolver and mark as `parsed`. + + """ + self.reference_resolver = ReferenceResolver() + self._do_parse(config=self.config) + self.parsed = True + + def get_resolved_content(self, id: str): + """ + Get the resolved result of config items with specified id, if not resolved, try to resolve it first. + If the config item is instantiable, the resolved result is the instance. + If the config item is an expression, the resolved result is output of when evaluating the expression. + Otherwise, the resolved result is the updated config content of the config item. + + Args: + id: id name of expected config item, nested ids are joined by "#" mark. + for example: "transforms#5", "transforms#5##keys", etc. + + """ + if not self.parsed: + self.parse_config() + return self.reference_resolver.get_resolved_content(id=id) + + def get_config_item(self, id: str, resolve: bool = False): + """ + Get the parsed config item, if `resolve=True` and not resolved, try to resolve it first. + It can be used to modify the config in other program and support lazy instantiation. + + Args: + id: id name of expected config component, nested ids are joined by "#" mark. + for example: "transforms#5", "transforms#5##keys", etc. + + """ + if not self.parsed: + self.parse_config() + return self.reference_resolver.get_item(id=id, resolve=resolve) diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py new file mode 100644 index 0000000000..7e07129e50 --- /dev/null +++ b/tests/test_config_parser.py @@ -0,0 +1,76 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest import skipUnless + +from parameterized import parameterized + +from monai.apps import ConfigParser +from monai.apps.manifest.config_item import ConfigComponent +from monai.data import DataLoader, Dataset +from monai.transforms import Compose, LoadImaged, RandTorchVisiond +from monai.utils import optional_import + +_, has_tv = optional_import("torchvision") + +# test the resolved and parsed instances +TEST_CASE_1 = [ + { + "transform": { + "": "Compose", + "": { + "transforms": [ + {"": "LoadImaged", "": {"keys": "image"}}, + { + "": "RandTorchVisiond", + "": {"keys": "image", "name": "ColorJitter", "brightness": 0.25}, + }, + ] + }, + }, + "dataset": {"": "Dataset", "": {"data": [1, 2], "transform": "@transform"}}, + "dataloader": { + "": "DataLoader", + "": {"dataset": "@dataset", "batch_size": 2, "collate_fn": "monai.data.list_data_collate"}, + }, + }, + ["transform", "transform##transforms#0", "transform##transforms#1", "dataset", "dataloader"], + [Compose, LoadImaged, RandTorchVisiond, Dataset, DataLoader], +] + + +class TestConfigComponent(unittest.TestCase): + def test_config_content(self): + parser = ConfigParser(config={}) + test_config = {"preprocessing": [{"": "LoadImage"}], "dataset": {"": "Dataset"}} + parser.update_config(config=test_config) + self.assertEqual(str(parser.get_config()), str(test_config)) + parser.update_config(config={"": "CacheDataset"}, id="dataset") + self.assertDictEqual(parser.get_config(id="dataset"), {"": "CacheDataset"}) + parser.update_config(config="Dataset", id="dataset#") + self.assertEqual(parser.get_config(id="dataset#"), "Dataset") + + @parameterized.expand([TEST_CASE_1]) + @skipUnless(has_tv, "Requires torchvision.") + def test_parse(self, config, expected_ids, output_types): + parser = ConfigParser(config=config, globals={"monai": "monai"}) + for id, cls in zip(expected_ids, output_types): + item = parser.get_config_item(id, resolve=True) + # test lazy instantiation + if isinstance(item, ConfigComponent): + self.assertTrue(isinstance(item.instantiate(), cls)) + # test get instance directly + self.assertTrue(isinstance(parser.get_resolved_content(id), cls)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_reference_resolver.py b/tests/test_reference_resolver.py index a62d6befd9..4f73233c24 100644 --- a/tests/test_reference_resolver.py +++ b/tests/test_reference_resolver.py @@ -1,4 +1,4 @@ -# Copyright 2020 - 2021 MONAI Consortium +# Copyright (c) MONAI Consortium # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at From 04e3ee79736f2eac16ae52d374e93719791da2a1 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Sun, 20 Feb 2022 08:27:22 +0800 Subject: [PATCH 02/21] [DLMED] simplify parse logic Signed-off-by: Nic Ma --- monai/apps/manifest/config_item.py | 2 +- monai/apps/manifest/config_parser.py | 29 +++++++++-------------- monai/apps/manifest/reference_resolver.py | 10 +++++++- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/monai/apps/manifest/config_item.py b/monai/apps/manifest/config_item.py index 075d00b961..a25dfcedf2 100644 --- a/monai/apps/manifest/config_item.py +++ b/monai/apps/manifest/config_item.py @@ -128,7 +128,7 @@ def __init__(self, config: Any, id: str = "") -> None: self.config = config self.id = id - def get_id(self) -> Optional[str]: + def get_id(self) -> str: """ Get the ID name of current config item, useful to identify config items during parsing. diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index f7180078cb..3e0efeb642 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -33,11 +33,12 @@ class ConfigParser: - `id="preprocessing#0##keys", config="image"` There are 3 levels config information during the parsing: - - For the input config content, it supports to `get` and `update` the whole content or part of it specified with id, - it can be useful for lazy instantiation, etc. - - After parsing, all the config items are independent `ConfigItem`, can get it before / after resolving references. - - After resolving, the resolved output of every `ConfigItem` is python objects or instances, can be used in other - programs directly. + - `get_config()`: For the input config content, it supports to `get` and `update` the whole content or part of it + specified with id, it can be useful for lazy instantiation. + - `get_config_item(resolve=True/False)`: After parsing, all the config items are independent `ConfigItem`, + can get it before / after resolving references. + - `get_resolved_content()`: After resolving, the resolved output of `ConfigItem` is python objects or instances, + can be used in other programs directly. Args: config: input config content to parse. @@ -59,8 +60,6 @@ def __init__( globals: Optional[Dict[str, Any]] = None, ): self.config = None - self.update_config(config=config) - self.globals: Dict[str, Any] = {} globals = {"monai": "monai", "torch": "torch", "np": "numpy"} if globals is None else globals if globals is not None: @@ -68,9 +67,8 @@ def __init__( self.globals[k] = importlib.import_module(v) if isinstance(v, str) else v self.locator = ComponentLocator(excludes=excludes) - self.reference_resolver: ReferenceResolver = ReferenceResolver() - # flag to identify the parsing status of current config content - self.parsed = False + self.reference_resolver = ReferenceResolver() + self.update_config(config=config) def update_config(self, config: Any, id: str = ""): """ @@ -92,7 +90,7 @@ def update_config(self, config: Any, id: str = ""): else: self.config = config # must totally parse again as the content is modified - self.parsed = False + self.parse_config() def get_config(self, id: str = ""): """ @@ -140,12 +138,11 @@ def _do_parse(self, config, id: str = ""): def parse_config(self): """ - Parse the config content, add every config item to the resolver and mark as `parsed`. + Parse the config content, add every config item to the resolver. """ - self.reference_resolver = ReferenceResolver() + self.reference_resolver.reset() self._do_parse(config=self.config) - self.parsed = True def get_resolved_content(self, id: str): """ @@ -159,8 +156,6 @@ def get_resolved_content(self, id: str): for example: "transforms#5", "transforms#5##keys", etc. """ - if not self.parsed: - self.parse_config() return self.reference_resolver.get_resolved_content(id=id) def get_config_item(self, id: str, resolve: bool = False): @@ -173,6 +168,4 @@ def get_config_item(self, id: str, resolve: bool = False): for example: "transforms#5", "transforms#5##keys", etc. """ - if not self.parsed: - self.parse_config() return self.reference_resolver.get_item(id=id, resolve=resolve) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 32d089370a..56177a5532 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -44,9 +44,17 @@ class ReferenceResolver: def __init__(self, items: Optional[Sequence[ConfigItem]] = None): # save the items in a dictionary with the `ConfigItem.id` as key - self.items = {} if items is None else {i.get_id(): i for i in items} + self.items: Dict[str, Any] = {} if items is None else {i.get_id(): i for i in items} self.resolved_content: Dict[str, Any] = {} + def reset(self): + """ + Clear all the added `ConfigItem` and all the resolved content. + + """ + self.items = {} + self.resolved_content = {} + def add_item(self, item: ConfigItem): """ Add a ``ConfigItem`` to the resolver. From 9abb571336f0b87afc198520c4d6d7f033a1fdc8 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 21 Feb 2022 15:53:51 +0800 Subject: [PATCH 03/21] [DLMED] add more function tests Signed-off-by: Nic Ma --- monai/utils/module.py | 7 ++++--- tests/test_config_parser.py | 40 +++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/monai/utils/module.py b/monai/utils/module.py index 8b7745c3ee..ca47ae6b2d 100644 --- a/monai/utils/module.py +++ b/monai/utils/module.py @@ -10,13 +10,13 @@ # limitations under the License. import enum -import inspect import os import re import sys import warnings from functools import partial, wraps from importlib import import_module +from inspect import isclass, isfunction, ismethod from pkgutil import walk_packages from pydoc import locate from re import match @@ -212,9 +212,10 @@ def instantiate(path: str, **kwargs): component = locate(path) if component is None: raise ModuleNotFoundError(f"Cannot locate '{path}'.") - if inspect.isclass(component): + if isclass(component): return component(**kwargs) - if inspect.isfunction(component): + # support regular function, static method and class method + if isfunction(component) or (ismethod(component) and isclass(getattr(component, "__self__", None))): return partial(component, **kwargs) warnings.warn(f"Component to instantiate must represent a valid class or function, but got {path}.") diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 7e07129e50..314340adce 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -48,6 +48,34 @@ ] +class TestClass: + @staticmethod + def compute(a, b, func=lambda x, y: x + y): + return func(a, b) + + @classmethod + def cls_compute(cls, a, b, func=lambda x, y: x + y): + return cls.compute(a, b, func) + + def __call__(self, a, b): + return self.compute(a, b) + + +TEST_CASE_2 = [ + { + "basic_func": "$lambda x, y: x + y", + "static_func": "$TestClass.compute", + "cls_func": "$TestClass.cls_compute", + "lambda_static_func": "$lambda x, y: TestClass.compute(x, y)", + "lambda_cls_func": "$lambda x, y: TestClass.cls_compute(x, y)", + "compute": {"": "tests.test_config_parser.TestClass.compute", "": {"func": "@basic_func"}}, + "cls_compute": {"": "tests.test_config_parser.TestClass.cls_compute", "": {"func": "@basic_func"}}, + "call_compute": {"": "tests.test_config_parser.TestClass"}, + "error_func": "$TestClass.__call__", + } +] + + class TestConfigComponent(unittest.TestCase): def test_config_content(self): parser = ConfigParser(config={}) @@ -71,6 +99,18 @@ def test_parse(self, config, expected_ids, output_types): # test get instance directly self.assertTrue(isinstance(parser.get_resolved_content(id), cls)) + @parameterized.expand([TEST_CASE_2]) + def test_function(self, config): + parser = ConfigParser(config=config, globals={"TestClass": TestClass}) + for id in config: + func = parser.get_resolved_content(id=id) + self.assertTrue(id in parser.reference_resolver.resolved_content) + if id == "error_func": + with self.assertRaises(TypeError): + func(1, 2) + continue + self.assertEqual(func(1, 2), 3) + if __name__ == "__main__": unittest.main() From fe3f7c284ed2124b5b01be99729454f0993ca5ee Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 21 Feb 2022 18:10:12 +0800 Subject: [PATCH 04/21] [DLMED] fix torchvision tests Signed-off-by: Nic Ma --- tests/test_config_item.py | 4 ++-- tests/test_config_parser.py | 6 +++--- tests/test_reference_resolver.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_config_item.py b/tests/test_config_item.py index b2c2fec6c6..0b4fdc51bf 100644 --- a/tests/test_config_item.py +++ b/tests/test_config_item.py @@ -20,9 +20,9 @@ from monai.apps import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem from monai.data import DataLoader, Dataset from monai.transforms import LoadImaged, RandTorchVisiond -from monai.utils import optional_import +from monai.utils import min_version, optional_import -_, has_tv = optional_import("torchvision") +_, has_tv = optional_import("torchvision", "0.8.0", min_version) TEST_CASE_1 = [{"lr": 0.001}, 0.0001] diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 314340adce..9aa1e7cfdd 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -18,9 +18,9 @@ from monai.apps.manifest.config_item import ConfigComponent from monai.data import DataLoader, Dataset from monai.transforms import Compose, LoadImaged, RandTorchVisiond -from monai.utils import optional_import +from monai.utils import min_version, optional_import -_, has_tv = optional_import("torchvision") +_, has_tv = optional_import("torchvision", "0.8.0", min_version) # test the resolved and parsed instances TEST_CASE_1 = [ @@ -88,7 +88,7 @@ def test_config_content(self): self.assertEqual(parser.get_config(id="dataset#"), "Dataset") @parameterized.expand([TEST_CASE_1]) - @skipUnless(has_tv, "Requires torchvision.") + @skipUnless(has_tv, "Requires torchvision >= 0.8.0.") def test_parse(self, config, expected_ids, output_types): parser = ConfigParser(config=config, globals={"monai": "monai"}) for id, cls in zip(expected_ids, output_types): diff --git a/tests/test_reference_resolver.py b/tests/test_reference_resolver.py index 4f73233c24..90d0f3ea72 100644 --- a/tests/test_reference_resolver.py +++ b/tests/test_reference_resolver.py @@ -19,9 +19,9 @@ from monai.apps.manifest.config_item import ComponentLocator, ConfigExpression, ConfigItem from monai.data import DataLoader from monai.transforms import LoadImaged, RandTorchVisiond -from monai.utils import optional_import +from monai.utils import min_version, optional_import -_, has_tv = optional_import("torchvision") +_, has_tv = optional_import("torchvision", "0.8.0", min_version) # test instance with no dependencies TEST_CASE_1 = [ From 886b607ad02e2dd036f18009f324b2db67185c59 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 21 Feb 2022 22:08:30 +0800 Subject: [PATCH 05/21] [DLMED] enhance update_config to support multiple items Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 35 ++++++++++++++-------------- tests/test_config_parser.py | 6 ++--- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index 3e0efeb642..2f4c3bcede 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -68,27 +68,26 @@ def __init__( self.locator = ComponentLocator(excludes=excludes) self.reference_resolver = ReferenceResolver() - self.update_config(config=config) + self.update_config({"": config}) - def update_config(self, config: Any, id: str = ""): + def update_config(self, content: Dict[str, Any]): """ - Set config content for the parser, if `id` provided, `config` will replace the config item with `id`. - - Args: - config: target config content to set. - id: id name to specify the target position, joined by "#" mark for nested content, use index from 0 for list. - for example: "transforms#5", "transforms#5##keys", etc. - default to update all the config content. + Update config content for the parser, every `key` and `value` in the `content` is corresponding to + the target `id` position and new config value in order. + Nested config id is joined by "#" mark, use index from 0 for list. + For example: "transforms#5", "transforms#5##keys", etc. + If `id` is `""`, replace `self.config`. """ - if id != "" and isinstance(self.config, (dict, list)): - keys = id.split("#") - # get the last second config item and replace it - last_id = "#".join(keys[:-1]) - conf_ = self.get_config(id=last_id) - conf_[keys[-1] if isinstance(conf_, dict) else int(keys[-1])] = config - else: - self.config = config + for id, config in content.items(): + if id != "" and isinstance(self.config, (dict, list)): + keys = id.split("#") + # get the last second config item and replace it + last_id = "#".join(keys[:-1]) + conf_ = self.get_config(id=last_id) + conf_[keys[-1] if isinstance(conf_, dict) else int(keys[-1])] = config + else: + self.config = config # must totally parse again as the content is modified self.parse_config() @@ -97,7 +96,7 @@ def get_config(self, id: str = ""): Get config content of current config, if `id` provided, get the config item with `id`. Args: - id: nested id name to specify the expected position, joined by "#" mark, use index from 0 for list. + id: id name to specify the expected position, nested config is joined by "#" mark, use index from 0 for list. for example: "transforms#5", "transforms#5##keys", etc. default to get all the config content. diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 9aa1e7cfdd..4814b12523 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -80,11 +80,11 @@ class TestConfigComponent(unittest.TestCase): def test_config_content(self): parser = ConfigParser(config={}) test_config = {"preprocessing": [{"": "LoadImage"}], "dataset": {"": "Dataset"}} - parser.update_config(config=test_config) + parser.update_config(content={"": test_config}) self.assertEqual(str(parser.get_config()), str(test_config)) - parser.update_config(config={"": "CacheDataset"}, id="dataset") + parser.update_config(content={"dataset": {"": "CacheDataset"}}) self.assertDictEqual(parser.get_config(id="dataset"), {"": "CacheDataset"}) - parser.update_config(config="Dataset", id="dataset#") + parser.update_config(content={"dataset#": "Dataset"}) self.assertEqual(parser.get_config(id="dataset#"), "Dataset") @parameterized.expand([TEST_CASE_1]) From b90b73201d97f19267c00bcaf7060d818ff9aac4 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 22 Feb 2022 17:50:33 +0800 Subject: [PATCH 06/21] [DLMED] enhance doc and unit tests Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 51 ++++++++++++++++++++++------ tests/test_config_parser.py | 10 ++++-- 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index 2f4c3bcede..106d8529ff 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -32,13 +32,44 @@ class ConfigParser: - `id="preprocessing#0#", config={"keys": "image"}` - `id="preprocessing#0##keys", config="image"` - There are 3 levels config information during the parsing: - - `get_config()`: For the input config content, it supports to `get` and `update` the whole content or part of it - specified with id, it can be useful for lazy instantiation. - - `get_config_item(resolve=True/False)`: After parsing, all the config items are independent `ConfigItem`, - can get it before / after resolving references. - - `get_resolved_content()`: After resolving, the resolved output of `ConfigItem` is python objects or instances, - can be used in other programs directly. + A typical workflow of config parsing is as follows: + + - Initialize `ConfigParser` with the `config` content. + - Call ``get_resolved_content()`` to get expected component with `id`, which is automatically parsed. + + .. code-block:: python + + config = { + "preprocessing": {"": "LoadImage"}, + "net": {"": "UNet", "": ...}, + "trainer": {"": "SupervisedTrainer", "": {"network": "@net", ...}}, + } + parser = ConfigParser(config=config) + trainer = parser.get_resolved_content(id="trainer") + trainer.run() + + It's also flexible to modify config content and do `lazy instantiation` from 2 levels: + + 1. Modify original config content, then totally parse again: + + .. code-block:: python + + parser = ConfigParser(...) + config = parser.get_config() + config["processing"][2][""]["interp_order"] = "bilinear" + parser.parse_config() + + 2. After parsing, all the config items are independent `ConfigItem`, get the expected `ConfigItem`. + Its config content is already resolved references, can modify and use it directly: + + .. code-block:: python + + parser = ConfigParser(...) + trainer_config_item = parser.get_config_item(id="trainer") + config = trainer_config_item.get_config() + config["max_epochs"] = 100 + trainer = trainer_config_item.instantiate() + trainer.run() Args: config: input config content to parse. @@ -157,9 +188,9 @@ def get_resolved_content(self, id: str): """ return self.reference_resolver.get_resolved_content(id=id) - def get_config_item(self, id: str, resolve: bool = False): + def get_config_item(self, id: str): """ - Get the parsed config item, if `resolve=True` and not resolved, try to resolve it first. + Get the parsed config item which is already resolved all the references. It can be used to modify the config in other program and support lazy instantiation. Args: @@ -167,4 +198,4 @@ def get_config_item(self, id: str, resolve: bool = False): for example: "transforms#5", "transforms#5##keys", etc. """ - return self.reference_resolver.get_item(id=id, resolve=resolve) + return self.reference_resolver.get_item(id=id, resolve=True) diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 4814b12523..46bbc98444 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -91,9 +91,15 @@ def test_config_content(self): @skipUnless(has_tv, "Requires torchvision >= 0.8.0.") def test_parse(self, config, expected_ids, output_types): parser = ConfigParser(config=config, globals={"monai": "monai"}) + # test lazy instantiation with original config content + config = parser.get_config() + config["transform"][""]["transforms"][0][""]["keys"] = "label" + parser.parse_config() + self.assertEqual(parser.get_resolved_content(id="transform##transforms#0").keys[0], "label") + for id, cls in zip(expected_ids, output_types): - item = parser.get_config_item(id, resolve=True) - # test lazy instantiation + item = parser.get_config_item(id=id) + # test lazy instantiation with resolved config item if isinstance(item, ConfigComponent): self.assertTrue(isinstance(item.instantiate(), cls)) # test get instance directly From bfc74fc9371abff21fe9382063b3d248e9175812 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 22 Feb 2022 23:58:48 +0800 Subject: [PATCH 07/21] [DLMED] clear API Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 72 ++++++++++------------------ tests/test_config_parser.py | 12 ++--- 2 files changed, 28 insertions(+), 56 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index 106d8529ff..22715cbc08 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -19,13 +19,13 @@ class ConfigParser: """ - Parse a config content, access or update the items of config content with unique ID. + Parse a config source, access or update the content of the config source with unique ID. A typical usage is a config dictionary contains all the necessary information to define training workflow in JSON. - For more details of the config format, please check :py:class:`monai.apps.ConfigComponent`. + For more details of the config format, please check :py:class:`monai.apps.ConfigItem`. - It can recursively parse the config content, treat every item as a `ConfigItem` with unique ID, the ID is joined - by "#" mark for nested content. For example: - The config content `{"preprocessing": [{"": "LoadImage", "": {"keys": "image"}}]}` is parsed as items: + It can recursively parse the config source, treat every item as a `ConfigItem` with unique ID, the ID is joined + by "#" mark for nested items. For example: + The config source `{"preprocessing": [{"": "LoadImage", "": {"keys": "image"}}]}` is parsed as items: - `id="preprocessing", config=[{"": "LoadImage", "": {"keys": "image"}}]` - `id="preprocessing#0", config={"": "LoadImage", "": {"keys": "image"}}` - `id="preprocessing#0#", config="LoadImage"` @@ -34,8 +34,8 @@ class ConfigParser: A typical workflow of config parsing is as follows: - - Initialize `ConfigParser` with the `config` content. - - Call ``get_resolved_content()`` to get expected component with `id`, which is automatically parsed. + - Initialize `ConfigParser` with the `config` source. + - Call ``get_parsed_content()`` to get expected component with `id`, which will be automatically parsed. .. code-block:: python @@ -45,12 +45,10 @@ class ConfigParser: "trainer": {"": "SupervisedTrainer", "": {"network": "@net", ...}}, } parser = ConfigParser(config=config) - trainer = parser.get_resolved_content(id="trainer") + trainer = parser.get_parsed_content(id="trainer") trainer.run() - It's also flexible to modify config content and do `lazy instantiation` from 2 levels: - - 1. Modify original config content, then totally parse again: + It's also flexible to modify config source at runtime and parse again: .. code-block:: python @@ -58,22 +56,12 @@ class ConfigParser: config = parser.get_config() config["processing"][2][""]["interp_order"] = "bilinear" parser.parse_config() - - 2. After parsing, all the config items are independent `ConfigItem`, get the expected `ConfigItem`. - Its config content is already resolved references, can modify and use it directly: - - .. code-block:: python - - parser = ConfigParser(...) - trainer_config_item = parser.get_config_item(id="trainer") - config = trainer_config_item.get_config() - config["max_epochs"] = 100 - trainer = trainer_config_item.instantiate() + trainer = parser.get_parsed_content(id="trainer") trainer.run() Args: - config: input config content to parse. - id: specified ID name for the config content. + config: input config source to parse. + id: specified ID name for the config sources. excludes: when importing modules to instantiate components, if any string of the `excludes` exists in the full module name, don't import this module. globals: pre-import packages as global variables to evaluate the python `eval` expressions. @@ -103,7 +91,7 @@ def __init__( def update_config(self, content: Dict[str, Any]): """ - Update config content for the parser, every `key` and `value` in the `content` is corresponding to + Update config source for the parser, every `key` and `value` in the `content` is corresponding to the target `id` position and new config value in order. Nested config id is joined by "#" mark, use index from 0 for list. For example: "transforms#5", "transforms#5##keys", etc. @@ -124,12 +112,12 @@ def update_config(self, content: Dict[str, Any]): def get_config(self, id: str = ""): """ - Get config content of current config, if `id` provided, get the config item with `id`. + Get config source in the parser, if `id` provided, get the config item with `id`. Args: id: id name to specify the expected position, nested config is joined by "#" mark, use index from 0 for list. for example: "transforms#5", "transforms#5##keys", etc. - default to get all the config content. + default to get all the config source data. """ config = self.config @@ -141,10 +129,10 @@ def get_config(self, id: str = ""): def _do_parse(self, config, id: str = ""): """ - Recursively parse the nested config content, add every config item to the resolver. + Recursively parse the nested data in config source, add every config item to the resolver. Args: - config: config content to parse. + config: config source to parse. id: id name of current config item, nested ids are joined by "#" mark. defaults to None. for example: "transforms#5", "transforms#5##keys", etc. default to empty string. @@ -168,18 +156,20 @@ def _do_parse(self, config, id: str = ""): def parse_config(self): """ - Parse the config content, add every config item to the resolver. + Parse the config source, add every config item to the resolver. """ self.reference_resolver.reset() self._do_parse(config=self.config) - def get_resolved_content(self, id: str): + def get_parsed_content(self, id: str): """ - Get the resolved result of config items with specified id, if not resolved, try to resolve it first. - If the config item is instantiable, the resolved result is the instance. - If the config item is an expression, the resolved result is output of when evaluating the expression. - Otherwise, the resolved result is the updated config content of the config item. + Get the parsed result of config item with specified id, if having references not resolved, + try to resolve it first. + + If the config item is `ConfigComponent`, the parsed result is the instance. + If the config item is `ConfigExpression`, the parsed result is output of evaluating the expression. + Otherwise, the parsed result is the updated `self.config` data of `ConfigItem`. Args: id: id name of expected config item, nested ids are joined by "#" mark. @@ -187,15 +177,3 @@ def get_resolved_content(self, id: str): """ return self.reference_resolver.get_resolved_content(id=id) - - def get_config_item(self, id: str): - """ - Get the parsed config item which is already resolved all the references. - It can be used to modify the config in other program and support lazy instantiation. - - Args: - id: id name of expected config component, nested ids are joined by "#" mark. - for example: "transforms#5", "transforms#5##keys", etc. - - """ - return self.reference_resolver.get_item(id=id, resolve=True) diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 46bbc98444..11de6c52be 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -15,7 +15,6 @@ from parameterized import parameterized from monai.apps import ConfigParser -from monai.apps.manifest.config_item import ConfigComponent from monai.data import DataLoader, Dataset from monai.transforms import Compose, LoadImaged, RandTorchVisiond from monai.utils import min_version, optional_import @@ -95,21 +94,16 @@ def test_parse(self, config, expected_ids, output_types): config = parser.get_config() config["transform"][""]["transforms"][0][""]["keys"] = "label" parser.parse_config() - self.assertEqual(parser.get_resolved_content(id="transform##transforms#0").keys[0], "label") + self.assertEqual(parser.get_parsed_content(id="transform##transforms#0").keys[0], "label") for id, cls in zip(expected_ids, output_types): - item = parser.get_config_item(id=id) - # test lazy instantiation with resolved config item - if isinstance(item, ConfigComponent): - self.assertTrue(isinstance(item.instantiate(), cls)) - # test get instance directly - self.assertTrue(isinstance(parser.get_resolved_content(id), cls)) + self.assertTrue(isinstance(parser.get_parsed_content(id), cls)) @parameterized.expand([TEST_CASE_2]) def test_function(self, config): parser = ConfigParser(config=config, globals={"TestClass": TestClass}) for id in config: - func = parser.get_resolved_content(id=id) + func = parser.get_parsed_content(id=id) self.assertTrue(id in parser.reference_resolver.resolved_content) if id == "error_func": with self.assertRaises(TypeError): From 362e33660ea13611c100baae9319e580871300f9 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Wed, 23 Feb 2022 23:28:12 +0800 Subject: [PATCH 08/21] [DLMED] simplify usage APIs Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 111 ++++++++++++++++----------- tests/test_config_parser.py | 31 +++++--- 2 files changed, 87 insertions(+), 55 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index 22715cbc08..921a7daadb 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -35,7 +35,7 @@ class ConfigParser: A typical workflow of config parsing is as follows: - Initialize `ConfigParser` with the `config` source. - - Call ``get_parsed_content()`` to get expected component with `id`, which will be automatically parsed. + - Call ``get_parsed_content()`` to get expected component with `id`. .. code-block:: python @@ -53,15 +53,12 @@ class ConfigParser: .. code-block:: python parser = ConfigParser(...) - config = parser.get_config() - config["processing"][2][""]["interp_order"] = "bilinear" - parser.parse_config() + parser["processing"][2][""]["interp_order"] = "bilinear" trainer = parser.get_parsed_content(id="trainer") trainer.run() Args: config: input config source to parse. - id: specified ID name for the config sources. excludes: when importing modules to instantiate components, if any string of the `excludes` exists in the full module name, don't import this module. globals: pre-import packages as global variables to evaluate the python `eval` expressions. @@ -87,54 +84,80 @@ def __init__( self.locator = ComponentLocator(excludes=excludes) self.reference_resolver = ReferenceResolver() - self.update_config({"": config}) + self.set(config=config) - def update_config(self, content: Dict[str, Any]): + def __getitem__(self, id: Union[str, int]): """ - Update config source for the parser, every `key` and `value` in the `content` is corresponding to - the target `id` position and new config value in order. - Nested config id is joined by "#" mark, use index from 0 for list. - For example: "transforms#5", "transforms#5##keys", etc. - If `id` is `""`, replace `self.config`. + Get config source in the parser with provided `id` for target position. + + Args: + id: id name to specify the expected position, nested config is joined by "#" mark, + use string or int index from 0 for list, for example: "transforms#5", "transforms#5##keys". + if `id=""`, get all the config source data in `self.config`. """ - for id, config in content.items(): - if id != "" and isinstance(self.config, (dict, list)): - keys = id.split("#") - # get the last second config item and replace it - last_id = "#".join(keys[:-1]) - conf_ = self.get_config(id=last_id) - conf_[keys[-1] if isinstance(conf_, dict) else int(keys[-1])] = config - else: - self.config = config - # must totally parse again as the content is modified - self.parse_config() + config = self.config + if id != "": + keys = str(id).split("#") + for k in keys: + if not isinstance(config, (dict, list)): + raise ValueError(f"config must be dict or list for key `{k}`, but got: {config}.") + config = config[k] if isinstance(config, dict) else config[int(k)] + return config + + def __setitem__(self, id: Union[str, int], config: Any): + """ + Set config source for the parser at target position `id``. + Nested config `id` is joined by "#" mark, use string or int index from 0 for list item. + For example: "transforms#5", "transforms#5##keys". + If `id` is `""`, replace all the config source data in `self.config`. + Must totally parse again as the config source is modified. - def get_config(self, id: str = ""): """ - Get config source in the parser, if `id` provided, get the config item with `id`. + if id != "": + keys = str(id).split("#") + # get the last second config item and replace it + last_id = "#".join(keys[:-1]) + conf_ = self[last_id] + conf_[keys[-1] if isinstance(conf_, dict) else int(keys[-1])] = config + else: + self.config = config + self.reference_resolver.reset() + + def get(self, id: str = "", default: Optional[Any] = None): + """ + Get config source in the parser with provided `id` for target position. Args: id: id name to specify the expected position, nested config is joined by "#" mark, use index from 0 for list. - for example: "transforms#5", "transforms#5##keys", etc. - default to get all the config source data. + for example: "transforms#5", "transforms#5##keys". + default to get all the config source data in `self.config`. + default: default value to return if the specified `id` is invalid. """ - config = self.config - if id != "" and isinstance(config, (dict, list)): - keys = id.split("#") - for k in keys: - config = config[k] if isinstance(config, dict) else config[int(k)] - return config + try: + return self[id] + except KeyError: + return default + + def set(self, config: Any, id: str = ""): + """ + Set config source for the parser at target position `id``, nested config id is joined by "#" mark, + use index from 0 for list. For example: "transforms#5", "transforms#5##keys". + If `id` is `""`, replace all the config source data in `self.config`. + Must totally parse again as the config source is modified. + + """ + self[id] = config def _do_parse(self, config, id: str = ""): """ - Recursively parse the nested data in config source, add every config item to the resolver. + Recursively parse the nested data in config source, add every item as `ConfigItem` to the resolver. Args: config: config source to parse. id: id name of current config item, nested ids are joined by "#" mark. defaults to None. - for example: "transforms#5", "transforms#5##keys", etc. + for example: "transforms#5", "transforms#5##keys". default to empty string. """ @@ -154,26 +177,28 @@ def _do_parse(self, config, id: str = ""): else: self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) - def parse_config(self): + def parse(self): """ - Parse the config source, add every config item to the resolver. + Recursively parse the config source, add every item as `ConfigItem` to the resolver. """ - self.reference_resolver.reset() self._do_parse(config=self.config) def get_parsed_content(self, id: str): """ - Get the parsed result of config item with specified id, if having references not resolved, + Get the parsed result of `ConfigItem` with specified `id`, if having references not resolved, try to resolve it first. - If the config item is `ConfigComponent`, the parsed result is the instance. - If the config item is `ConfigExpression`, the parsed result is output of evaluating the expression. + If the item is `ConfigComponent`, the parsed result is the instance. + If the item is `ConfigExpression`, the parsed result is output of evaluating the expression. Otherwise, the parsed result is the updated `self.config` data of `ConfigItem`. Args: - id: id name of expected config item, nested ids are joined by "#" mark. - for example: "transforms#5", "transforms#5##keys", etc. + id: id name of expected `ConfigItem`, nested items are joined by "#" mark as the `id`. + for example: "transforms#5", "transforms#5##keys". """ + if len(self.reference_resolver.resolved_content) == 0: + # not parsed the config source yet, parse it + self.parse() return self.reference_resolver.get_resolved_content(id=id) diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 11de6c52be..21f874c1c5 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -77,25 +77,32 @@ def __call__(self, a, b): class TestConfigComponent(unittest.TestCase): def test_config_content(self): - parser = ConfigParser(config={}) test_config = {"preprocessing": [{"": "LoadImage"}], "dataset": {"": "Dataset"}} - parser.update_config(content={"": test_config}) - self.assertEqual(str(parser.get_config()), str(test_config)) - parser.update_config(content={"dataset": {"": "CacheDataset"}}) - self.assertDictEqual(parser.get_config(id="dataset"), {"": "CacheDataset"}) - parser.update_config(content={"dataset#": "Dataset"}) - self.assertEqual(parser.get_config(id="dataset#"), "Dataset") + parser = ConfigParser(config=test_config) + # test `get`, `set`, `__getitem__`, `__setitem__` + self.assertEqual(str(parser.get()), str(test_config)) + parser.set(config=test_config) + self.assertListEqual(parser["preprocessing"], test_config["preprocessing"]) + parser["dataset"] = {"": "CacheDataset"} + self.assertEqual(parser["dataset"][""], "CacheDataset") + # test nested ids + parser["dataset#"] = "Dataset" + self.assertEqual(parser["dataset#"], "Dataset") + # test int id + parser.set(["test1", "test2", "test3"]) + parser[1] = "test4" + self.assertEqual(parser[1], "test4") @parameterized.expand([TEST_CASE_1]) @skipUnless(has_tv, "Requires torchvision >= 0.8.0.") def test_parse(self, config, expected_ids, output_types): parser = ConfigParser(config=config, globals={"monai": "monai"}) # test lazy instantiation with original config content - config = parser.get_config() - config["transform"][""]["transforms"][0][""]["keys"] = "label" - parser.parse_config() - self.assertEqual(parser.get_parsed_content(id="transform##transforms#0").keys[0], "label") - + parser["transform"][""]["transforms"][0][""]["keys"] = "label1" + self.assertEqual(parser.get_parsed_content(id="transform##transforms#0").keys[0], "label1") + # test nested id + parser["transform##transforms#0##keys"] = "label2" + self.assertEqual(parser.get_parsed_content(id="transform##transforms#0").keys[0], "label2") for id, cls in zip(expected_ids, output_types): self.assertTrue(isinstance(parser.get_parsed_content(id), cls)) From e7559c5753969486e3e6c72180931d4d48956ace Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 24 Feb 2022 00:04:55 +0800 Subject: [PATCH 09/21] [DLMED] support root config Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 17 ++++++++--------- monai/apps/manifest/reference_resolver.py | 2 -- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index 921a7daadb..ebed087597 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -167,15 +167,14 @@ def _do_parse(self, config, id: str = ""): sub_id = f"{id}#{k}" if id != "" else k self._do_parse(config=v, id=sub_id) - if id != "": - # copy every config item to make them independent and add them to the resolver - item_conf = deepcopy(config) - if ConfigComponent.is_instantiable(item_conf): - self.reference_resolver.add_item(ConfigComponent(config=item_conf, id=id, locator=self.locator)) - elif ConfigExpression.is_expression(item_conf): - self.reference_resolver.add_item(ConfigExpression(config=item_conf, id=id, globals=self.globals)) - else: - self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) + # copy every config item to make them independent and add them to the resolver + item_conf = deepcopy(config) + if ConfigComponent.is_instantiable(item_conf): + self.reference_resolver.add_item(ConfigComponent(config=item_conf, id=id, locator=self.locator)) + elif ConfigExpression.is_expression(item_conf): + self.reference_resolver.add_item(ConfigExpression(config=item_conf, id=id, globals=self.globals)) + else: + self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) def parse(self): """ diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 56177a5532..746c7d0823 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -64,8 +64,6 @@ def add_item(self, item: ConfigItem): """ id = item.get_id() - if id == "": - raise ValueError("id should not be empty when resolving reference.") if id in self.items: warnings.warn(f"id '{id}' is already added.") return From 420346bae8b4b189c77221941f36f3843d3ec987 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 24 Feb 2022 00:23:48 +0800 Subject: [PATCH 10/21] [DLMED] fix update reference typo Signed-off-by: Nic Ma --- monai/apps/manifest/reference_resolver.py | 4 ++-- tests/test_config_parser.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 746c7d0823..a19204975c 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -116,7 +116,7 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None): if d not in self.resolved_content: # this referring item is not resolved if d not in self.items: - raise ValueError(f"the referring item `{d}` is not defined in config.") + raise ValueError(f"the referring item `{d}` is not defined in the config content.") # recursively resolve the reference first self._resolve_one_item(id=d, waiting_list=waiting_list) @@ -235,7 +235,7 @@ def update_config_with_refs(config, id: str, refs: Optional[Dict] = None): for idx, v in config.items() if isinstance(config, dict) else enumerate(config): sub_id = f"{id}#{idx}" if id != "" else f"{idx}" if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): - updated = ReferenceResolver.update_config_with_refs(v, sub_id, refs_) + updated = refs_[sub_id] else: updated = ReferenceResolver.update_config_with_refs(v, sub_id, refs_) ret.update({idx: updated}) if isinstance(ret, dict) else ret.append(updated) diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 21f874c1c5..dcb4e0801e 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -105,6 +105,11 @@ def test_parse(self, config, expected_ids, output_types): self.assertEqual(parser.get_parsed_content(id="transform##transforms#0").keys[0], "label2") for id, cls in zip(expected_ids, output_types): self.assertTrue(isinstance(parser.get_parsed_content(id), cls)) + # test root content + root = parser.get_parsed_content(id="") + print("!!!!!!!!", root) + for v, cls in zip(root.values(), [Compose, Dataset, DataLoader]): + self.assertTrue(isinstance(v, cls)) @parameterized.expand([TEST_CASE_2]) def test_function(self, config): From 3c1205c2aa4e9dc98c02f3165b94fb1df7a722dd Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 23 Feb 2022 22:50:40 +0000 Subject: [PATCH 11/21] revise APIs and docstrings Signed-off-by: Wenqi Li --- docs/source/apps.rst | 1 + monai/apps/manifest/config_item.py | 3 + monai/apps/manifest/config_parser.py | 176 ++++++++++++---------- monai/apps/manifest/reference_resolver.py | 46 +++--- monai/utils/module.py | 7 +- tests/test_config_parser.py | 1 - 6 files changed, 132 insertions(+), 102 deletions(-) diff --git a/docs/source/apps.rst b/docs/source/apps.rst index d0fe131d85..8c0214d584 100644 --- a/docs/source/apps.rst +++ b/docs/source/apps.rst @@ -46,6 +46,7 @@ Model Manifest .. autoclass:: ConfigParser :members: + :special-members: __getitem__, __setitem__ .. autoclass:: ReferenceResolver :members: diff --git a/monai/apps/manifest/config_item.py b/monai/apps/manifest/config_item.py index a25dfcedf2..a6406795cf 100644 --- a/monai/apps/manifest/config_item.py +++ b/monai/apps/manifest/config_item.py @@ -153,6 +153,9 @@ def get_config(self): """ return self.config + def __repr__(self) -> str: + return str(self.config) + class ConfigComponent(ConfigItem, Instantiable): """ diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index ebed087597..1a90696e87 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -19,53 +19,58 @@ class ConfigParser: """ - Parse a config source, access or update the content of the config source with unique ID. - A typical usage is a config dictionary contains all the necessary information to define training workflow in JSON. - For more details of the config format, please check :py:class:`monai.apps.ConfigItem`. - - It can recursively parse the config source, treat every item as a `ConfigItem` with unique ID, the ID is joined - by "#" mark for nested items. For example: - The config source `{"preprocessing": [{"": "LoadImage", "": {"keys": "image"}}]}` is parsed as items: - - `id="preprocessing", config=[{"": "LoadImage", "": {"keys": "image"}}]` - - `id="preprocessing#0", config={"": "LoadImage", "": {"keys": "image"}}` - - `id="preprocessing#0#", config="LoadImage"` - - `id="preprocessing#0#", config={"keys": "image"}` - - `id="preprocessing#0##keys", config="image"` + The primary configuration parser. It traverses a structured config (in the form of nested Python dict or list), + creates ``ConfigItem``, and assign unique IDs according to the structures. + This class provides convenient access to the set of ``ConfigItem`` of the config by ID. A typical workflow of config parsing is as follows: - - Initialize `ConfigParser` with the `config` source. - - Call ``get_parsed_content()`` to get expected component with `id`. + - Initialize ``ConfigParser`` with the ``config`` source. + - Call ``get_parsed_content()`` to get expected component with `id`. .. code-block:: python + from monai.apps import ConfigParser + + config = { - "preprocessing": {"": "LoadImage"}, - "net": {"": "UNet", "": ...}, - "trainer": {"": "SupervisedTrainer", "": {"network": "@net", ...}}, + "my_dims": 2, + "dims_1": "$@my_dims + 1", + "my_xform": {"": "LoadImage"}, + "my_net": {"": "BasicUNet", + "": {"spatial_dims": "@dims_1", "in_channels": 1, "out_channels": 4}}, + "trainer": {"": "SupervisedTrainer", + "": {"network": "@my_net", "preprocessing": "@my_xform"}} } - parser = ConfigParser(config=config) - trainer = parser.get_parsed_content(id="trainer") - trainer.run() + # in the example $@my_dims + 1 is an expression, which adds 1 to the value of @my_dims + parser = ConfigParser(config) - It's also flexible to modify config source at runtime and parse again: + # get/set the configuration content, but do not instantiate the components + trainer = parser.get_parsed_content("trainer", instantiate=False) + print(trainer) + print(parser["my_net"][""]["in_channels"]) # original input channels 1 + parser["my_net"][""]["in_channels"] = 4 # change input channels to 4 + print(parser["my_net"][""]["in_channels"]) - .. code-block:: python + # instantiate the network component + parser.parse(True) + net = parser.get_parsed_content("my_net", instantiate=True) + print(net) - parser = ConfigParser(...) - parser["processing"][2][""]["interp_order"] = "bilinear" - trainer = parser.get_parsed_content(id="trainer") - trainer.run() Args: config: input config source to parse. - excludes: when importing modules to instantiate components, if any string of the `excludes` exists - in the full module name, don't import this module. - globals: pre-import packages as global variables to evaluate the python `eval` expressions. - for example, pre-import `monai`, then execute `eval("monai.data.list_data_collate")`. - default to `{"monai": "monai", "torch": "torch", "np": "numpy"}` as `numpy` and `torch` - are MONAI mininum requirements. - if the value in global is string, will import it immediately. + excludes: when importing modules to instantiate components, + excluding components from modules specified in ``excludes``. + globals: pre-import packages as global variables to ``ConfigExpression``, + so that expressions, for example, ``"$monai.data.list_data_collate"`` can use ``monai`` modules. + The current supported globals and alias names are + ``{"monai": "monai", "torch": "torch", "np": "numpy", "numpy": "numpy"}``. + These are MONAI's minimal dependencies. + + See also: + + - :py:class:`monai.apps.ConfigItem` """ @@ -77,7 +82,7 @@ def __init__( ): self.config = None self.globals: Dict[str, Any] = {} - globals = {"monai": "monai", "torch": "torch", "np": "numpy"} if globals is None else globals + globals = {"monai": "monai", "torch": "torch", "np": "numpy", "numpy": "numpy"} if globals is None else globals if globals is not None: for k, v in globals.items(): self.globals[k] = importlib.import_module(v) if isinstance(v, str) else v @@ -88,51 +93,57 @@ def __init__( def __getitem__(self, id: Union[str, int]): """ - Get config source in the parser with provided `id` for target position. + Get the config by id. Args: - id: id name to specify the expected position, nested config is joined by "#" mark, - use string or int index from 0 for list, for example: "transforms#5", "transforms#5##keys". - if `id=""`, get all the config source data in `self.config`. + id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to + go one level further into the nested structures. + Use digits indexing from "0" for list or other strings for dict. + For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. """ + if id == "": + return self.config config = self.config - if id != "": - keys = str(id).split("#") - for k in keys: - if not isinstance(config, (dict, list)): - raise ValueError(f"config must be dict or list for key `{k}`, but got: {config}.") - config = config[k] if isinstance(config, dict) else config[int(k)] + for k in str(id).split("#"): + if not isinstance(config, (dict, list)): + raise ValueError(f"config must be dict or list for key `{k}`, but got {type(config)}: {config}.") + indexing = k if isinstance(config, dict) else int(k) + config = config[indexing] return config def __setitem__(self, id: Union[str, int], config: Any): """ - Set config source for the parser at target position `id``. - Nested config `id` is joined by "#" mark, use string or int index from 0 for list item. - For example: "transforms#5", "transforms#5##keys". - If `id` is `""`, replace all the config source data in `self.config`. - Must totally parse again as the config source is modified. + Set config by ``id``. + + Args: + id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to + go one level further into the nested structures. + Use digits indexing from "0" for list or other strings for dict. + For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. + config: config to set at location ``id``. """ - if id != "": - keys = str(id).split("#") - # get the last second config item and replace it - last_id = "#".join(keys[:-1]) - conf_ = self[last_id] - conf_[keys[-1] if isinstance(conf_, dict) else int(keys[-1])] = config - else: + if id == "": self.config = config + self.reference_resolver.reset() + return + keys = str(id).split("#") + # get the last second config item and replace it + last_id = "#".join(keys[:-1]) + conf_ = self[last_id] + indexing = keys[-1] if isinstance(conf_, dict) else int(keys[-1]) + conf_[indexing] = config self.reference_resolver.reset() + return def get(self, id: str = "", default: Optional[Any] = None): """ - Get config source in the parser with provided `id` for target position. + Get the config by id. Args: - id: id name to specify the expected position, nested config is joined by "#" mark, use index from 0 for list. - for example: "transforms#5", "transforms#5##keys". - default to get all the config source data in `self.config`. - default: default value to return if the specified `id` is invalid. + id: id to specify the expected position. See also :py:meth:`__getitem__`. + default: default value to return if the specified ``id`` is invalid. """ try: @@ -142,10 +153,7 @@ def get(self, id: str = "", default: Optional[Any] = None): def set(self, config: Any, id: str = ""): """ - Set config source for the parser at target position `id``, nested config id is joined by "#" mark, - use index from 0 for list. For example: "transforms#5", "transforms#5##keys". - If `id` is `""`, replace all the config source data in `self.config`. - Must totally parse again as the config source is modified. + Set config by ``id``. See also :py:meth:`__setitem__`. """ self[id] = config @@ -156,9 +164,10 @@ def _do_parse(self, config, id: str = ""): Args: config: config source to parse. - id: id name of current config item, nested ids are joined by "#" mark. defaults to None. - for example: "transforms#5", "transforms#5##keys". - default to empty string. + id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to + go one level further into the nested structures. + Use digits indexing from "0" for list or other strings for dict. + For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. """ if isinstance(config, (dict, list)): @@ -176,28 +185,35 @@ def _do_parse(self, config, id: str = ""): else: self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) - def parse(self): + def parse(self, reset: bool = False): """ - Recursively parse the config source, add every item as `ConfigItem` to the resolver. + Recursively parse the config source, add every item as ``ConfigItem`` to the resolver. + + Args: + reset: whether to reset the ``reference_resolver`` before parsing. Defaults to False. """ + if reset: + self.reference_resolver.reset() self._do_parse(config=self.config) - def get_parsed_content(self, id: str): + def get_parsed_content(self, id: str = "", **kwargs): """ - Get the parsed result of `ConfigItem` with specified `id`, if having references not resolved, - try to resolve it first. + Get the parsed result of ``ConfigItem`` with the specified ``id``. - If the item is `ConfigComponent`, the parsed result is the instance. - If the item is `ConfigExpression`, the parsed result is output of evaluating the expression. - Otherwise, the parsed result is the updated `self.config` data of `ConfigItem`. + - If the item is ``ConfigComponent`` and ``instantiate=True``, the result is the instance. + - If the item is ``ConfigExpression`` and ``eval_expr=True``, the result is the evaluated output. Args: - id: id name of expected `ConfigItem`, nested items are joined by "#" mark as the `id`. - for example: "transforms#5", "transforms#5##keys". + id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to + go one level further into the nested structures. + Use digits indexing from "0" for list or other strings for dict. + For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. + kwargs: additional keyword arguments to be passed to ``_resolve_one_item``. + Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. """ - if len(self.reference_resolver.resolved_content) == 0: + if not self.reference_resolver.is_resolved(): # not parsed the config source yet, parse it self.parse() - return self.reference_resolver.get_resolved_content(id=id) + return self.reference_resolver.get_resolved_content(id=id, **kwargs) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index a19204975c..9bb9e43253 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -10,10 +10,10 @@ # limitations under the License. import re -import warnings from typing import Any, Dict, Optional, Sequence, Set from monai.apps.manifest.config_item import ConfigComponent, ConfigExpression, ConfigItem +from monai.utils import look_up_option class ReferenceResolver: @@ -55,6 +55,9 @@ def reset(self): self.items = {} self.resolved_content = {} + def is_resolved(self) -> bool: + return bool(self.resolved_content) + def add_item(self, item: ConfigItem): """ Add a ``ConfigItem`` to the resolver. @@ -65,11 +68,11 @@ def add_item(self, item: ConfigItem): """ id = item.get_id() if id in self.items: - warnings.warn(f"id '{id}' is already added.") + # warnings.warn(f"id '{id}' is already added.") return self.items[id] = item - def get_item(self, id: str, resolve: bool = False): + def get_item(self, id: str, resolve: bool = False, **kwargs): """ Get the ``ConfigItem`` by id. @@ -79,13 +82,14 @@ def get_item(self, id: str, resolve: bool = False): Args: id: id of the expected config item. resolve: whether to resolve the item if it is not resolved, default to False. - + kwargs: keyword arguments to pass to ``_resolve_one_item()``. + Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. """ if resolve and id not in self.resolved_content: - self._resolve_one_item(id=id) + self._resolve_one_item(id=id, **kwargs) return self.items.get(id) - def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None): + def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None, **kwargs): """ Resolve one ``ConfigItem`` of ``id``, cache the resolved result in ``resolved_content``. If it has unresolved references, recursively resolve the referring items first. @@ -95,6 +99,8 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None): waiting_list: set of ids pending to be resolved. It's used to detect circular references such as: `{"name": "A", "dep": "@B"}` and `{"name": "B", "dep": "@A"}`. + kwargs: keyword arguments to pass to ``_resolve_one_item()``. + Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. """ item = self.items[id] # if invalid id name, raise KeyError @@ -105,42 +111,46 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None): waiting_list.add(id) ref_ids = self.find_refs_in_config(config=item_config, id=id) - - # if current item has reference already in the waiting list, that's circular references for d in ref_ids: + # if current item has reference already in the waiting list, that's circular references if d in waiting_list: raise ValueError(f"detected circular references for id='{d}' in the config content.") - - # # check whether the component has any unresolved references - for d in ref_ids: + # check whether the component has any unresolved references if d not in self.resolved_content: # this referring item is not resolved - if d not in self.items: - raise ValueError(f"the referring item `{d}` is not defined in the config content.") + try: + look_up_option(d, self.items, print_all_options=False) + except ValueError as err: + raise ValueError(f"the referring item `@{d}` is not defined in the config content.") from err # recursively resolve the reference first - self._resolve_one_item(id=d, waiting_list=waiting_list) + self._resolve_one_item(id=d, waiting_list=waiting_list, **kwargs) + waiting_list.remove(d) # all references are resolved, then try to resolve current config item new_config = self.update_config_with_refs(config=item_config, id=id, refs=self.resolved_content) item.update_config(config=new_config) # save the resolved result into `resolved_content` to recursively resolve others if isinstance(item, ConfigComponent): - self.resolved_content[id] = item.instantiate() + self.resolved_content[id] = item.instantiate() if kwargs.get("instantiate", True) else item elif isinstance(item, ConfigExpression): - self.resolved_content[id] = item.evaluate(locals={"refs": self.resolved_content}) + self.resolved_content[id] = ( + item.evaluate(locals={"refs": self.resolved_content}) if kwargs.get("eval_expr", True) else item + ) else: self.resolved_content[id] = new_config - def get_resolved_content(self, id: str): + def get_resolved_content(self, id: str, **kwargs): """ Get the resolved ``ConfigItem`` by id. If there are unresolved references, try to resolve them first. Args: id: id name of the expected item. + kwargs: additional keyword arguments to be passed to ``_resolve_one_item``. + Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. """ if id not in self.resolved_content: - self._resolve_one_item(id=id) + self._resolve_one_item(id=id, **kwargs) return self.resolved_content[id] @staticmethod diff --git a/monai/utils/module.py b/monai/utils/module.py index ca47ae6b2d..de2152d182 100644 --- a/monai/utils/module.py +++ b/monai/utils/module.py @@ -47,7 +47,7 @@ ] -def look_up_option(opt_str, supported: Union[Collection, enum.EnumMeta], default="no_default"): +def look_up_option(opt_str, supported: Union[Collection, enum.EnumMeta], default="no_default", print_all_options=True): """ Look up the option in the supported collection and return the matched item. Raise a value error possibly with a guess of the closest match. @@ -58,6 +58,7 @@ def look_up_option(opt_str, supported: Union[Collection, enum.EnumMeta], default default: If it is given, this method will return `default` when `opt_str` is not found, instead of raising a `ValueError`. Otherwise, it defaults to `"no_default"`, so that the method may raise a `ValueError`. + print_all_options: whether to print all available options when `opt_str` is not found. Defaults to True Examples: @@ -113,12 +114,12 @@ class Color(Enum): if edit_dist <= 3: edit_dists[key] = edit_dist - supported_msg = f"Available options are {set_to_check}.\n" + supported_msg = f"Available options are {set_to_check}.\n" if print_all_options else "" if edit_dists: guess_at_spelling = min(edit_dists, key=edit_dists.get) # type: ignore raise ValueError( f"By '{opt_str}', did you mean '{guess_at_spelling}'?\n" - + f"'{opt_str}' is not a valid option.\n" + + f"'{opt_str}' is not a valid value.\n" + supported_msg ) raise ValueError(f"Unsupported option '{opt_str}', " + supported_msg) diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index dcb4e0801e..2b58d6959e 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -107,7 +107,6 @@ def test_parse(self, config, expected_ids, output_types): self.assertTrue(isinstance(parser.get_parsed_content(id), cls)) # test root content root = parser.get_parsed_content(id="") - print("!!!!!!!!", root) for v, cls in zip(root.values(), [Compose, Dataset, DataLoader]): self.assertTrue(isinstance(v, cls)) From 43ce0e53685cfdadb0410ddc27a492641bc8a409 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 23 Feb 2022 23:01:36 +0000 Subject: [PATCH 12/21] remove->discard Signed-off-by: Wenqi Li --- monai/apps/manifest/reference_resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 9bb9e43253..212f8640ef 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -124,7 +124,7 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None, ** raise ValueError(f"the referring item `@{d}` is not defined in the config content.") from err # recursively resolve the reference first self._resolve_one_item(id=d, waiting_list=waiting_list, **kwargs) - waiting_list.remove(d) + waiting_list.discard(d) # all references are resolved, then try to resolve current config item new_config = self.update_config_with_refs(config=item_config, id=id, refs=self.resolved_content) From b86f897f4bca8ab2e7af1900cff036fe19604db1 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 24 Feb 2022 18:28:55 +0800 Subject: [PATCH 13/21] [DLMED] update code Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 4 ++-- monai/apps/manifest/reference_resolver.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index 1a90696e87..af6c9f1b9f 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -185,12 +185,12 @@ def _do_parse(self, config, id: str = ""): else: self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) - def parse(self, reset: bool = False): + def parse(self, reset: bool = True): """ Recursively parse the config source, add every item as ``ConfigItem`` to the resolver. Args: - reset: whether to reset the ``reference_resolver`` before parsing. Defaults to False. + reset: whether to reset the ``reference_resolver`` before parsing. Defaults to `True`. """ if reset: diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 212f8640ef..45249668c7 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -68,7 +68,6 @@ def add_item(self, item: ConfigItem): """ id = item.get_id() if id in self.items: - # warnings.warn(f"id '{id}' is already added.") return self.items[id] = item From 28c5c07a47b7412911cbb8b1ad0a41d0a02f14cc Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 24 Feb 2022 18:46:20 +0800 Subject: [PATCH 14/21] [DLMED] update doc Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index af6c9f1b9f..d58c0b1337 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -45,9 +45,7 @@ class ConfigParser: # in the example $@my_dims + 1 is an expression, which adds 1 to the value of @my_dims parser = ConfigParser(config) - # get/set the configuration content, but do not instantiate the components - trainer = parser.get_parsed_content("trainer", instantiate=False) - print(trainer) + # get/set configuration content print(parser["my_net"][""]["in_channels"]) # original input channels 1 parser["my_net"][""]["in_channels"] = 4 # change input channels to 4 print(parser["my_net"][""]["in_channels"]) @@ -57,6 +55,9 @@ class ConfigParser: net = parser.get_parsed_content("my_net", instantiate=True) print(net) + # also support to get the configuration content of parsed `ConfigItem` + trainer = parser.get_parsed_content("trainer", instantiate=False) + print(trainer) Args: config: input config source to parse. @@ -161,6 +162,8 @@ def set(self, config: Any, id: str = ""): def _do_parse(self, config, id: str = ""): """ Recursively parse the nested data in config source, add every item as `ConfigItem` to the resolver. + Note that the configuration content of every `ConfigItem` will be decoupled from the config source + of parser during parsing. Args: config: config source to parse. @@ -203,6 +206,10 @@ def get_parsed_content(self, id: str = "", **kwargs): - If the item is ``ConfigComponent`` and ``instantiate=True``, the result is the instance. - If the item is ``ConfigExpression`` and ``eval_expr=True``, the result is the evaluated output. + - Else, the result is the configuration content of `ConfigItem`. + + Note that the configuration content of every `ConfigItem` is decoupled from the config source of parser + during parsing. Args: id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to From 7d2903f10658a0b56b368df47148de215b1f0f98 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 24 Feb 2022 10:43:17 +0000 Subject: [PATCH 15/21] update usage Signed-off-by: Wenqi Li --- monai/apps/manifest/config_parser.py | 3 +++ monai/apps/manifest/reference_resolver.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index d58c0b1337..b6f431c7ce 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -92,6 +92,9 @@ def __init__( self.reference_resolver = ReferenceResolver() self.set(config=config) + def __repr__(self): + return f"{self.config}" + def __getitem__(self, id: Union[str, int]): """ Get the config by id. diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 45249668c7..238dfc55ae 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -102,7 +102,10 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None, ** Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. """ - item = self.items[id] # if invalid id name, raise KeyError + try: + item = look_up_option(id, self.items, print_all_options=False) + except ValueError as err: + raise KeyError(f"id='{id}' is not found in the config resolver.") from err item_config = item.get_config() if waiting_list is None: From 337c51dcadb38b0b3aaadf7c76183f6ff281a6dd Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 24 Feb 2022 23:21:15 +0000 Subject: [PATCH 16/21] update regex for Signed-off-by: Wenqi Li --- monai/apps/manifest/reference_resolver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 238dfc55ae..1872292921 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -167,7 +167,7 @@ def match_refs_pattern(value: str) -> Set[str]: """ refs: Set[str] = set() # regular expression pattern to match "@XXX" or "@XXX#YYY" - result = re.compile(r"@\w*[\#\w]*").findall(value) + result = re.compile(r"@(?:(?:<\w*>)|(?:\w*)|)(?:(?:#<\w*>)|(?:#\w*))*").findall(value) for item in result: if ConfigExpression.is_expression(value) or value == item: # only check when string starts with "$" or the whole content is "@XXX" @@ -187,7 +187,7 @@ def update_refs_pattern(value: str, refs: Dict) -> str: """ # regular expression pattern to match "@XXX" or "@XXX#YYY" - result = re.compile(r"@\w*[\#\w]*").findall(value) + result = re.compile(r"@(?:(?:<\w*>)|(?:\w*)|)(?:(?:#<\w*>)|(?:#\w*))*").findall(value) for item in result: ref_id = item[1:] if ref_id not in refs: From 5b693bb387b3fa2f0e1a958b839f9c14851a1875 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 24 Feb 2022 23:29:18 +0000 Subject: [PATCH 17/21] revise docstring Signed-off-by: Wenqi Li --- monai/apps/manifest/config_parser.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index b6f431c7ce..21be2a2089 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -45,7 +45,7 @@ class ConfigParser: # in the example $@my_dims + 1 is an expression, which adds 1 to the value of @my_dims parser = ConfigParser(config) - # get/set configuration content + # get/set configuration content, the set method should happen before calling parse() print(parser["my_net"][""]["in_channels"]) # original input channels 1 parser["my_net"][""]["in_channels"] = 4 # change input channels to 4 print(parser["my_net"][""]["in_channels"]) @@ -118,7 +118,8 @@ def __getitem__(self, id: Union[str, int]): def __setitem__(self, id: Union[str, int], config: Any): """ - Set config by ``id``. + Set config by ``id``. Note that this method should be used before ``parse()`` or ``get_parsed_content()`` + to ensure the updates are included in the parsed content. Args: id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to @@ -165,8 +166,6 @@ def set(self, config: Any, id: str = ""): def _do_parse(self, config, id: str = ""): """ Recursively parse the nested data in config source, add every item as `ConfigItem` to the resolver. - Note that the configuration content of every `ConfigItem` will be decoupled from the config source - of parser during parsing. Args: config: config source to parse. @@ -211,9 +210,6 @@ def get_parsed_content(self, id: str = "", **kwargs): - If the item is ``ConfigExpression`` and ``eval_expr=True``, the result is the evaluated output. - Else, the result is the configuration content of `ConfigItem`. - Note that the configuration content of every `ConfigItem` is decoupled from the config source of parser - during parsing. - Args: id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to go one level further into the nested structures. From 4aadb09d6573c8d08601cd278fbfb8e8c8e7e17f Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 25 Feb 2022 12:11:30 +0000 Subject: [PATCH 18/21] refactor to not hard code special chars Signed-off-by: Wenqi Li --- monai/apps/manifest/config_item.py | 10 +++-- monai/apps/manifest/config_parser.py | 33 +++++++------- monai/apps/manifest/reference_resolver.py | 55 +++++++++++++---------- tests/test_config_parser.py | 3 +- 4 files changed, 56 insertions(+), 45 deletions(-) diff --git a/monai/apps/manifest/config_item.py b/monai/apps/manifest/config_item.py index a6406795cf..69cda4b985 100644 --- a/monai/apps/manifest/config_item.py +++ b/monai/apps/manifest/config_item.py @@ -307,6 +307,8 @@ class ConfigExpression(ConfigItem): """ + prefix = "$" + def __init__(self, config: Any, id: str = "", globals: Optional[Dict] = None) -> None: super().__init__(config=config, id=id) self.globals = globals @@ -323,10 +325,10 @@ def evaluate(self, locals: Optional[Dict] = None): value = self.get_config() if not ConfigExpression.is_expression(value): return None - return eval(value[1:], self.globals, locals) + return eval(value[len(self.prefix) :], self.globals, locals) - @staticmethod - def is_expression(config: Union[Dict, List, str]) -> bool: + @classmethod + def is_expression(cls, config: Union[Dict, List, str]) -> bool: """ Check whether the config is an executable expression string. Currently, a string starts with ``"$"`` character is interpreted as an expression. @@ -335,4 +337,4 @@ def is_expression(config: Union[Dict, List, str]) -> bool: config: input config content to check. """ - return isinstance(config, str) and config.startswith("$") + return isinstance(config, str) and config.startswith(cls.prefix) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index 21be2a2089..145c06718a 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -32,7 +32,6 @@ class ConfigParser: from monai.apps import ConfigParser - config = { "my_dims": 2, "dims_1": "$@my_dims + 1", @@ -89,7 +88,7 @@ def __init__( self.globals[k] = importlib.import_module(v) if isinstance(v, str) else v self.locator = ComponentLocator(excludes=excludes) - self.reference_resolver = ReferenceResolver() + self.ref_resolver = ReferenceResolver() self.set(config=config) def __repr__(self): @@ -109,7 +108,7 @@ def __getitem__(self, id: Union[str, int]): if id == "": return self.config config = self.config - for k in str(id).split("#"): + for k in str(id).split(self.ref_resolver.sep): if not isinstance(config, (dict, list)): raise ValueError(f"config must be dict or list for key `{k}`, but got {type(config)}: {config}.") indexing = k if isinstance(config, dict) else int(k) @@ -131,15 +130,15 @@ def __setitem__(self, id: Union[str, int], config: Any): """ if id == "": self.config = config - self.reference_resolver.reset() + self.ref_resolver.reset() return - keys = str(id).split("#") - # get the last second config item and replace it - last_id = "#".join(keys[:-1]) + keys = str(id).split(self.ref_resolver.sep) + # get the last parent level config item and replace it + last_id = self.ref_resolver.sep.join(keys[:-1]) conf_ = self[last_id] indexing = keys[-1] if isinstance(conf_, dict) else int(keys[-1]) conf_[indexing] = config - self.reference_resolver.reset() + self.ref_resolver.reset() return def get(self, id: str = "", default: Optional[Any] = None): @@ -178,17 +177,17 @@ def _do_parse(self, config, id: str = ""): if isinstance(config, (dict, list)): subs = enumerate(config) if isinstance(config, list) else config.items() for k, v in subs: - sub_id = f"{id}#{k}" if id != "" else k + sub_id = f"{id}{self.ref_resolver.sep}{k}" if id != "" else k self._do_parse(config=v, id=sub_id) # copy every config item to make them independent and add them to the resolver item_conf = deepcopy(config) if ConfigComponent.is_instantiable(item_conf): - self.reference_resolver.add_item(ConfigComponent(config=item_conf, id=id, locator=self.locator)) + self.ref_resolver.add_item(ConfigComponent(config=item_conf, id=id, locator=self.locator)) elif ConfigExpression.is_expression(item_conf): - self.reference_resolver.add_item(ConfigExpression(config=item_conf, id=id, globals=self.globals)) + self.ref_resolver.add_item(ConfigExpression(config=item_conf, id=id, globals=self.globals)) else: - self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) + self.ref_resolver.add_item(ConfigItem(config=item_conf, id=id)) def parse(self, reset: bool = True): """ @@ -199,7 +198,7 @@ def parse(self, reset: bool = True): """ if reset: - self.reference_resolver.reset() + self.ref_resolver.reset() self._do_parse(config=self.config) def get_parsed_content(self, id: str = "", **kwargs): @@ -216,10 +215,10 @@ def get_parsed_content(self, id: str = "", **kwargs): Use digits indexing from "0" for list or other strings for dict. For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. kwargs: additional keyword arguments to be passed to ``_resolve_one_item``. - Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. + Currently support ``reset`` (for parse), ``instantiate`` and ``eval_expr``. All defaulting to True. """ - if not self.reference_resolver.is_resolved(): + if not self.ref_resolver.is_resolved(): # not parsed the config source yet, parse it - self.parse() - return self.reference_resolver.get_resolved_content(id=id, **kwargs) + self.parse(kwargs.get("reset", True)) + return self.ref_resolver.get_resolved_content(id=id, **kwargs) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 1872292921..dcfae1819a 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -42,6 +42,12 @@ class ReferenceResolver: """ + _vars = "__local_refs" + sep = "#" # separator for key indexing + ref = "@" # reference prefix + # match a reference string, e.g. "@id#key", "@id#key#0", "@##key" + id_matcher = re.compile(rf"{ref}(?:(?:<\w*>)|(?:\w*))(?:(?:{sep}<\w*>)|(?:{sep}\w*))*") + def __init__(self, items: Optional[Sequence[ConfigItem]] = None): # save the items in a dictionary with the `ConfigItem.id` as key self.items: Dict[str, Any] = {} if items is None else {i.get_id(): i for i in items} @@ -116,7 +122,7 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None, ** for d in ref_ids: # if current item has reference already in the waiting list, that's circular references if d in waiting_list: - raise ValueError(f"detected circular references for id='{d}' in the config content.") + raise ValueError(f"detected circular references '{d}' for id='{id}' in the config content.") # check whether the component has any unresolved references if d not in self.resolved_content: # this referring item is not resolved @@ -135,8 +141,9 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None, ** if isinstance(item, ConfigComponent): self.resolved_content[id] = item.instantiate() if kwargs.get("instantiate", True) else item elif isinstance(item, ConfigExpression): + run_eval = kwargs.get("eval_expr", True) self.resolved_content[id] = ( - item.evaluate(locals={"refs": self.resolved_content}) if kwargs.get("eval_expr", True) else item + item.evaluate(locals={f"{self._vars}": self.resolved_content}) if run_eval else item ) else: self.resolved_content[id] = new_config @@ -155,8 +162,8 @@ def get_resolved_content(self, id: str, **kwargs): self._resolve_one_item(id=id, **kwargs) return self.resolved_content[id] - @staticmethod - def match_refs_pattern(value: str) -> Set[str]: + @classmethod + def match_refs_pattern(cls, value: str) -> Set[str]: """ Match regular expression for the input string to find the references. The reference string starts with ``"@"``, like: ``"@XXX#YYY#ZZZ"``. @@ -167,15 +174,16 @@ def match_refs_pattern(value: str) -> Set[str]: """ refs: Set[str] = set() # regular expression pattern to match "@XXX" or "@XXX#YYY" - result = re.compile(r"@(?:(?:<\w*>)|(?:\w*)|)(?:(?:#<\w*>)|(?:#\w*))*").findall(value) + result = cls.id_matcher.findall(value) + value_is_expr = ConfigExpression.is_expression(value) for item in result: - if ConfigExpression.is_expression(value) or value == item: + if value_is_expr or value == item: # only check when string starts with "$" or the whole content is "@XXX" - refs.add(item[1:]) + refs.add(item[len(cls.ref) :]) return refs - @staticmethod - def update_refs_pattern(value: str, refs: Dict) -> str: + @classmethod + def update_refs_pattern(cls, value: str, refs: Dict) -> str: """ Match regular expression for the input string to update content with the references. The reference part starts with ``"@"``, like: ``"@XXX#YYY#ZZZ"``. @@ -187,21 +195,22 @@ def update_refs_pattern(value: str, refs: Dict) -> str: """ # regular expression pattern to match "@XXX" or "@XXX#YYY" - result = re.compile(r"@(?:(?:<\w*>)|(?:\w*)|)(?:(?:#<\w*>)|(?:#\w*))*").findall(value) + result = cls.id_matcher.findall(value) + value_is_expr = ConfigExpression.is_expression(value) for item in result: - ref_id = item[1:] + ref_id = item[len(cls.ref) :] # remove the ref prefix "@" if ref_id not in refs: raise KeyError(f"can not find expected ID '{ref_id}' in the references.") - if ConfigExpression.is_expression(value): + if value_is_expr: # replace with local code, will be used in the `evaluate` logic with `locals={"refs": ...}` - value = value.replace(item, f"refs['{ref_id}']") + value = value.replace(item, f"{cls._vars}['{ref_id}']") elif value == item: # the whole content is "@XXX", it will avoid the case that regular string contains "@" value = refs[ref_id] return value - @staticmethod - def find_refs_in_config(config, id: str, refs: Optional[Set[str]] = None) -> Set[str]: + @classmethod + def find_refs_in_config(cls, config, id: str, refs: Optional[Set[str]] = None) -> Set[str]: """ Recursively search all the content of input config item to get the ids of references. References mean: the IDs of other config items (``"@XXX"`` in this config item), or the @@ -216,18 +225,18 @@ def find_refs_in_config(config, id: str, refs: Optional[Set[str]] = None) -> Set """ refs_: Set[str] = refs or set() if isinstance(config, str): - return refs_.union(ReferenceResolver.match_refs_pattern(value=config)) + return refs_.union(cls.match_refs_pattern(value=config)) if not isinstance(config, (list, dict)): return refs_ for k, v in config.items() if isinstance(config, dict) else enumerate(config): - sub_id = f"{id}#{k}" if id != "" else f"{k}" + sub_id = f"{id}{cls.sep}{k}" if id != "" else f"{k}" if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): refs_.add(sub_id) - refs_ = ReferenceResolver.find_refs_in_config(v, sub_id, refs_) + refs_ = cls.find_refs_in_config(v, sub_id, refs_) return refs_ - @staticmethod - def update_config_with_refs(config, id: str, refs: Optional[Dict] = None): + @classmethod + def update_config_with_refs(cls, config, id: str, refs: Optional[Dict] = None): """ With all the references in ``refs``, update the input config content with references and return the new config. @@ -240,15 +249,15 @@ def update_config_with_refs(config, id: str, refs: Optional[Dict] = None): """ refs_: Dict = refs or {} if isinstance(config, str): - return ReferenceResolver.update_refs_pattern(config, refs_) + return cls.update_refs_pattern(config, refs_) if not isinstance(config, (list, dict)): return config ret = type(config)() for idx, v in config.items() if isinstance(config, dict) else enumerate(config): - sub_id = f"{id}#{idx}" if id != "" else f"{idx}" + sub_id = f"{id}{cls.sep}{idx}" if id != "" else f"{idx}" if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): updated = refs_[sub_id] else: - updated = ReferenceResolver.update_config_with_refs(v, sub_id, refs_) + updated = cls.update_config_with_refs(v, sub_id, refs_) ret.update({idx: updated}) if isinstance(ret, dict) else ret.append(updated) return ret diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 2b58d6959e..6e865eee25 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -71,6 +71,7 @@ def __call__(self, a, b): "cls_compute": {"": "tests.test_config_parser.TestClass.cls_compute", "": {"func": "@basic_func"}}, "call_compute": {"": "tests.test_config_parser.TestClass"}, "error_func": "$TestClass.__call__", + "": "$lambda x, y: x + y", } ] @@ -115,7 +116,7 @@ def test_function(self, config): parser = ConfigParser(config=config, globals={"TestClass": TestClass}) for id in config: func = parser.get_parsed_content(id=id) - self.assertTrue(id in parser.reference_resolver.resolved_content) + self.assertTrue(id in parser.ref_resolver.resolved_content) if id == "error_func": with self.assertRaises(TypeError): func(1, 2) From eabfff0e8520efe8c29ae1f0638d56e4b02a7040 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 25 Feb 2022 14:35:58 +0000 Subject: [PATCH 19/21] flag to switch eval expr Signed-off-by: Wenqi Li --- monai/apps/manifest/config_item.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/monai/apps/manifest/config_item.py b/monai/apps/manifest/config_item.py index 69cda4b985..8a3ffe703a 100644 --- a/monai/apps/manifest/config_item.py +++ b/monai/apps/manifest/config_item.py @@ -10,6 +10,7 @@ # limitations under the License. import inspect +import os import sys import warnings from abc import ABC, abstractmethod @@ -308,6 +309,7 @@ class ConfigExpression(ConfigItem): """ prefix = "$" + run_eval = False if os.environ.get("MONAI_EVAL_EXPR", "1") == "0" else True def __init__(self, config: Any, id: str = "", globals: Optional[Dict] = None) -> None: super().__init__(config=config, id=id) @@ -325,6 +327,8 @@ def evaluate(self, locals: Optional[Dict] = None): value = self.get_config() if not ConfigExpression.is_expression(value): return None + if not self.run_eval: + return f"{value[len(self.prefix) :]}" return eval(value[len(self.prefix) :], self.globals, locals) @classmethod From 5b1fae6fbbd207c9ebc07f496589c7ca0ec7e9f8 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 1 Mar 2022 10:36:55 +0800 Subject: [PATCH 20/21] [DLMED] change apps.manifest to bundle Signed-off-by: Nic Ma --- docs/source/api.rst | 1 + docs/source/apps.rst | 27 ++------------- docs/source/bundle.rst | 34 +++++++++++++++++++ docs/source/conf.py | 1 + monai/README.md | 2 ++ monai/__init__.py | 1 + monai/apps/__init__.py | 1 - monai/{apps/manifest => bundle}/__init__.py | 0 .../{apps/manifest => bundle}/config_item.py | 4 +-- .../manifest => bundle}/config_parser.py | 4 +-- .../manifest => bundle}/reference_resolver.py | 2 +- tests/test_component_locator.py | 2 +- tests/test_config_item.py | 2 +- tests/test_config_parser.py | 2 +- tests/test_reference_resolver.py | 4 +-- 15 files changed, 51 insertions(+), 36 deletions(-) create mode 100644 docs/source/bundle.rst rename monai/{apps/manifest => bundle}/__init__.py (100%) rename monai/{apps/manifest => bundle}/config_item.py (98%) rename monai/{apps/manifest => bundle}/config_parser.py (98%) rename monai/{apps/manifest => bundle}/reference_resolver.py (99%) diff --git a/docs/source/api.rst b/docs/source/api.rst index 0596a25514..c2b19adeb2 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -7,6 +7,7 @@ API Reference :maxdepth: 1 apps + bundle transforms losses networks diff --git a/docs/source/apps.rst b/docs/source/apps.rst index 8c0214d584..239ae9eb17 100644 --- a/docs/source/apps.rst +++ b/docs/source/apps.rst @@ -19,8 +19,8 @@ Applications :members: -Clara MMARs ------------ +`Clara MMARs` +------------- .. autofunction:: download_mmar .. autofunction:: load_from_mmar @@ -29,29 +29,6 @@ Clara MMARs :annotation: -Model Manifest --------------- - -.. autoclass:: ComponentLocator - :members: - -.. autoclass:: ConfigComponent - :members: - -.. autoclass:: ConfigExpression - :members: - -.. autoclass:: ConfigItem - :members: - -.. autoclass:: ConfigParser - :members: - :special-members: __getitem__, __setitem__ - -.. autoclass:: ReferenceResolver - :members: - - `Utilities` ----------- diff --git a/docs/source/bundle.rst b/docs/source/bundle.rst new file mode 100644 index 0000000000..03d4e07d17 --- /dev/null +++ b/docs/source/bundle.rst @@ -0,0 +1,34 @@ +:github_url: https://github.com/Project-MONAI/MONAI + +.. _bundle: + +Model Bundle +============ +.. currentmodule:: monai.bundle + +`Config Item` +------------- +.. autoclass:: Instantiable + :members: + +.. autoclass:: ComponentLocator + :members: + +.. autoclass:: ConfigComponent + :members: + +.. autoclass:: ConfigExpression + :members: + +.. autoclass:: ConfigItem + :members: + +`Reference Resolver` +-------------------- +.. autoclass:: ReferenceResolver + :members: + +`Config Parser` +--------------- +.. autoclass:: ConfigParser + :members: diff --git a/docs/source/conf.py b/docs/source/conf.py index 8af3fe8b75..db0ca11be3 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -40,6 +40,7 @@ "engines", "data", "apps", + "bundle", "config", "handlers", "losses", diff --git a/monai/README.md b/monai/README.md index a224996f38..2c30531bf3 100644 --- a/monai/README.md +++ b/monai/README.md @@ -2,6 +2,8 @@ * **apps**: high level medical domain specific deep learning applications. +* **bundle**: components to build the portable self-descriptive model bundle. + * **config**: for system configuration and diagnostic output. * **csrc**: for C++/CUDA extensions. diff --git a/monai/__init__.py b/monai/__init__.py index 68a232b46d..a823a3e1e2 100644 --- a/monai/__init__.py +++ b/monai/__init__.py @@ -49,6 +49,7 @@ __all__ = [ "apps", + "bundle", "config", "data", "engines", diff --git a/monai/apps/__init__.py b/monai/apps/__init__.py index 51c4003458..893f7877d2 100644 --- a/monai/apps/__init__.py +++ b/monai/apps/__init__.py @@ -10,6 +10,5 @@ # limitations under the License. from .datasets import CrossValidation, DecathlonDataset, MedNISTDataset -from .manifest import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, ConfigParser, ReferenceResolver from .mmars import MODEL_DESC, RemoteMMARKeys, download_mmar, get_model_spec, load_from_mmar from .utils import SUPPORTED_HASH_TYPES, check_hash, download_and_extract, download_url, extractall, get_logger, logger diff --git a/monai/apps/manifest/__init__.py b/monai/bundle/__init__.py similarity index 100% rename from monai/apps/manifest/__init__.py rename to monai/bundle/__init__.py diff --git a/monai/apps/manifest/config_item.py b/monai/bundle/config_item.py similarity index 98% rename from monai/apps/manifest/config_item.py rename to monai/bundle/config_item.py index 8a3ffe703a..44cdd3c634 100644 --- a/monai/apps/manifest/config_item.py +++ b/monai/bundle/config_item.py @@ -191,7 +191,7 @@ class ConfigComponent(ConfigItem, Instantiable): locator: a ``ComponentLocator`` to convert a module name string into the actual python module. if `None`, a ``ComponentLocator(excludes=excludes)`` will be used. excludes: if ``locator`` is None, create a new ``ComponentLocator`` with ``excludes``. - See also: :py:class:`monai.apps.manifest.ComponentLocator`. + See also: :py:class:`monai.bundle.ComponentLocator`. """ @@ -295,7 +295,7 @@ class ConfigExpression(ConfigItem): .. code-block:: python import monai - from monai.apps.manifest import ConfigExpression + from monai.bundle import ConfigExpression config = "$monai.__version__" expression = ConfigExpression(config, id="test", globals={"monai": monai}) diff --git a/monai/apps/manifest/config_parser.py b/monai/bundle/config_parser.py similarity index 98% rename from monai/apps/manifest/config_parser.py rename to monai/bundle/config_parser.py index 145c06718a..5ebcfd03b4 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/bundle/config_parser.py @@ -13,8 +13,8 @@ from copy import deepcopy from typing import Any, Dict, Optional, Sequence, Union -from monai.apps.manifest.config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem -from monai.apps.manifest.reference_resolver import ReferenceResolver +from monai.bundle.config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem +from monai.bundle.reference_resolver import ReferenceResolver class ConfigParser: diff --git a/monai/apps/manifest/reference_resolver.py b/monai/bundle/reference_resolver.py similarity index 99% rename from monai/apps/manifest/reference_resolver.py rename to monai/bundle/reference_resolver.py index dcfae1819a..45d897af05 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/bundle/reference_resolver.py @@ -12,7 +12,7 @@ import re from typing import Any, Dict, Optional, Sequence, Set -from monai.apps.manifest.config_item import ConfigComponent, ConfigExpression, ConfigItem +from monai.bundle.config_item import ConfigComponent, ConfigExpression, ConfigItem from monai.utils import look_up_option diff --git a/tests/test_component_locator.py b/tests/test_component_locator.py index eafb2152d1..ebb2cca7b3 100644 --- a/tests/test_component_locator.py +++ b/tests/test_component_locator.py @@ -12,7 +12,7 @@ import unittest from pydoc import locate -from monai.apps.manifest import ComponentLocator +from monai.bundle import ComponentLocator from monai.utils import optional_import _, has_ignite = optional_import("ignite") diff --git a/tests/test_config_item.py b/tests/test_config_item.py index 0b4fdc51bf..1284efab56 100644 --- a/tests/test_config_item.py +++ b/tests/test_config_item.py @@ -17,7 +17,7 @@ from parameterized import parameterized import monai -from monai.apps import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem +from monai.bundle import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem from monai.data import DataLoader, Dataset from monai.transforms import LoadImaged, RandTorchVisiond from monai.utils import min_version, optional_import diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 6e865eee25..5b5aa2b816 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -14,7 +14,7 @@ from parameterized import parameterized -from monai.apps import ConfigParser +from monai.bundle.config_parser import ConfigParser from monai.data import DataLoader, Dataset from monai.transforms import Compose, LoadImaged, RandTorchVisiond from monai.utils import min_version, optional_import diff --git a/tests/test_reference_resolver.py b/tests/test_reference_resolver.py index 90d0f3ea72..e16a795c40 100644 --- a/tests/test_reference_resolver.py +++ b/tests/test_reference_resolver.py @@ -15,8 +15,8 @@ from parameterized import parameterized import monai -from monai.apps import ConfigComponent, ReferenceResolver -from monai.apps.manifest.config_item import ComponentLocator, ConfigExpression, ConfigItem +from monai.bundle.config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem +from monai.bundle.reference_resolver import ReferenceResolver from monai.data import DataLoader from monai.transforms import LoadImaged, RandTorchVisiond from monai.utils import min_version, optional_import From b6020d97b5ff34001e1bfc84819b674ef29e1fdf Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 1 Mar 2022 10:51:33 +0800 Subject: [PATCH 21/21] [DLMED] add missing component Signed-off-by: Nic Ma --- monai/bundle/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py index b8ddf57f93..68e2d543bb 100644 --- a/monai/bundle/__init__.py +++ b/monai/bundle/__init__.py @@ -9,6 +9,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem +from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, Instantiable from .config_parser import ConfigParser from .reference_resolver import ReferenceResolver