From a3193bae88fac8c1974fca6db129ad0478c2dc59 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Sat, 19 Feb 2022 12:28:32 +0800 Subject: [PATCH 01/76] [DLMED] add config parser Signed-off-by: Nic Ma --- docs/source/apps.rst | 3 + monai/apps/__init__.py | 2 +- monai/apps/manifest/__init__.py | 1 + monai/apps/manifest/config_parser.py | 178 +++++++++++++++++++++++++++ tests/test_config_parser.py | 76 ++++++++++++ tests/test_reference_resolver.py | 2 +- 6 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 monai/apps/manifest/config_parser.py create mode 100644 tests/test_config_parser.py diff --git a/docs/source/apps.rst b/docs/source/apps.rst index 4b1cdc6f43..d0fe131d85 100644 --- a/docs/source/apps.rst +++ b/docs/source/apps.rst @@ -44,6 +44,9 @@ Model Manifest .. autoclass:: ConfigItem :members: +.. autoclass:: ConfigParser + :members: + .. autoclass:: ReferenceResolver :members: diff --git a/monai/apps/__init__.py b/monai/apps/__init__.py index 0f233bc3ef..51c4003458 100644 --- a/monai/apps/__init__.py +++ b/monai/apps/__init__.py @@ -10,6 +10,6 @@ # limitations under the License. from .datasets import CrossValidation, DecathlonDataset, MedNISTDataset -from .manifest import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, ReferenceResolver +from .manifest import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, ConfigParser, ReferenceResolver from .mmars import MODEL_DESC, RemoteMMARKeys, download_mmar, get_model_spec, load_from_mmar from .utils import SUPPORTED_HASH_TYPES, check_hash, download_and_extract, download_url, extractall, get_logger, logger diff --git a/monai/apps/manifest/__init__.py b/monai/apps/manifest/__init__.py index 79c4376d5c..b8ddf57f93 100644 --- a/monai/apps/manifest/__init__.py +++ b/monai/apps/manifest/__init__.py @@ -10,4 +10,5 @@ # limitations under the License. from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem +from .config_parser import ConfigParser from .reference_resolver import ReferenceResolver diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py new file mode 100644 index 0000000000..f7180078cb --- /dev/null +++ b/monai/apps/manifest/config_parser.py @@ -0,0 +1,178 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +from copy import deepcopy +from typing import Any, Dict, Optional, Sequence, Union + +from monai.apps.manifest.config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem +from monai.apps.manifest.reference_resolver import ReferenceResolver + + +class ConfigParser: + """ + Parse a config content, access or update the items of config content with unique ID. + A typical usage is a config dictionary contains all the necessary information to define training workflow in JSON. + For more details of the config format, please check :py:class:`monai.apps.ConfigComponent`. + + It can recursively parse the config content, treat every item as a `ConfigItem` with unique ID, the ID is joined + by "#" mark for nested content. For example: + The config content `{"preprocessing": [{"": "LoadImage", "": {"keys": "image"}}]}` is parsed as items: + - `id="preprocessing", config=[{"": "LoadImage", "": {"keys": "image"}}]` + - `id="preprocessing#0", config={"": "LoadImage", "": {"keys": "image"}}` + - `id="preprocessing#0#", config="LoadImage"` + - `id="preprocessing#0#", config={"keys": "image"}` + - `id="preprocessing#0##keys", config="image"` + + There are 3 levels config information during the parsing: + - For the input config content, it supports to `get` and `update` the whole content or part of it specified with id, + it can be useful for lazy instantiation, etc. + - After parsing, all the config items are independent `ConfigItem`, can get it before / after resolving references. + - After resolving, the resolved output of every `ConfigItem` is python objects or instances, can be used in other + programs directly. + + Args: + config: input config content to parse. + id: specified ID name for the config content. + excludes: when importing modules to instantiate components, if any string of the `excludes` exists + in the full module name, don't import this module. + globals: pre-import packages as global variables to evaluate the python `eval` expressions. + for example, pre-import `monai`, then execute `eval("monai.data.list_data_collate")`. + default to `{"monai": "monai", "torch": "torch", "np": "numpy"}` as `numpy` and `torch` + are MONAI mininum requirements. + if the value in global is string, will import it immediately. + + """ + + def __init__( + self, + config: Any, + excludes: Optional[Union[Sequence[str], str]] = None, + globals: Optional[Dict[str, Any]] = None, + ): + self.config = None + self.update_config(config=config) + + self.globals: Dict[str, Any] = {} + globals = {"monai": "monai", "torch": "torch", "np": "numpy"} if globals is None else globals + if globals is not None: + for k, v in globals.items(): + self.globals[k] = importlib.import_module(v) if isinstance(v, str) else v + + self.locator = ComponentLocator(excludes=excludes) + self.reference_resolver: ReferenceResolver = ReferenceResolver() + # flag to identify the parsing status of current config content + self.parsed = False + + def update_config(self, config: Any, id: str = ""): + """ + Set config content for the parser, if `id` provided, `config` will replace the config item with `id`. + + Args: + config: target config content to set. + id: id name to specify the target position, joined by "#" mark for nested content, use index from 0 for list. + for example: "transforms#5", "transforms#5##keys", etc. + default to update all the config content. + + """ + if id != "" and isinstance(self.config, (dict, list)): + keys = id.split("#") + # get the last second config item and replace it + last_id = "#".join(keys[:-1]) + conf_ = self.get_config(id=last_id) + conf_[keys[-1] if isinstance(conf_, dict) else int(keys[-1])] = config + else: + self.config = config + # must totally parse again as the content is modified + self.parsed = False + + def get_config(self, id: str = ""): + """ + Get config content of current config, if `id` provided, get the config item with `id`. + + Args: + id: nested id name to specify the expected position, joined by "#" mark, use index from 0 for list. + for example: "transforms#5", "transforms#5##keys", etc. + default to get all the config content. + + """ + config = self.config + if id != "" and isinstance(config, (dict, list)): + keys = id.split("#") + for k in keys: + config = config[k] if isinstance(config, dict) else config[int(k)] + return config + + def _do_parse(self, config, id: str = ""): + """ + Recursively parse the nested config content, add every config item to the resolver. + + Args: + config: config content to parse. + id: id name of current config item, nested ids are joined by "#" mark. defaults to None. + for example: "transforms#5", "transforms#5##keys", etc. + default to empty string. + + """ + if isinstance(config, (dict, list)): + subs = enumerate(config) if isinstance(config, list) else config.items() + for k, v in subs: + sub_id = f"{id}#{k}" if id != "" else k + self._do_parse(config=v, id=sub_id) + + if id != "": + # copy every config item to make them independent and add them to the resolver + item_conf = deepcopy(config) + if ConfigComponent.is_instantiable(item_conf): + self.reference_resolver.add_item(ConfigComponent(config=item_conf, id=id, locator=self.locator)) + elif ConfigExpression.is_expression(item_conf): + self.reference_resolver.add_item(ConfigExpression(config=item_conf, id=id, globals=self.globals)) + else: + self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) + + def parse_config(self): + """ + Parse the config content, add every config item to the resolver and mark as `parsed`. + + """ + self.reference_resolver = ReferenceResolver() + self._do_parse(config=self.config) + self.parsed = True + + def get_resolved_content(self, id: str): + """ + Get the resolved result of config items with specified id, if not resolved, try to resolve it first. + If the config item is instantiable, the resolved result is the instance. + If the config item is an expression, the resolved result is output of when evaluating the expression. + Otherwise, the resolved result is the updated config content of the config item. + + Args: + id: id name of expected config item, nested ids are joined by "#" mark. + for example: "transforms#5", "transforms#5##keys", etc. + + """ + if not self.parsed: + self.parse_config() + return self.reference_resolver.get_resolved_content(id=id) + + def get_config_item(self, id: str, resolve: bool = False): + """ + Get the parsed config item, if `resolve=True` and not resolved, try to resolve it first. + It can be used to modify the config in other program and support lazy instantiation. + + Args: + id: id name of expected config component, nested ids are joined by "#" mark. + for example: "transforms#5", "transforms#5##keys", etc. + + """ + if not self.parsed: + self.parse_config() + return self.reference_resolver.get_item(id=id, resolve=resolve) diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py new file mode 100644 index 0000000000..7e07129e50 --- /dev/null +++ b/tests/test_config_parser.py @@ -0,0 +1,76 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest import skipUnless + +from parameterized import parameterized + +from monai.apps import ConfigParser +from monai.apps.manifest.config_item import ConfigComponent +from monai.data import DataLoader, Dataset +from monai.transforms import Compose, LoadImaged, RandTorchVisiond +from monai.utils import optional_import + +_, has_tv = optional_import("torchvision") + +# test the resolved and parsed instances +TEST_CASE_1 = [ + { + "transform": { + "": "Compose", + "": { + "transforms": [ + {"": "LoadImaged", "": {"keys": "image"}}, + { + "": "RandTorchVisiond", + "": {"keys": "image", "name": "ColorJitter", "brightness": 0.25}, + }, + ] + }, + }, + "dataset": {"": "Dataset", "": {"data": [1, 2], "transform": "@transform"}}, + "dataloader": { + "": "DataLoader", + "": {"dataset": "@dataset", "batch_size": 2, "collate_fn": "monai.data.list_data_collate"}, + }, + }, + ["transform", "transform##transforms#0", "transform##transforms#1", "dataset", "dataloader"], + [Compose, LoadImaged, RandTorchVisiond, Dataset, DataLoader], +] + + +class TestConfigComponent(unittest.TestCase): + def test_config_content(self): + parser = ConfigParser(config={}) + test_config = {"preprocessing": [{"": "LoadImage"}], "dataset": {"": "Dataset"}} + parser.update_config(config=test_config) + self.assertEqual(str(parser.get_config()), str(test_config)) + parser.update_config(config={"": "CacheDataset"}, id="dataset") + self.assertDictEqual(parser.get_config(id="dataset"), {"": "CacheDataset"}) + parser.update_config(config="Dataset", id="dataset#") + self.assertEqual(parser.get_config(id="dataset#"), "Dataset") + + @parameterized.expand([TEST_CASE_1]) + @skipUnless(has_tv, "Requires torchvision.") + def test_parse(self, config, expected_ids, output_types): + parser = ConfigParser(config=config, globals={"monai": "monai"}) + for id, cls in zip(expected_ids, output_types): + item = parser.get_config_item(id, resolve=True) + # test lazy instantiation + if isinstance(item, ConfigComponent): + self.assertTrue(isinstance(item.instantiate(), cls)) + # test get instance directly + self.assertTrue(isinstance(parser.get_resolved_content(id), cls)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_reference_resolver.py b/tests/test_reference_resolver.py index a62d6befd9..4f73233c24 100644 --- a/tests/test_reference_resolver.py +++ b/tests/test_reference_resolver.py @@ -1,4 +1,4 @@ -# Copyright 2020 - 2021 MONAI Consortium +# Copyright (c) MONAI Consortium # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at From 04e3ee79736f2eac16ae52d374e93719791da2a1 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Sun, 20 Feb 2022 08:27:22 +0800 Subject: [PATCH 02/76] [DLMED] simplify parse logic Signed-off-by: Nic Ma --- monai/apps/manifest/config_item.py | 2 +- monai/apps/manifest/config_parser.py | 29 +++++++++-------------- monai/apps/manifest/reference_resolver.py | 10 +++++++- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/monai/apps/manifest/config_item.py b/monai/apps/manifest/config_item.py index 075d00b961..a25dfcedf2 100644 --- a/monai/apps/manifest/config_item.py +++ b/monai/apps/manifest/config_item.py @@ -128,7 +128,7 @@ def __init__(self, config: Any, id: str = "") -> None: self.config = config self.id = id - def get_id(self) -> Optional[str]: + def get_id(self) -> str: """ Get the ID name of current config item, useful to identify config items during parsing. diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index f7180078cb..3e0efeb642 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -33,11 +33,12 @@ class ConfigParser: - `id="preprocessing#0##keys", config="image"` There are 3 levels config information during the parsing: - - For the input config content, it supports to `get` and `update` the whole content or part of it specified with id, - it can be useful for lazy instantiation, etc. - - After parsing, all the config items are independent `ConfigItem`, can get it before / after resolving references. - - After resolving, the resolved output of every `ConfigItem` is python objects or instances, can be used in other - programs directly. + - `get_config()`: For the input config content, it supports to `get` and `update` the whole content or part of it + specified with id, it can be useful for lazy instantiation. + - `get_config_item(resolve=True/False)`: After parsing, all the config items are independent `ConfigItem`, + can get it before / after resolving references. + - `get_resolved_content()`: After resolving, the resolved output of `ConfigItem` is python objects or instances, + can be used in other programs directly. Args: config: input config content to parse. @@ -59,8 +60,6 @@ def __init__( globals: Optional[Dict[str, Any]] = None, ): self.config = None - self.update_config(config=config) - self.globals: Dict[str, Any] = {} globals = {"monai": "monai", "torch": "torch", "np": "numpy"} if globals is None else globals if globals is not None: @@ -68,9 +67,8 @@ def __init__( self.globals[k] = importlib.import_module(v) if isinstance(v, str) else v self.locator = ComponentLocator(excludes=excludes) - self.reference_resolver: ReferenceResolver = ReferenceResolver() - # flag to identify the parsing status of current config content - self.parsed = False + self.reference_resolver = ReferenceResolver() + self.update_config(config=config) def update_config(self, config: Any, id: str = ""): """ @@ -92,7 +90,7 @@ def update_config(self, config: Any, id: str = ""): else: self.config = config # must totally parse again as the content is modified - self.parsed = False + self.parse_config() def get_config(self, id: str = ""): """ @@ -140,12 +138,11 @@ def _do_parse(self, config, id: str = ""): def parse_config(self): """ - Parse the config content, add every config item to the resolver and mark as `parsed`. + Parse the config content, add every config item to the resolver. """ - self.reference_resolver = ReferenceResolver() + self.reference_resolver.reset() self._do_parse(config=self.config) - self.parsed = True def get_resolved_content(self, id: str): """ @@ -159,8 +156,6 @@ def get_resolved_content(self, id: str): for example: "transforms#5", "transforms#5##keys", etc. """ - if not self.parsed: - self.parse_config() return self.reference_resolver.get_resolved_content(id=id) def get_config_item(self, id: str, resolve: bool = False): @@ -173,6 +168,4 @@ def get_config_item(self, id: str, resolve: bool = False): for example: "transforms#5", "transforms#5##keys", etc. """ - if not self.parsed: - self.parse_config() return self.reference_resolver.get_item(id=id, resolve=resolve) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 32d089370a..56177a5532 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -44,9 +44,17 @@ class ReferenceResolver: def __init__(self, items: Optional[Sequence[ConfigItem]] = None): # save the items in a dictionary with the `ConfigItem.id` as key - self.items = {} if items is None else {i.get_id(): i for i in items} + self.items: Dict[str, Any] = {} if items is None else {i.get_id(): i for i in items} self.resolved_content: Dict[str, Any] = {} + def reset(self): + """ + Clear all the added `ConfigItem` and all the resolved content. + + """ + self.items = {} + self.resolved_content = {} + def add_item(self, item: ConfigItem): """ Add a ``ConfigItem`` to the resolver. From 9abb571336f0b87afc198520c4d6d7f033a1fdc8 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 21 Feb 2022 15:53:51 +0800 Subject: [PATCH 03/76] [DLMED] add more function tests Signed-off-by: Nic Ma --- monai/utils/module.py | 7 ++++--- tests/test_config_parser.py | 40 +++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/monai/utils/module.py b/monai/utils/module.py index 8b7745c3ee..ca47ae6b2d 100644 --- a/monai/utils/module.py +++ b/monai/utils/module.py @@ -10,13 +10,13 @@ # limitations under the License. import enum -import inspect import os import re import sys import warnings from functools import partial, wraps from importlib import import_module +from inspect import isclass, isfunction, ismethod from pkgutil import walk_packages from pydoc import locate from re import match @@ -212,9 +212,10 @@ def instantiate(path: str, **kwargs): component = locate(path) if component is None: raise ModuleNotFoundError(f"Cannot locate '{path}'.") - if inspect.isclass(component): + if isclass(component): return component(**kwargs) - if inspect.isfunction(component): + # support regular function, static method and class method + if isfunction(component) or (ismethod(component) and isclass(getattr(component, "__self__", None))): return partial(component, **kwargs) warnings.warn(f"Component to instantiate must represent a valid class or function, but got {path}.") diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 7e07129e50..314340adce 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -48,6 +48,34 @@ ] +class TestClass: + @staticmethod + def compute(a, b, func=lambda x, y: x + y): + return func(a, b) + + @classmethod + def cls_compute(cls, a, b, func=lambda x, y: x + y): + return cls.compute(a, b, func) + + def __call__(self, a, b): + return self.compute(a, b) + + +TEST_CASE_2 = [ + { + "basic_func": "$lambda x, y: x + y", + "static_func": "$TestClass.compute", + "cls_func": "$TestClass.cls_compute", + "lambda_static_func": "$lambda x, y: TestClass.compute(x, y)", + "lambda_cls_func": "$lambda x, y: TestClass.cls_compute(x, y)", + "compute": {"": "tests.test_config_parser.TestClass.compute", "": {"func": "@basic_func"}}, + "cls_compute": {"": "tests.test_config_parser.TestClass.cls_compute", "": {"func": "@basic_func"}}, + "call_compute": {"": "tests.test_config_parser.TestClass"}, + "error_func": "$TestClass.__call__", + } +] + + class TestConfigComponent(unittest.TestCase): def test_config_content(self): parser = ConfigParser(config={}) @@ -71,6 +99,18 @@ def test_parse(self, config, expected_ids, output_types): # test get instance directly self.assertTrue(isinstance(parser.get_resolved_content(id), cls)) + @parameterized.expand([TEST_CASE_2]) + def test_function(self, config): + parser = ConfigParser(config=config, globals={"TestClass": TestClass}) + for id in config: + func = parser.get_resolved_content(id=id) + self.assertTrue(id in parser.reference_resolver.resolved_content) + if id == "error_func": + with self.assertRaises(TypeError): + func(1, 2) + continue + self.assertEqual(func(1, 2), 3) + if __name__ == "__main__": unittest.main() From fe3f7c284ed2124b5b01be99729454f0993ca5ee Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 21 Feb 2022 18:10:12 +0800 Subject: [PATCH 04/76] [DLMED] fix torchvision tests Signed-off-by: Nic Ma --- tests/test_config_item.py | 4 ++-- tests/test_config_parser.py | 6 +++--- tests/test_reference_resolver.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_config_item.py b/tests/test_config_item.py index b2c2fec6c6..0b4fdc51bf 100644 --- a/tests/test_config_item.py +++ b/tests/test_config_item.py @@ -20,9 +20,9 @@ from monai.apps import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem from monai.data import DataLoader, Dataset from monai.transforms import LoadImaged, RandTorchVisiond -from monai.utils import optional_import +from monai.utils import min_version, optional_import -_, has_tv = optional_import("torchvision") +_, has_tv = optional_import("torchvision", "0.8.0", min_version) TEST_CASE_1 = [{"lr": 0.001}, 0.0001] diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 314340adce..9aa1e7cfdd 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -18,9 +18,9 @@ from monai.apps.manifest.config_item import ConfigComponent from monai.data import DataLoader, Dataset from monai.transforms import Compose, LoadImaged, RandTorchVisiond -from monai.utils import optional_import +from monai.utils import min_version, optional_import -_, has_tv = optional_import("torchvision") +_, has_tv = optional_import("torchvision", "0.8.0", min_version) # test the resolved and parsed instances TEST_CASE_1 = [ @@ -88,7 +88,7 @@ def test_config_content(self): self.assertEqual(parser.get_config(id="dataset#"), "Dataset") @parameterized.expand([TEST_CASE_1]) - @skipUnless(has_tv, "Requires torchvision.") + @skipUnless(has_tv, "Requires torchvision >= 0.8.0.") def test_parse(self, config, expected_ids, output_types): parser = ConfigParser(config=config, globals={"monai": "monai"}) for id, cls in zip(expected_ids, output_types): diff --git a/tests/test_reference_resolver.py b/tests/test_reference_resolver.py index 4f73233c24..90d0f3ea72 100644 --- a/tests/test_reference_resolver.py +++ b/tests/test_reference_resolver.py @@ -19,9 +19,9 @@ from monai.apps.manifest.config_item import ComponentLocator, ConfigExpression, ConfigItem from monai.data import DataLoader from monai.transforms import LoadImaged, RandTorchVisiond -from monai.utils import optional_import +from monai.utils import min_version, optional_import -_, has_tv = optional_import("torchvision") +_, has_tv = optional_import("torchvision", "0.8.0", min_version) # test instance with no dependencies TEST_CASE_1 = [ From 24c4ca191f54af86fabce599efcdef52a701c673 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 21 Feb 2022 21:41:03 +0800 Subject: [PATCH 05/76] [DLMED] add standard run API Signed-off-by: Nic Ma --- monai/apps/__init__.py | 11 +++- monai/apps/manifest/__init__.py | 1 + monai/apps/manifest/run.py | 43 +++++++++++++++ monai/apps/manifest/utils.py | 95 +++++++++++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 monai/apps/manifest/run.py create mode 100644 monai/apps/manifest/utils.py diff --git a/monai/apps/__init__.py b/monai/apps/__init__.py index 0f233bc3ef..72da17a68f 100644 --- a/monai/apps/__init__.py +++ b/monai/apps/__init__.py @@ -10,6 +10,15 @@ # limitations under the License. from .datasets import CrossValidation, DecathlonDataset, MedNISTDataset -from .manifest import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, ReferenceResolver +from .manifest import ( + ComponentLocator, + ConfigComponent, + ConfigExpression, + ConfigItem, + ReferenceResolver, + parse_config_files, + parse_id_value, + read_config, +) from .mmars import MODEL_DESC, RemoteMMARKeys, download_mmar, get_model_spec, load_from_mmar from .utils import SUPPORTED_HASH_TYPES, check_hash, download_and_extract, download_url, extractall, get_logger, logger diff --git a/monai/apps/manifest/__init__.py b/monai/apps/manifest/__init__.py index 79c4376d5c..52abdf636a 100644 --- a/monai/apps/manifest/__init__.py +++ b/monai/apps/manifest/__init__.py @@ -11,3 +11,4 @@ from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem from .reference_resolver import ReferenceResolver +from .utils import parse_config_files, parse_id_value, read_config diff --git a/monai/apps/manifest/run.py b/monai/apps/manifest/run.py new file mode 100644 index 0000000000..a387ff12bb --- /dev/null +++ b/monai/apps/manifest/run.py @@ -0,0 +1,43 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +from monai.apps.manifest.utils import parse_config_files + + +def run(): + """ + Specify the metadata file, config file to run a standard training or evaluation program. + It's used to execute most of the supervised training or evaluation cases. + It supports to override the config content with specified `id` and `value`. + + """ + parser = argparse.ArgumentParser() + parser.add_argument("--metadata", "-m", type=str, help="filepath of the metadata file.", required=True) + parser.add_argument("--config", "-c", type=str, help="filepath of the config file.", required=True) + parser.add_argument("--override", "-o", metavar="ID=VALUE", nargs="*") + parser.add_argument( + "--target", "-c", type=str, + help=("ID name of the target workflow, it must have the `run` method, follow MONAI `BaseWorkflow`."), + required=True, + ) + + args = parser.parse_args() + + config_parser = parse_config_files(config_file=args.config, meta_file=args.metadata, override=args.override) + + workflow = config_parser.get_resolved_content(id=args.target) + workflow.run() + + +if __name__ == '__main__': + run() diff --git a/monai/apps/manifest/utils.py b/monai/apps/manifest/utils.py new file mode 100644 index 0000000000..3343db5fea --- /dev/null +++ b/monai/apps/manifest/utils.py @@ -0,0 +1,95 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from distutils.util import strtobool +import json +from typing import Any, List, Tuple +import yaml + +from monai.apps.manifest.config_parser import ConfigParser + + +def read_config(filepath: str): + """ + Read config file with specified file path. + Suppprt JSON and YAML formats. + + """ + with open(filepath, "r") as f: + if filepath.lower().endswith(".json"): + return json.load(f) + if filepath.lower().endswith((".yml", ".yaml")): + return yaml.load(f, Loader=yaml.FullLoader) + raise ValueError("only support JSON or YAML config file so far.") + + +def parse_id_value(pair: str) -> Tuple[str, Any]: + """ + Parse the "id=value" pair to `id` and `value`. + Will try to convert the correct data type of `value` from string. + + Args: + pair (str): input "id=value" pair to parse. + + """ + items = pair.split("=") + # we remove blanks around id + id = items[0].strip() + value = "" + if len(items) > 1: + # rejoin the rest + value = "=".join(items[1:]) + + # try to convert the correct data type + try: + value = int(value) + except ValueError: + try: + value = float(value) + except ValueError: + try: + value = bool(strtobool(str(value))) + except ValueError: + pass + return id, value + + +def parse_config_files(config_file: str, meta_file: str, override: List = []) -> ConfigParser: + """ + Read the config file, metadata file and override with specified `id=value` pairs. + The `id` identifies target position to override with the `value`. + If `value` starts with "", it will automatically read the `file` + and use the content as `value`. + + Args: + config_file: filepath of the config file. + meta_file: filepath of the metadata file. + override: list of "id=value" pairs to override the config content. + + """ + config = read_config(config_file) + if not isinstance(config, dict): + raise ValueError("input config file must be a dictionary.") + + config[""] = read_config(meta_file) + + parser = ConfigParser(config=config) + + if len(override) > 0: + kwargs = {} + for pair in override: + id, v = parse_id_value(pair) + if isinstance(v, str) and v.startswith(""): + v = read_config(v[6:]) + kwargs[id] = v + parser.update_config(**kwargs) + + return parser From 5dc87f0c087f089e15d37d71683d2191e9ef7221 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Feb 2022 13:44:51 +0000 Subject: [PATCH 06/76] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- monai/apps/manifest/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/apps/manifest/utils.py b/monai/apps/manifest/utils.py index 3343db5fea..9c4b547eb0 100644 --- a/monai/apps/manifest/utils.py +++ b/monai/apps/manifest/utils.py @@ -23,7 +23,7 @@ def read_config(filepath: str): Suppprt JSON and YAML formats. """ - with open(filepath, "r") as f: + with open(filepath) as f: if filepath.lower().endswith(".json"): return json.load(f) if filepath.lower().endswith((".yml", ".yaml")): From 886b607ad02e2dd036f18009f324b2db67185c59 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 21 Feb 2022 22:08:30 +0800 Subject: [PATCH 07/76] [DLMED] enhance update_config to support multiple items Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 35 ++++++++++++++-------------- tests/test_config_parser.py | 6 ++--- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index 3e0efeb642..2f4c3bcede 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -68,27 +68,26 @@ def __init__( self.locator = ComponentLocator(excludes=excludes) self.reference_resolver = ReferenceResolver() - self.update_config(config=config) + self.update_config({"": config}) - def update_config(self, config: Any, id: str = ""): + def update_config(self, content: Dict[str, Any]): """ - Set config content for the parser, if `id` provided, `config` will replace the config item with `id`. - - Args: - config: target config content to set. - id: id name to specify the target position, joined by "#" mark for nested content, use index from 0 for list. - for example: "transforms#5", "transforms#5##keys", etc. - default to update all the config content. + Update config content for the parser, every `key` and `value` in the `content` is corresponding to + the target `id` position and new config value in order. + Nested config id is joined by "#" mark, use index from 0 for list. + For example: "transforms#5", "transforms#5##keys", etc. + If `id` is `""`, replace `self.config`. """ - if id != "" and isinstance(self.config, (dict, list)): - keys = id.split("#") - # get the last second config item and replace it - last_id = "#".join(keys[:-1]) - conf_ = self.get_config(id=last_id) - conf_[keys[-1] if isinstance(conf_, dict) else int(keys[-1])] = config - else: - self.config = config + for id, config in content.items(): + if id != "" and isinstance(self.config, (dict, list)): + keys = id.split("#") + # get the last second config item and replace it + last_id = "#".join(keys[:-1]) + conf_ = self.get_config(id=last_id) + conf_[keys[-1] if isinstance(conf_, dict) else int(keys[-1])] = config + else: + self.config = config # must totally parse again as the content is modified self.parse_config() @@ -97,7 +96,7 @@ def get_config(self, id: str = ""): Get config content of current config, if `id` provided, get the config item with `id`. Args: - id: nested id name to specify the expected position, joined by "#" mark, use index from 0 for list. + id: id name to specify the expected position, nested config is joined by "#" mark, use index from 0 for list. for example: "transforms#5", "transforms#5##keys", etc. default to get all the config content. diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 9aa1e7cfdd..4814b12523 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -80,11 +80,11 @@ class TestConfigComponent(unittest.TestCase): def test_config_content(self): parser = ConfigParser(config={}) test_config = {"preprocessing": [{"": "LoadImage"}], "dataset": {"": "Dataset"}} - parser.update_config(config=test_config) + parser.update_config(content={"": test_config}) self.assertEqual(str(parser.get_config()), str(test_config)) - parser.update_config(config={"": "CacheDataset"}, id="dataset") + parser.update_config(content={"dataset": {"": "CacheDataset"}}) self.assertDictEqual(parser.get_config(id="dataset"), {"": "CacheDataset"}) - parser.update_config(config="Dataset", id="dataset#") + parser.update_config(content={"dataset#": "Dataset"}) self.assertEqual(parser.get_config(id="dataset#"), "Dataset") @parameterized.expand([TEST_CASE_1]) From d7cd6cbb818a8b1c12337ed2e1682a73be98ad2a Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 21 Feb 2022 22:13:31 +0800 Subject: [PATCH 08/76] [DLMED] update API Signed-off-by: Nic Ma --- monai/apps/manifest/utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/monai/apps/manifest/utils.py b/monai/apps/manifest/utils.py index 9c4b547eb0..4eba805541 100644 --- a/monai/apps/manifest/utils.py +++ b/monai/apps/manifest/utils.py @@ -65,7 +65,8 @@ def parse_id_value(pair: str) -> Tuple[str, Any]: def parse_config_files(config_file: str, meta_file: str, override: List = []) -> ConfigParser: """ Read the config file, metadata file and override with specified `id=value` pairs. - The `id` identifies target position to override with the `value`. + Put metadata in the config content with key "". + The `id` in `override` identifies target position to override with the `value`. If `value` starts with "", it will automatically read the `file` and use the content as `value`. @@ -84,12 +85,12 @@ def parse_config_files(config_file: str, meta_file: str, override: List = []) -> parser = ConfigParser(config=config) if len(override) > 0: - kwargs = {} + content = {} for pair in override: id, v = parse_id_value(pair) if isinstance(v, str) and v.startswith(""): v = read_config(v[6:]) - kwargs[id] = v - parser.update_config(**kwargs) + content[id] = v + parser.update_config(content) return parser From 67d19be14371dff11c3479ff6c188984775979c5 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 22 Feb 2022 00:11:14 +0800 Subject: [PATCH 09/76] [DLMED] simplify API Signed-off-by: Nic Ma --- monai/apps/manifest/run.py | 9 ++++++--- monai/apps/manifest/utils.py | 15 +++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/monai/apps/manifest/run.py b/monai/apps/manifest/run.py index a387ff12bb..b9d0bc28f7 100644 --- a/monai/apps/manifest/run.py +++ b/monai/apps/manifest/run.py @@ -11,7 +11,7 @@ import argparse -from monai.apps.manifest.utils import parse_config_files +from monai.apps.manifest.utils import parse_config_files, parse_id_value def run(): @@ -32,8 +32,11 @@ def run(): ) args = parser.parse_args() - - config_parser = parse_config_files(config_file=args.config, meta_file=args.metadata, override=args.override) + override = {} + for pair in args.override: + id, v = parse_id_value(pair) + override[id] = v + config_parser = parse_config_files(config_file=args.config, meta_file=args.metadata, override=override) workflow = config_parser.get_resolved_content(id=args.target) workflow.run() diff --git a/monai/apps/manifest/utils.py b/monai/apps/manifest/utils.py index 4eba805541..c85333a5bb 100644 --- a/monai/apps/manifest/utils.py +++ b/monai/apps/manifest/utils.py @@ -11,7 +11,7 @@ from distutils.util import strtobool import json -from typing import Any, List, Tuple +from typing import Any, Dict, List, Tuple import yaml from monai.apps.manifest.config_parser import ConfigParser @@ -62,7 +62,7 @@ def parse_id_value(pair: str) -> Tuple[str, Any]: return id, value -def parse_config_files(config_file: str, meta_file: str, override: List = []) -> ConfigParser: +def parse_config_files(config_file: str, meta_file: str, override: Dict = {}) -> ConfigParser: """ Read the config file, metadata file and override with specified `id=value` pairs. Put metadata in the config content with key "". @@ -73,7 +73,7 @@ def parse_config_files(config_file: str, meta_file: str, override: List = []) -> Args: config_file: filepath of the config file. meta_file: filepath of the metadata file. - override: list of "id=value" pairs to override the config content. + override: dict of `{id: value}` pairs to override the config content. """ config = read_config(config_file) @@ -85,12 +85,11 @@ def parse_config_files(config_file: str, meta_file: str, override: List = []) -> parser = ConfigParser(config=config) if len(override) > 0: - content = {} - for pair in override: - id, v = parse_id_value(pair) + for id in override: + v = override[id] if isinstance(v, str) and v.startswith(""): v = read_config(v[6:]) - content[id] = v - parser.update_config(content) + override[id] = v + parser.update_config(override) return parser From b90b73201d97f19267c00bcaf7060d818ff9aac4 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 22 Feb 2022 17:50:33 +0800 Subject: [PATCH 10/76] [DLMED] enhance doc and unit tests Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 51 ++++++++++++++++++++++------ tests/test_config_parser.py | 10 ++++-- 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index 2f4c3bcede..106d8529ff 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -32,13 +32,44 @@ class ConfigParser: - `id="preprocessing#0#", config={"keys": "image"}` - `id="preprocessing#0##keys", config="image"` - There are 3 levels config information during the parsing: - - `get_config()`: For the input config content, it supports to `get` and `update` the whole content or part of it - specified with id, it can be useful for lazy instantiation. - - `get_config_item(resolve=True/False)`: After parsing, all the config items are independent `ConfigItem`, - can get it before / after resolving references. - - `get_resolved_content()`: After resolving, the resolved output of `ConfigItem` is python objects or instances, - can be used in other programs directly. + A typical workflow of config parsing is as follows: + + - Initialize `ConfigParser` with the `config` content. + - Call ``get_resolved_content()`` to get expected component with `id`, which is automatically parsed. + + .. code-block:: python + + config = { + "preprocessing": {"": "LoadImage"}, + "net": {"": "UNet", "": ...}, + "trainer": {"": "SupervisedTrainer", "": {"network": "@net", ...}}, + } + parser = ConfigParser(config=config) + trainer = parser.get_resolved_content(id="trainer") + trainer.run() + + It's also flexible to modify config content and do `lazy instantiation` from 2 levels: + + 1. Modify original config content, then totally parse again: + + .. code-block:: python + + parser = ConfigParser(...) + config = parser.get_config() + config["processing"][2][""]["interp_order"] = "bilinear" + parser.parse_config() + + 2. After parsing, all the config items are independent `ConfigItem`, get the expected `ConfigItem`. + Its config content is already resolved references, can modify and use it directly: + + .. code-block:: python + + parser = ConfigParser(...) + trainer_config_item = parser.get_config_item(id="trainer") + config = trainer_config_item.get_config() + config["max_epochs"] = 100 + trainer = trainer_config_item.instantiate() + trainer.run() Args: config: input config content to parse. @@ -157,9 +188,9 @@ def get_resolved_content(self, id: str): """ return self.reference_resolver.get_resolved_content(id=id) - def get_config_item(self, id: str, resolve: bool = False): + def get_config_item(self, id: str): """ - Get the parsed config item, if `resolve=True` and not resolved, try to resolve it first. + Get the parsed config item which is already resolved all the references. It can be used to modify the config in other program and support lazy instantiation. Args: @@ -167,4 +198,4 @@ def get_config_item(self, id: str, resolve: bool = False): for example: "transforms#5", "transforms#5##keys", etc. """ - return self.reference_resolver.get_item(id=id, resolve=resolve) + return self.reference_resolver.get_item(id=id, resolve=True) diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 4814b12523..46bbc98444 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -91,9 +91,15 @@ def test_config_content(self): @skipUnless(has_tv, "Requires torchvision >= 0.8.0.") def test_parse(self, config, expected_ids, output_types): parser = ConfigParser(config=config, globals={"monai": "monai"}) + # test lazy instantiation with original config content + config = parser.get_config() + config["transform"][""]["transforms"][0][""]["keys"] = "label" + parser.parse_config() + self.assertEqual(parser.get_resolved_content(id="transform##transforms#0").keys[0], "label") + for id, cls in zip(expected_ids, output_types): - item = parser.get_config_item(id, resolve=True) - # test lazy instantiation + item = parser.get_config_item(id=id) + # test lazy instantiation with resolved config item if isinstance(item, ConfigComponent): self.assertTrue(isinstance(item.instantiate(), cls)) # test get instance directly From bfc74fc9371abff21fe9382063b3d248e9175812 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 22 Feb 2022 23:58:48 +0800 Subject: [PATCH 11/76] [DLMED] clear API Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 72 ++++++++++------------------ tests/test_config_parser.py | 12 ++--- 2 files changed, 28 insertions(+), 56 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index 106d8529ff..22715cbc08 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -19,13 +19,13 @@ class ConfigParser: """ - Parse a config content, access or update the items of config content with unique ID. + Parse a config source, access or update the content of the config source with unique ID. A typical usage is a config dictionary contains all the necessary information to define training workflow in JSON. - For more details of the config format, please check :py:class:`monai.apps.ConfigComponent`. + For more details of the config format, please check :py:class:`monai.apps.ConfigItem`. - It can recursively parse the config content, treat every item as a `ConfigItem` with unique ID, the ID is joined - by "#" mark for nested content. For example: - The config content `{"preprocessing": [{"": "LoadImage", "": {"keys": "image"}}]}` is parsed as items: + It can recursively parse the config source, treat every item as a `ConfigItem` with unique ID, the ID is joined + by "#" mark for nested items. For example: + The config source `{"preprocessing": [{"": "LoadImage", "": {"keys": "image"}}]}` is parsed as items: - `id="preprocessing", config=[{"": "LoadImage", "": {"keys": "image"}}]` - `id="preprocessing#0", config={"": "LoadImage", "": {"keys": "image"}}` - `id="preprocessing#0#", config="LoadImage"` @@ -34,8 +34,8 @@ class ConfigParser: A typical workflow of config parsing is as follows: - - Initialize `ConfigParser` with the `config` content. - - Call ``get_resolved_content()`` to get expected component with `id`, which is automatically parsed. + - Initialize `ConfigParser` with the `config` source. + - Call ``get_parsed_content()`` to get expected component with `id`, which will be automatically parsed. .. code-block:: python @@ -45,12 +45,10 @@ class ConfigParser: "trainer": {"": "SupervisedTrainer", "": {"network": "@net", ...}}, } parser = ConfigParser(config=config) - trainer = parser.get_resolved_content(id="trainer") + trainer = parser.get_parsed_content(id="trainer") trainer.run() - It's also flexible to modify config content and do `lazy instantiation` from 2 levels: - - 1. Modify original config content, then totally parse again: + It's also flexible to modify config source at runtime and parse again: .. code-block:: python @@ -58,22 +56,12 @@ class ConfigParser: config = parser.get_config() config["processing"][2][""]["interp_order"] = "bilinear" parser.parse_config() - - 2. After parsing, all the config items are independent `ConfigItem`, get the expected `ConfigItem`. - Its config content is already resolved references, can modify and use it directly: - - .. code-block:: python - - parser = ConfigParser(...) - trainer_config_item = parser.get_config_item(id="trainer") - config = trainer_config_item.get_config() - config["max_epochs"] = 100 - trainer = trainer_config_item.instantiate() + trainer = parser.get_parsed_content(id="trainer") trainer.run() Args: - config: input config content to parse. - id: specified ID name for the config content. + config: input config source to parse. + id: specified ID name for the config sources. excludes: when importing modules to instantiate components, if any string of the `excludes` exists in the full module name, don't import this module. globals: pre-import packages as global variables to evaluate the python `eval` expressions. @@ -103,7 +91,7 @@ def __init__( def update_config(self, content: Dict[str, Any]): """ - Update config content for the parser, every `key` and `value` in the `content` is corresponding to + Update config source for the parser, every `key` and `value` in the `content` is corresponding to the target `id` position and new config value in order. Nested config id is joined by "#" mark, use index from 0 for list. For example: "transforms#5", "transforms#5##keys", etc. @@ -124,12 +112,12 @@ def update_config(self, content: Dict[str, Any]): def get_config(self, id: str = ""): """ - Get config content of current config, if `id` provided, get the config item with `id`. + Get config source in the parser, if `id` provided, get the config item with `id`. Args: id: id name to specify the expected position, nested config is joined by "#" mark, use index from 0 for list. for example: "transforms#5", "transforms#5##keys", etc. - default to get all the config content. + default to get all the config source data. """ config = self.config @@ -141,10 +129,10 @@ def get_config(self, id: str = ""): def _do_parse(self, config, id: str = ""): """ - Recursively parse the nested config content, add every config item to the resolver. + Recursively parse the nested data in config source, add every config item to the resolver. Args: - config: config content to parse. + config: config source to parse. id: id name of current config item, nested ids are joined by "#" mark. defaults to None. for example: "transforms#5", "transforms#5##keys", etc. default to empty string. @@ -168,18 +156,20 @@ def _do_parse(self, config, id: str = ""): def parse_config(self): """ - Parse the config content, add every config item to the resolver. + Parse the config source, add every config item to the resolver. """ self.reference_resolver.reset() self._do_parse(config=self.config) - def get_resolved_content(self, id: str): + def get_parsed_content(self, id: str): """ - Get the resolved result of config items with specified id, if not resolved, try to resolve it first. - If the config item is instantiable, the resolved result is the instance. - If the config item is an expression, the resolved result is output of when evaluating the expression. - Otherwise, the resolved result is the updated config content of the config item. + Get the parsed result of config item with specified id, if having references not resolved, + try to resolve it first. + + If the config item is `ConfigComponent`, the parsed result is the instance. + If the config item is `ConfigExpression`, the parsed result is output of evaluating the expression. + Otherwise, the parsed result is the updated `self.config` data of `ConfigItem`. Args: id: id name of expected config item, nested ids are joined by "#" mark. @@ -187,15 +177,3 @@ def get_resolved_content(self, id: str): """ return self.reference_resolver.get_resolved_content(id=id) - - def get_config_item(self, id: str): - """ - Get the parsed config item which is already resolved all the references. - It can be used to modify the config in other program and support lazy instantiation. - - Args: - id: id name of expected config component, nested ids are joined by "#" mark. - for example: "transforms#5", "transforms#5##keys", etc. - - """ - return self.reference_resolver.get_item(id=id, resolve=True) diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 46bbc98444..11de6c52be 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -15,7 +15,6 @@ from parameterized import parameterized from monai.apps import ConfigParser -from monai.apps.manifest.config_item import ConfigComponent from monai.data import DataLoader, Dataset from monai.transforms import Compose, LoadImaged, RandTorchVisiond from monai.utils import min_version, optional_import @@ -95,21 +94,16 @@ def test_parse(self, config, expected_ids, output_types): config = parser.get_config() config["transform"][""]["transforms"][0][""]["keys"] = "label" parser.parse_config() - self.assertEqual(parser.get_resolved_content(id="transform##transforms#0").keys[0], "label") + self.assertEqual(parser.get_parsed_content(id="transform##transforms#0").keys[0], "label") for id, cls in zip(expected_ids, output_types): - item = parser.get_config_item(id=id) - # test lazy instantiation with resolved config item - if isinstance(item, ConfigComponent): - self.assertTrue(isinstance(item.instantiate(), cls)) - # test get instance directly - self.assertTrue(isinstance(parser.get_resolved_content(id), cls)) + self.assertTrue(isinstance(parser.get_parsed_content(id), cls)) @parameterized.expand([TEST_CASE_2]) def test_function(self, config): parser = ConfigParser(config=config, globals={"TestClass": TestClass}) for id in config: - func = parser.get_resolved_content(id=id) + func = parser.get_parsed_content(id=id) self.assertTrue(id in parser.reference_resolver.resolved_content) if id == "error_func": with self.assertRaises(TypeError): From 362e33660ea13611c100baae9319e580871300f9 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Wed, 23 Feb 2022 23:28:12 +0800 Subject: [PATCH 12/76] [DLMED] simplify usage APIs Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 111 ++++++++++++++++----------- tests/test_config_parser.py | 31 +++++--- 2 files changed, 87 insertions(+), 55 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index 22715cbc08..921a7daadb 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -35,7 +35,7 @@ class ConfigParser: A typical workflow of config parsing is as follows: - Initialize `ConfigParser` with the `config` source. - - Call ``get_parsed_content()`` to get expected component with `id`, which will be automatically parsed. + - Call ``get_parsed_content()`` to get expected component with `id`. .. code-block:: python @@ -53,15 +53,12 @@ class ConfigParser: .. code-block:: python parser = ConfigParser(...) - config = parser.get_config() - config["processing"][2][""]["interp_order"] = "bilinear" - parser.parse_config() + parser["processing"][2][""]["interp_order"] = "bilinear" trainer = parser.get_parsed_content(id="trainer") trainer.run() Args: config: input config source to parse. - id: specified ID name for the config sources. excludes: when importing modules to instantiate components, if any string of the `excludes` exists in the full module name, don't import this module. globals: pre-import packages as global variables to evaluate the python `eval` expressions. @@ -87,54 +84,80 @@ def __init__( self.locator = ComponentLocator(excludes=excludes) self.reference_resolver = ReferenceResolver() - self.update_config({"": config}) + self.set(config=config) - def update_config(self, content: Dict[str, Any]): + def __getitem__(self, id: Union[str, int]): """ - Update config source for the parser, every `key` and `value` in the `content` is corresponding to - the target `id` position and new config value in order. - Nested config id is joined by "#" mark, use index from 0 for list. - For example: "transforms#5", "transforms#5##keys", etc. - If `id` is `""`, replace `self.config`. + Get config source in the parser with provided `id` for target position. + + Args: + id: id name to specify the expected position, nested config is joined by "#" mark, + use string or int index from 0 for list, for example: "transforms#5", "transforms#5##keys". + if `id=""`, get all the config source data in `self.config`. """ - for id, config in content.items(): - if id != "" and isinstance(self.config, (dict, list)): - keys = id.split("#") - # get the last second config item and replace it - last_id = "#".join(keys[:-1]) - conf_ = self.get_config(id=last_id) - conf_[keys[-1] if isinstance(conf_, dict) else int(keys[-1])] = config - else: - self.config = config - # must totally parse again as the content is modified - self.parse_config() + config = self.config + if id != "": + keys = str(id).split("#") + for k in keys: + if not isinstance(config, (dict, list)): + raise ValueError(f"config must be dict or list for key `{k}`, but got: {config}.") + config = config[k] if isinstance(config, dict) else config[int(k)] + return config + + def __setitem__(self, id: Union[str, int], config: Any): + """ + Set config source for the parser at target position `id``. + Nested config `id` is joined by "#" mark, use string or int index from 0 for list item. + For example: "transforms#5", "transforms#5##keys". + If `id` is `""`, replace all the config source data in `self.config`. + Must totally parse again as the config source is modified. - def get_config(self, id: str = ""): """ - Get config source in the parser, if `id` provided, get the config item with `id`. + if id != "": + keys = str(id).split("#") + # get the last second config item and replace it + last_id = "#".join(keys[:-1]) + conf_ = self[last_id] + conf_[keys[-1] if isinstance(conf_, dict) else int(keys[-1])] = config + else: + self.config = config + self.reference_resolver.reset() + + def get(self, id: str = "", default: Optional[Any] = None): + """ + Get config source in the parser with provided `id` for target position. Args: id: id name to specify the expected position, nested config is joined by "#" mark, use index from 0 for list. - for example: "transforms#5", "transforms#5##keys", etc. - default to get all the config source data. + for example: "transforms#5", "transforms#5##keys". + default to get all the config source data in `self.config`. + default: default value to return if the specified `id` is invalid. """ - config = self.config - if id != "" and isinstance(config, (dict, list)): - keys = id.split("#") - for k in keys: - config = config[k] if isinstance(config, dict) else config[int(k)] - return config + try: + return self[id] + except KeyError: + return default + + def set(self, config: Any, id: str = ""): + """ + Set config source for the parser at target position `id``, nested config id is joined by "#" mark, + use index from 0 for list. For example: "transforms#5", "transforms#5##keys". + If `id` is `""`, replace all the config source data in `self.config`. + Must totally parse again as the config source is modified. + + """ + self[id] = config def _do_parse(self, config, id: str = ""): """ - Recursively parse the nested data in config source, add every config item to the resolver. + Recursively parse the nested data in config source, add every item as `ConfigItem` to the resolver. Args: config: config source to parse. id: id name of current config item, nested ids are joined by "#" mark. defaults to None. - for example: "transforms#5", "transforms#5##keys", etc. + for example: "transforms#5", "transforms#5##keys". default to empty string. """ @@ -154,26 +177,28 @@ def _do_parse(self, config, id: str = ""): else: self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) - def parse_config(self): + def parse(self): """ - Parse the config source, add every config item to the resolver. + Recursively parse the config source, add every item as `ConfigItem` to the resolver. """ - self.reference_resolver.reset() self._do_parse(config=self.config) def get_parsed_content(self, id: str): """ - Get the parsed result of config item with specified id, if having references not resolved, + Get the parsed result of `ConfigItem` with specified `id`, if having references not resolved, try to resolve it first. - If the config item is `ConfigComponent`, the parsed result is the instance. - If the config item is `ConfigExpression`, the parsed result is output of evaluating the expression. + If the item is `ConfigComponent`, the parsed result is the instance. + If the item is `ConfigExpression`, the parsed result is output of evaluating the expression. Otherwise, the parsed result is the updated `self.config` data of `ConfigItem`. Args: - id: id name of expected config item, nested ids are joined by "#" mark. - for example: "transforms#5", "transforms#5##keys", etc. + id: id name of expected `ConfigItem`, nested items are joined by "#" mark as the `id`. + for example: "transforms#5", "transforms#5##keys". """ + if len(self.reference_resolver.resolved_content) == 0: + # not parsed the config source yet, parse it + self.parse() return self.reference_resolver.get_resolved_content(id=id) diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 11de6c52be..21f874c1c5 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -77,25 +77,32 @@ def __call__(self, a, b): class TestConfigComponent(unittest.TestCase): def test_config_content(self): - parser = ConfigParser(config={}) test_config = {"preprocessing": [{"": "LoadImage"}], "dataset": {"": "Dataset"}} - parser.update_config(content={"": test_config}) - self.assertEqual(str(parser.get_config()), str(test_config)) - parser.update_config(content={"dataset": {"": "CacheDataset"}}) - self.assertDictEqual(parser.get_config(id="dataset"), {"": "CacheDataset"}) - parser.update_config(content={"dataset#": "Dataset"}) - self.assertEqual(parser.get_config(id="dataset#"), "Dataset") + parser = ConfigParser(config=test_config) + # test `get`, `set`, `__getitem__`, `__setitem__` + self.assertEqual(str(parser.get()), str(test_config)) + parser.set(config=test_config) + self.assertListEqual(parser["preprocessing"], test_config["preprocessing"]) + parser["dataset"] = {"": "CacheDataset"} + self.assertEqual(parser["dataset"][""], "CacheDataset") + # test nested ids + parser["dataset#"] = "Dataset" + self.assertEqual(parser["dataset#"], "Dataset") + # test int id + parser.set(["test1", "test2", "test3"]) + parser[1] = "test4" + self.assertEqual(parser[1], "test4") @parameterized.expand([TEST_CASE_1]) @skipUnless(has_tv, "Requires torchvision >= 0.8.0.") def test_parse(self, config, expected_ids, output_types): parser = ConfigParser(config=config, globals={"monai": "monai"}) # test lazy instantiation with original config content - config = parser.get_config() - config["transform"][""]["transforms"][0][""]["keys"] = "label" - parser.parse_config() - self.assertEqual(parser.get_parsed_content(id="transform##transforms#0").keys[0], "label") - + parser["transform"][""]["transforms"][0][""]["keys"] = "label1" + self.assertEqual(parser.get_parsed_content(id="transform##transforms#0").keys[0], "label1") + # test nested id + parser["transform##transforms#0##keys"] = "label2" + self.assertEqual(parser.get_parsed_content(id="transform##transforms#0").keys[0], "label2") for id, cls in zip(expected_ids, output_types): self.assertTrue(isinstance(parser.get_parsed_content(id), cls)) From e7559c5753969486e3e6c72180931d4d48956ace Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 24 Feb 2022 00:04:55 +0800 Subject: [PATCH 13/76] [DLMED] support root config Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 17 ++++++++--------- monai/apps/manifest/reference_resolver.py | 2 -- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index 921a7daadb..ebed087597 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -167,15 +167,14 @@ def _do_parse(self, config, id: str = ""): sub_id = f"{id}#{k}" if id != "" else k self._do_parse(config=v, id=sub_id) - if id != "": - # copy every config item to make them independent and add them to the resolver - item_conf = deepcopy(config) - if ConfigComponent.is_instantiable(item_conf): - self.reference_resolver.add_item(ConfigComponent(config=item_conf, id=id, locator=self.locator)) - elif ConfigExpression.is_expression(item_conf): - self.reference_resolver.add_item(ConfigExpression(config=item_conf, id=id, globals=self.globals)) - else: - self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) + # copy every config item to make them independent and add them to the resolver + item_conf = deepcopy(config) + if ConfigComponent.is_instantiable(item_conf): + self.reference_resolver.add_item(ConfigComponent(config=item_conf, id=id, locator=self.locator)) + elif ConfigExpression.is_expression(item_conf): + self.reference_resolver.add_item(ConfigExpression(config=item_conf, id=id, globals=self.globals)) + else: + self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) def parse(self): """ diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 56177a5532..746c7d0823 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -64,8 +64,6 @@ def add_item(self, item: ConfigItem): """ id = item.get_id() - if id == "": - raise ValueError("id should not be empty when resolving reference.") if id in self.items: warnings.warn(f"id '{id}' is already added.") return From 420346bae8b4b189c77221941f36f3843d3ec987 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 24 Feb 2022 00:23:48 +0800 Subject: [PATCH 14/76] [DLMED] fix update reference typo Signed-off-by: Nic Ma --- monai/apps/manifest/reference_resolver.py | 4 ++-- tests/test_config_parser.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 746c7d0823..a19204975c 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -116,7 +116,7 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None): if d not in self.resolved_content: # this referring item is not resolved if d not in self.items: - raise ValueError(f"the referring item `{d}` is not defined in config.") + raise ValueError(f"the referring item `{d}` is not defined in the config content.") # recursively resolve the reference first self._resolve_one_item(id=d, waiting_list=waiting_list) @@ -235,7 +235,7 @@ def update_config_with_refs(config, id: str, refs: Optional[Dict] = None): for idx, v in config.items() if isinstance(config, dict) else enumerate(config): sub_id = f"{id}#{idx}" if id != "" else f"{idx}" if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): - updated = ReferenceResolver.update_config_with_refs(v, sub_id, refs_) + updated = refs_[sub_id] else: updated = ReferenceResolver.update_config_with_refs(v, sub_id, refs_) ret.update({idx: updated}) if isinstance(ret, dict) else ret.append(updated) diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 21f874c1c5..dcb4e0801e 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -105,6 +105,11 @@ def test_parse(self, config, expected_ids, output_types): self.assertEqual(parser.get_parsed_content(id="transform##transforms#0").keys[0], "label2") for id, cls in zip(expected_ids, output_types): self.assertTrue(isinstance(parser.get_parsed_content(id), cls)) + # test root content + root = parser.get_parsed_content(id="") + print("!!!!!!!!", root) + for v, cls in zip(root.values(), [Compose, Dataset, DataLoader]): + self.assertTrue(isinstance(v, cls)) @parameterized.expand([TEST_CASE_2]) def test_function(self, config): From 4492a912933941dca0a332df5b568cd2e3b5080d Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 24 Feb 2022 01:27:49 +0800 Subject: [PATCH 15/76] [DLMED] update usage API Signed-off-by: Nic Ma --- monai/apps/manifest/run.py | 2 ++ monai/apps/manifest/utils.py | 8 +++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/monai/apps/manifest/run.py b/monai/apps/manifest/run.py index b9d0bc28f7..1199e57aa9 100644 --- a/monai/apps/manifest/run.py +++ b/monai/apps/manifest/run.py @@ -19,6 +19,8 @@ def run(): Specify the metadata file, config file to run a standard training or evaluation program. It's used to execute most of the supervised training or evaluation cases. It supports to override the config content with specified `id` and `value`. + The `override` arg can also be used to provide default value for placeholders. For example: + put a placeholder `"data": "@runtime_value"` in the config, then define `runtime_value` in `override`. """ parser = argparse.ArgumentParser() diff --git a/monai/apps/manifest/utils.py b/monai/apps/manifest/utils.py index c85333a5bb..0a67bffee8 100644 --- a/monai/apps/manifest/utils.py +++ b/monai/apps/manifest/utils.py @@ -73,7 +73,7 @@ def parse_config_files(config_file: str, meta_file: str, override: Dict = {}) -> Args: config_file: filepath of the config file. meta_file: filepath of the metadata file. - override: dict of `{id: value}` pairs to override the config content. + override: dict of `{id: value}` pairs to override or add the config content. """ config = read_config(config_file) @@ -85,11 +85,9 @@ def parse_config_files(config_file: str, meta_file: str, override: Dict = {}) -> parser = ConfigParser(config=config) if len(override) > 0: - for id in override: - v = override[id] + for id, v in override.items(): if isinstance(v, str) and v.startswith(""): v = read_config(v[6:]) - override[id] = v - parser.update_config(override) + parser[id] = v return parser From 3c1205c2aa4e9dc98c02f3165b94fb1df7a722dd Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 23 Feb 2022 22:50:40 +0000 Subject: [PATCH 16/76] revise APIs and docstrings Signed-off-by: Wenqi Li --- docs/source/apps.rst | 1 + monai/apps/manifest/config_item.py | 3 + monai/apps/manifest/config_parser.py | 176 ++++++++++++---------- monai/apps/manifest/reference_resolver.py | 46 +++--- monai/utils/module.py | 7 +- tests/test_config_parser.py | 1 - 6 files changed, 132 insertions(+), 102 deletions(-) diff --git a/docs/source/apps.rst b/docs/source/apps.rst index d0fe131d85..8c0214d584 100644 --- a/docs/source/apps.rst +++ b/docs/source/apps.rst @@ -46,6 +46,7 @@ Model Manifest .. autoclass:: ConfigParser :members: + :special-members: __getitem__, __setitem__ .. autoclass:: ReferenceResolver :members: diff --git a/monai/apps/manifest/config_item.py b/monai/apps/manifest/config_item.py index a25dfcedf2..a6406795cf 100644 --- a/monai/apps/manifest/config_item.py +++ b/monai/apps/manifest/config_item.py @@ -153,6 +153,9 @@ def get_config(self): """ return self.config + def __repr__(self) -> str: + return str(self.config) + class ConfigComponent(ConfigItem, Instantiable): """ diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index ebed087597..1a90696e87 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -19,53 +19,58 @@ class ConfigParser: """ - Parse a config source, access or update the content of the config source with unique ID. - A typical usage is a config dictionary contains all the necessary information to define training workflow in JSON. - For more details of the config format, please check :py:class:`monai.apps.ConfigItem`. - - It can recursively parse the config source, treat every item as a `ConfigItem` with unique ID, the ID is joined - by "#" mark for nested items. For example: - The config source `{"preprocessing": [{"": "LoadImage", "": {"keys": "image"}}]}` is parsed as items: - - `id="preprocessing", config=[{"": "LoadImage", "": {"keys": "image"}}]` - - `id="preprocessing#0", config={"": "LoadImage", "": {"keys": "image"}}` - - `id="preprocessing#0#", config="LoadImage"` - - `id="preprocessing#0#", config={"keys": "image"}` - - `id="preprocessing#0##keys", config="image"` + The primary configuration parser. It traverses a structured config (in the form of nested Python dict or list), + creates ``ConfigItem``, and assign unique IDs according to the structures. + This class provides convenient access to the set of ``ConfigItem`` of the config by ID. A typical workflow of config parsing is as follows: - - Initialize `ConfigParser` with the `config` source. - - Call ``get_parsed_content()`` to get expected component with `id`. + - Initialize ``ConfigParser`` with the ``config`` source. + - Call ``get_parsed_content()`` to get expected component with `id`. .. code-block:: python + from monai.apps import ConfigParser + + config = { - "preprocessing": {"": "LoadImage"}, - "net": {"": "UNet", "": ...}, - "trainer": {"": "SupervisedTrainer", "": {"network": "@net", ...}}, + "my_dims": 2, + "dims_1": "$@my_dims + 1", + "my_xform": {"": "LoadImage"}, + "my_net": {"": "BasicUNet", + "": {"spatial_dims": "@dims_1", "in_channels": 1, "out_channels": 4}}, + "trainer": {"": "SupervisedTrainer", + "": {"network": "@my_net", "preprocessing": "@my_xform"}} } - parser = ConfigParser(config=config) - trainer = parser.get_parsed_content(id="trainer") - trainer.run() + # in the example $@my_dims + 1 is an expression, which adds 1 to the value of @my_dims + parser = ConfigParser(config) - It's also flexible to modify config source at runtime and parse again: + # get/set the configuration content, but do not instantiate the components + trainer = parser.get_parsed_content("trainer", instantiate=False) + print(trainer) + print(parser["my_net"][""]["in_channels"]) # original input channels 1 + parser["my_net"][""]["in_channels"] = 4 # change input channels to 4 + print(parser["my_net"][""]["in_channels"]) - .. code-block:: python + # instantiate the network component + parser.parse(True) + net = parser.get_parsed_content("my_net", instantiate=True) + print(net) - parser = ConfigParser(...) - parser["processing"][2][""]["interp_order"] = "bilinear" - trainer = parser.get_parsed_content(id="trainer") - trainer.run() Args: config: input config source to parse. - excludes: when importing modules to instantiate components, if any string of the `excludes` exists - in the full module name, don't import this module. - globals: pre-import packages as global variables to evaluate the python `eval` expressions. - for example, pre-import `monai`, then execute `eval("monai.data.list_data_collate")`. - default to `{"monai": "monai", "torch": "torch", "np": "numpy"}` as `numpy` and `torch` - are MONAI mininum requirements. - if the value in global is string, will import it immediately. + excludes: when importing modules to instantiate components, + excluding components from modules specified in ``excludes``. + globals: pre-import packages as global variables to ``ConfigExpression``, + so that expressions, for example, ``"$monai.data.list_data_collate"`` can use ``monai`` modules. + The current supported globals and alias names are + ``{"monai": "monai", "torch": "torch", "np": "numpy", "numpy": "numpy"}``. + These are MONAI's minimal dependencies. + + See also: + + - :py:class:`monai.apps.ConfigItem` """ @@ -77,7 +82,7 @@ def __init__( ): self.config = None self.globals: Dict[str, Any] = {} - globals = {"monai": "monai", "torch": "torch", "np": "numpy"} if globals is None else globals + globals = {"monai": "monai", "torch": "torch", "np": "numpy", "numpy": "numpy"} if globals is None else globals if globals is not None: for k, v in globals.items(): self.globals[k] = importlib.import_module(v) if isinstance(v, str) else v @@ -88,51 +93,57 @@ def __init__( def __getitem__(self, id: Union[str, int]): """ - Get config source in the parser with provided `id` for target position. + Get the config by id. Args: - id: id name to specify the expected position, nested config is joined by "#" mark, - use string or int index from 0 for list, for example: "transforms#5", "transforms#5##keys". - if `id=""`, get all the config source data in `self.config`. + id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to + go one level further into the nested structures. + Use digits indexing from "0" for list or other strings for dict. + For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. """ + if id == "": + return self.config config = self.config - if id != "": - keys = str(id).split("#") - for k in keys: - if not isinstance(config, (dict, list)): - raise ValueError(f"config must be dict or list for key `{k}`, but got: {config}.") - config = config[k] if isinstance(config, dict) else config[int(k)] + for k in str(id).split("#"): + if not isinstance(config, (dict, list)): + raise ValueError(f"config must be dict or list for key `{k}`, but got {type(config)}: {config}.") + indexing = k if isinstance(config, dict) else int(k) + config = config[indexing] return config def __setitem__(self, id: Union[str, int], config: Any): """ - Set config source for the parser at target position `id``. - Nested config `id` is joined by "#" mark, use string or int index from 0 for list item. - For example: "transforms#5", "transforms#5##keys". - If `id` is `""`, replace all the config source data in `self.config`. - Must totally parse again as the config source is modified. + Set config by ``id``. + + Args: + id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to + go one level further into the nested structures. + Use digits indexing from "0" for list or other strings for dict. + For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. + config: config to set at location ``id``. """ - if id != "": - keys = str(id).split("#") - # get the last second config item and replace it - last_id = "#".join(keys[:-1]) - conf_ = self[last_id] - conf_[keys[-1] if isinstance(conf_, dict) else int(keys[-1])] = config - else: + if id == "": self.config = config + self.reference_resolver.reset() + return + keys = str(id).split("#") + # get the last second config item and replace it + last_id = "#".join(keys[:-1]) + conf_ = self[last_id] + indexing = keys[-1] if isinstance(conf_, dict) else int(keys[-1]) + conf_[indexing] = config self.reference_resolver.reset() + return def get(self, id: str = "", default: Optional[Any] = None): """ - Get config source in the parser with provided `id` for target position. + Get the config by id. Args: - id: id name to specify the expected position, nested config is joined by "#" mark, use index from 0 for list. - for example: "transforms#5", "transforms#5##keys". - default to get all the config source data in `self.config`. - default: default value to return if the specified `id` is invalid. + id: id to specify the expected position. See also :py:meth:`__getitem__`. + default: default value to return if the specified ``id`` is invalid. """ try: @@ -142,10 +153,7 @@ def get(self, id: str = "", default: Optional[Any] = None): def set(self, config: Any, id: str = ""): """ - Set config source for the parser at target position `id``, nested config id is joined by "#" mark, - use index from 0 for list. For example: "transforms#5", "transforms#5##keys". - If `id` is `""`, replace all the config source data in `self.config`. - Must totally parse again as the config source is modified. + Set config by ``id``. See also :py:meth:`__setitem__`. """ self[id] = config @@ -156,9 +164,10 @@ def _do_parse(self, config, id: str = ""): Args: config: config source to parse. - id: id name of current config item, nested ids are joined by "#" mark. defaults to None. - for example: "transforms#5", "transforms#5##keys". - default to empty string. + id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to + go one level further into the nested structures. + Use digits indexing from "0" for list or other strings for dict. + For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. """ if isinstance(config, (dict, list)): @@ -176,28 +185,35 @@ def _do_parse(self, config, id: str = ""): else: self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) - def parse(self): + def parse(self, reset: bool = False): """ - Recursively parse the config source, add every item as `ConfigItem` to the resolver. + Recursively parse the config source, add every item as ``ConfigItem`` to the resolver. + + Args: + reset: whether to reset the ``reference_resolver`` before parsing. Defaults to False. """ + if reset: + self.reference_resolver.reset() self._do_parse(config=self.config) - def get_parsed_content(self, id: str): + def get_parsed_content(self, id: str = "", **kwargs): """ - Get the parsed result of `ConfigItem` with specified `id`, if having references not resolved, - try to resolve it first. + Get the parsed result of ``ConfigItem`` with the specified ``id``. - If the item is `ConfigComponent`, the parsed result is the instance. - If the item is `ConfigExpression`, the parsed result is output of evaluating the expression. - Otherwise, the parsed result is the updated `self.config` data of `ConfigItem`. + - If the item is ``ConfigComponent`` and ``instantiate=True``, the result is the instance. + - If the item is ``ConfigExpression`` and ``eval_expr=True``, the result is the evaluated output. Args: - id: id name of expected `ConfigItem`, nested items are joined by "#" mark as the `id`. - for example: "transforms#5", "transforms#5##keys". + id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to + go one level further into the nested structures. + Use digits indexing from "0" for list or other strings for dict. + For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. + kwargs: additional keyword arguments to be passed to ``_resolve_one_item``. + Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. """ - if len(self.reference_resolver.resolved_content) == 0: + if not self.reference_resolver.is_resolved(): # not parsed the config source yet, parse it self.parse() - return self.reference_resolver.get_resolved_content(id=id) + return self.reference_resolver.get_resolved_content(id=id, **kwargs) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index a19204975c..9bb9e43253 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -10,10 +10,10 @@ # limitations under the License. import re -import warnings from typing import Any, Dict, Optional, Sequence, Set from monai.apps.manifest.config_item import ConfigComponent, ConfigExpression, ConfigItem +from monai.utils import look_up_option class ReferenceResolver: @@ -55,6 +55,9 @@ def reset(self): self.items = {} self.resolved_content = {} + def is_resolved(self) -> bool: + return bool(self.resolved_content) + def add_item(self, item: ConfigItem): """ Add a ``ConfigItem`` to the resolver. @@ -65,11 +68,11 @@ def add_item(self, item: ConfigItem): """ id = item.get_id() if id in self.items: - warnings.warn(f"id '{id}' is already added.") + # warnings.warn(f"id '{id}' is already added.") return self.items[id] = item - def get_item(self, id: str, resolve: bool = False): + def get_item(self, id: str, resolve: bool = False, **kwargs): """ Get the ``ConfigItem`` by id. @@ -79,13 +82,14 @@ def get_item(self, id: str, resolve: bool = False): Args: id: id of the expected config item. resolve: whether to resolve the item if it is not resolved, default to False. - + kwargs: keyword arguments to pass to ``_resolve_one_item()``. + Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. """ if resolve and id not in self.resolved_content: - self._resolve_one_item(id=id) + self._resolve_one_item(id=id, **kwargs) return self.items.get(id) - def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None): + def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None, **kwargs): """ Resolve one ``ConfigItem`` of ``id``, cache the resolved result in ``resolved_content``. If it has unresolved references, recursively resolve the referring items first. @@ -95,6 +99,8 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None): waiting_list: set of ids pending to be resolved. It's used to detect circular references such as: `{"name": "A", "dep": "@B"}` and `{"name": "B", "dep": "@A"}`. + kwargs: keyword arguments to pass to ``_resolve_one_item()``. + Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. """ item = self.items[id] # if invalid id name, raise KeyError @@ -105,42 +111,46 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None): waiting_list.add(id) ref_ids = self.find_refs_in_config(config=item_config, id=id) - - # if current item has reference already in the waiting list, that's circular references for d in ref_ids: + # if current item has reference already in the waiting list, that's circular references if d in waiting_list: raise ValueError(f"detected circular references for id='{d}' in the config content.") - - # # check whether the component has any unresolved references - for d in ref_ids: + # check whether the component has any unresolved references if d not in self.resolved_content: # this referring item is not resolved - if d not in self.items: - raise ValueError(f"the referring item `{d}` is not defined in the config content.") + try: + look_up_option(d, self.items, print_all_options=False) + except ValueError as err: + raise ValueError(f"the referring item `@{d}` is not defined in the config content.") from err # recursively resolve the reference first - self._resolve_one_item(id=d, waiting_list=waiting_list) + self._resolve_one_item(id=d, waiting_list=waiting_list, **kwargs) + waiting_list.remove(d) # all references are resolved, then try to resolve current config item new_config = self.update_config_with_refs(config=item_config, id=id, refs=self.resolved_content) item.update_config(config=new_config) # save the resolved result into `resolved_content` to recursively resolve others if isinstance(item, ConfigComponent): - self.resolved_content[id] = item.instantiate() + self.resolved_content[id] = item.instantiate() if kwargs.get("instantiate", True) else item elif isinstance(item, ConfigExpression): - self.resolved_content[id] = item.evaluate(locals={"refs": self.resolved_content}) + self.resolved_content[id] = ( + item.evaluate(locals={"refs": self.resolved_content}) if kwargs.get("eval_expr", True) else item + ) else: self.resolved_content[id] = new_config - def get_resolved_content(self, id: str): + def get_resolved_content(self, id: str, **kwargs): """ Get the resolved ``ConfigItem`` by id. If there are unresolved references, try to resolve them first. Args: id: id name of the expected item. + kwargs: additional keyword arguments to be passed to ``_resolve_one_item``. + Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. """ if id not in self.resolved_content: - self._resolve_one_item(id=id) + self._resolve_one_item(id=id, **kwargs) return self.resolved_content[id] @staticmethod diff --git a/monai/utils/module.py b/monai/utils/module.py index ca47ae6b2d..de2152d182 100644 --- a/monai/utils/module.py +++ b/monai/utils/module.py @@ -47,7 +47,7 @@ ] -def look_up_option(opt_str, supported: Union[Collection, enum.EnumMeta], default="no_default"): +def look_up_option(opt_str, supported: Union[Collection, enum.EnumMeta], default="no_default", print_all_options=True): """ Look up the option in the supported collection and return the matched item. Raise a value error possibly with a guess of the closest match. @@ -58,6 +58,7 @@ def look_up_option(opt_str, supported: Union[Collection, enum.EnumMeta], default default: If it is given, this method will return `default` when `opt_str` is not found, instead of raising a `ValueError`. Otherwise, it defaults to `"no_default"`, so that the method may raise a `ValueError`. + print_all_options: whether to print all available options when `opt_str` is not found. Defaults to True Examples: @@ -113,12 +114,12 @@ class Color(Enum): if edit_dist <= 3: edit_dists[key] = edit_dist - supported_msg = f"Available options are {set_to_check}.\n" + supported_msg = f"Available options are {set_to_check}.\n" if print_all_options else "" if edit_dists: guess_at_spelling = min(edit_dists, key=edit_dists.get) # type: ignore raise ValueError( f"By '{opt_str}', did you mean '{guess_at_spelling}'?\n" - + f"'{opt_str}' is not a valid option.\n" + + f"'{opt_str}' is not a valid value.\n" + supported_msg ) raise ValueError(f"Unsupported option '{opt_str}', " + supported_msg) diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index dcb4e0801e..2b58d6959e 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -107,7 +107,6 @@ def test_parse(self, config, expected_ids, output_types): self.assertTrue(isinstance(parser.get_parsed_content(id), cls)) # test root content root = parser.get_parsed_content(id="") - print("!!!!!!!!", root) for v, cls in zip(root.values(), [Compose, Dataset, DataLoader]): self.assertTrue(isinstance(v, cls)) From 43ce0e53685cfdadb0410ddc27a492641bc8a409 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 23 Feb 2022 23:01:36 +0000 Subject: [PATCH 17/76] remove->discard Signed-off-by: Wenqi Li --- monai/apps/manifest/reference_resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 9bb9e43253..212f8640ef 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -124,7 +124,7 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None, ** raise ValueError(f"the referring item `@{d}` is not defined in the config content.") from err # recursively resolve the reference first self._resolve_one_item(id=d, waiting_list=waiting_list, **kwargs) - waiting_list.remove(d) + waiting_list.discard(d) # all references are resolved, then try to resolve current config item new_config = self.update_config_with_refs(config=item_config, id=id, refs=self.resolved_content) From 5fff7d32f984e53e0180f2cf971e3d41f990e3d8 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 24 Feb 2022 11:40:21 +0800 Subject: [PATCH 18/76] [DLMED] minor change Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 15 +++++++++++---- monai/apps/manifest/reference_resolver.py | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index 1a90696e87..df4bd92017 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -45,9 +45,7 @@ class ConfigParser: # in the example $@my_dims + 1 is an expression, which adds 1 to the value of @my_dims parser = ConfigParser(config) - # get/set the configuration content, but do not instantiate the components - trainer = parser.get_parsed_content("trainer", instantiate=False) - print(trainer) + # get/set the configuration source content print(parser["my_net"][""]["in_channels"]) # original input channels 1 parser["my_net"][""]["in_channels"] = 4 # change input channels to 4 print(parser["my_net"][""]["in_channels"]) @@ -57,6 +55,9 @@ class ConfigParser: net = parser.get_parsed_content("my_net", instantiate=True) print(net) + # also support to get the parsed `ConfigItem` content, but do not instantiate the components + trainer = parser.get_parsed_content("trainer", instantiate=False) + print(trainer) Args: config: input config source to parse. @@ -185,9 +186,11 @@ def _do_parse(self, config, id: str = ""): else: self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) - def parse(self, reset: bool = False): + def parse(self, reset: bool = True): """ Recursively parse the config source, add every item as ``ConfigItem`` to the resolver. + Note that the configuration content in every `ConfigItem` will be decoupled from the + configuration source of parser during parsing. Args: reset: whether to reset the ``reference_resolver`` before parsing. Defaults to False. @@ -203,6 +206,10 @@ def get_parsed_content(self, id: str = "", **kwargs): - If the item is ``ConfigComponent`` and ``instantiate=True``, the result is the instance. - If the item is ``ConfigExpression`` and ``eval_expr=True``, the result is the evaluated output. + - Else the result is the configurtion content of `ConfigItem`. + + Note that the configuration content in every `ConfigItem` is decoupled from the configuration + source during parsing. Args: id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 212f8640ef..f643dbd9c0 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -68,7 +68,6 @@ def add_item(self, item: ConfigItem): """ id = item.get_id() if id in self.items: - # warnings.warn(f"id '{id}' is already added.") return self.items[id] = item @@ -84,6 +83,7 @@ def get_item(self, id: str, resolve: bool = False, **kwargs): resolve: whether to resolve the item if it is not resolved, default to False. kwargs: keyword arguments to pass to ``_resolve_one_item()``. Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. + """ if resolve and id not in self.resolved_content: self._resolve_one_item(id=id, **kwargs) From b86f897f4bca8ab2e7af1900cff036fe19604db1 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 24 Feb 2022 18:28:55 +0800 Subject: [PATCH 19/76] [DLMED] update code Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 4 ++-- monai/apps/manifest/reference_resolver.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index 1a90696e87..af6c9f1b9f 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -185,12 +185,12 @@ def _do_parse(self, config, id: str = ""): else: self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) - def parse(self, reset: bool = False): + def parse(self, reset: bool = True): """ Recursively parse the config source, add every item as ``ConfigItem`` to the resolver. Args: - reset: whether to reset the ``reference_resolver`` before parsing. Defaults to False. + reset: whether to reset the ``reference_resolver`` before parsing. Defaults to `True`. """ if reset: diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 212f8640ef..45249668c7 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -68,7 +68,6 @@ def add_item(self, item: ConfigItem): """ id = item.get_id() if id in self.items: - # warnings.warn(f"id '{id}' is already added.") return self.items[id] = item From c8bba6ad07031fbcc6c016b960a56622b757a91d Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 24 Feb 2022 10:43:17 +0000 Subject: [PATCH 20/76] update usage Signed-off-by: Wenqi Li --- monai/apps/manifest/config_parser.py | 3 +++ monai/apps/manifest/reference_resolver.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index af6c9f1b9f..b4e86ab8fe 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -91,6 +91,9 @@ def __init__( self.reference_resolver = ReferenceResolver() self.set(config=config) + def __repr__(self): + return f"{self.config}" + def __getitem__(self, id: Union[str, int]): """ Get the config by id. diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 45249668c7..238dfc55ae 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -102,7 +102,10 @@ def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None, ** Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. """ - item = self.items[id] # if invalid id name, raise KeyError + try: + item = look_up_option(id, self.items, print_all_options=False) + except ValueError as err: + raise KeyError(f"id='{id}' is not found in the config resolver.") from err item_config = item.get_config() if waiting_list is None: From 28c5c07a47b7412911cbb8b1ad0a41d0a02f14cc Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 24 Feb 2022 18:46:20 +0800 Subject: [PATCH 21/76] [DLMED] update doc Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index af6c9f1b9f..d58c0b1337 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -45,9 +45,7 @@ class ConfigParser: # in the example $@my_dims + 1 is an expression, which adds 1 to the value of @my_dims parser = ConfigParser(config) - # get/set the configuration content, but do not instantiate the components - trainer = parser.get_parsed_content("trainer", instantiate=False) - print(trainer) + # get/set configuration content print(parser["my_net"][""]["in_channels"]) # original input channels 1 parser["my_net"][""]["in_channels"] = 4 # change input channels to 4 print(parser["my_net"][""]["in_channels"]) @@ -57,6 +55,9 @@ class ConfigParser: net = parser.get_parsed_content("my_net", instantiate=True) print(net) + # also support to get the configuration content of parsed `ConfigItem` + trainer = parser.get_parsed_content("trainer", instantiate=False) + print(trainer) Args: config: input config source to parse. @@ -161,6 +162,8 @@ def set(self, config: Any, id: str = ""): def _do_parse(self, config, id: str = ""): """ Recursively parse the nested data in config source, add every item as `ConfigItem` to the resolver. + Note that the configuration content of every `ConfigItem` will be decoupled from the config source + of parser during parsing. Args: config: config source to parse. @@ -203,6 +206,10 @@ def get_parsed_content(self, id: str = "", **kwargs): - If the item is ``ConfigComponent`` and ``instantiate=True``, the result is the instance. - If the item is ``ConfigExpression`` and ``eval_expr=True``, the result is the evaluated output. + - Else, the result is the configuration content of `ConfigItem`. + + Note that the configuration content of every `ConfigItem` is decoupled from the config source of parser + during parsing. Args: id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to From 273e968c4c33267fe1ba22b599994b99a4106efe Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 24 Feb 2022 21:34:59 +0800 Subject: [PATCH 22/76] [DLMED] add end-to-end test Signed-off-by: Nic Ma --- monai/apps/manifest/reference_resolver.py | 4 +- monai/apps/manifest/run.py | 11 +- monai/apps/manifest/utils.py | 2 +- tests/min_tests.py | 1 + tests/test_manifest_run.py | 172 ++++++++++++++++++++++ 5 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 tests/test_manifest_run.py diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 665f770b4c..93aebdc482 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -168,7 +168,7 @@ def match_refs_pattern(value: str) -> Set[str]: """ refs: Set[str] = set() # regular expression pattern to match "@XXX" or "@XXX#YYY" - result = re.compile(r"@\w*[\#\w]*").findall(value) + result = re.compile(r"@[<>\w]*[\#<>\w]*").findall(value) for item in result: if ConfigExpression.is_expression(value) or value == item: # only check when string starts with "$" or the whole content is "@XXX" @@ -188,7 +188,7 @@ def update_refs_pattern(value: str, refs: Dict) -> str: """ # regular expression pattern to match "@XXX" or "@XXX#YYY" - result = re.compile(r"@\w*[\#\w]*").findall(value) + result = re.compile(r"@[<>\w]*[\#<>\w]*").findall(value) for item in result: ref_id = item[1:] if ref_id not in refs: diff --git a/monai/apps/manifest/run.py b/monai/apps/manifest/run.py index 1199e57aa9..b38c969d14 100644 --- a/monai/apps/manifest/run.py +++ b/monai/apps/manifest/run.py @@ -28,19 +28,20 @@ def run(): parser.add_argument("--config", "-c", type=str, help="filepath of the config file.", required=True) parser.add_argument("--override", "-o", metavar="ID=VALUE", nargs="*") parser.add_argument( - "--target", "-c", type=str, + "--target", "-t", type=str, help=("ID name of the target workflow, it must have the `run` method, follow MONAI `BaseWorkflow`."), required=True, ) args = parser.parse_args() override = {} - for pair in args.override: - id, v = parse_id_value(pair) - override[id] = v + if args.override is not None: + for pair in args.override: + id, v = parse_id_value(pair) + override[id] = v config_parser = parse_config_files(config_file=args.config, meta_file=args.metadata, override=override) - workflow = config_parser.get_resolved_content(id=args.target) + workflow = config_parser.get_parsed_content(id=args.target) workflow.run() diff --git a/monai/apps/manifest/utils.py b/monai/apps/manifest/utils.py index 0a67bffee8..36505c6cbd 100644 --- a/monai/apps/manifest/utils.py +++ b/monai/apps/manifest/utils.py @@ -11,7 +11,7 @@ from distutils.util import strtobool import json -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, Tuple import yaml from monai.apps.manifest.config_parser import ConfigParser diff --git a/tests/min_tests.py b/tests/min_tests.py index e0710a93ec..3b47c6faa1 100644 --- a/tests/min_tests.py +++ b/tests/min_tests.py @@ -159,6 +159,7 @@ def run_testsuit(): "test_zoomd", "test_prepare_batch_default_dist", "test_parallel_execution_dist", + "test_manifest_run", ] assert sorted(exclude_cases) == sorted(set(exclude_cases)), f"Duplicated items in {exclude_cases}" diff --git a/tests/test_manifest_run.py b/tests/test_manifest_run.py new file mode 100644 index 0000000000..89aceffe2c --- /dev/null +++ b/tests/test_manifest_run.py @@ -0,0 +1,172 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import sys +import os +import unittest +import numpy as np +import tempfile +import nibabel as nib + +from parameterized import parameterized +from monai.transforms import LoadImage + +TEST_CASE_1 = [ + { + "device": "$torch.device('cuda' if torch.cuda.is_available() else 'cpu')", + "network_def": { + "": "UNet", + "": { + "spatial_dims": 3, + "in_channels": 1, + "out_channels": 2, + "channels": [16, 32, 64, 128, 256], + "strides": [2, 2, 2, 2], + "num_res_units": 2, + "norm": "batch" + } + }, + "network": "$@network_def.to(@device)", + "preprocessing": { + "": "Compose", + "": { + "transforms": [ + { + "": "LoadImaged", + "": { + "keys": "image" + } + }, + { + "": "EnsureChannelFirstd", + "": { + "keys": "image" + } + }, + { + "": "ScaleIntensityd", + "": { + "keys": "image" + } + }, + { + "": "EnsureTyped", + "": { + "keys": "image" + } + } + ] + } + }, + "dataset": { + "": "Dataset", + "": { + "data": "@#datalist", # test placeholger with `datalist` + "transform": "@preprocessing" + } + }, + "dataloader": { + "": "DataLoader", + "": { + "dataset": "@dataset", + "batch_size": 1, + "shuffle": False, + "num_workers": 4 + } + }, + "inferer": { + "": "SlidingWindowInferer", + "": { + "roi_size": [96, 96, 96], + "sw_batch_size": 4, + "overlap": 0.5 + } + }, + "postprocessing": { + "": "Compose", + "": { + "transforms": [ + { + "": "Activationsd", + "": { + "keys": "pred", + "softmax": True + } + }, + { + "": "AsDiscreted", + "": { + "keys": "pred", + "argmax": True + } + }, + { + "": "SaveImaged", + "": { + "keys": "pred", + "meta_keys": "image_meta_dict", + "output_dir": "@#output_dir" # test placeholger with `output_dir` + } + } + ] + } + }, + "evaluator": { + "": "SupervisedEvaluator", + "": { + "device": "@device", + "val_data_loader": "@dataloader", + "network": "@network", + "inferer": "@inferer", + "postprocessing": "@postprocessing", + "amp": False + } + } + }, + (128, 128, 128), +] + + +class TestChannelPad(unittest.TestCase): + @parameterized.expand([TEST_CASE_1]) + def test_shape(self, config, expected_shape): + logging.basicConfig(stream=sys.stdout, level=logging.INFO) + test_image = np.random.rand(128, 128, 128) + with tempfile.TemporaryDirectory() as tempdir: + filename = os.path.join(tempdir, "image.nii") + nib.save(nib.Nifti1Image(test_image, np.eye(4)), filename) + + meta = { + "datalist": [{"image": filename}], + "output_dir": tempdir, + "window": (96, 96, 96), + } + metafile = os.path.join(tempdir, "meta.json") + with open(metafile, "w") as f: + json.dump(meta, f) + + configfile = os.path.join(tempdir, "config.json") + with open(configfile, "w") as f: + json.dump(config, f) + + os.system( + f"python -m monai.apps.manifest.run -m {metafile} -c {configfile}" + f" -o 'evaluator##amp'=False -t evaluator" + ) + + saved = LoadImage(image_only=True)(os.path.join(tempdir, "image", "image_trans.nii.gz")) + self.assertTupleEqual(saved.shape, expected_shape) + + +if __name__ == "__main__": + unittest.main() From 3db14f77c122acd167b176e982eb80fe8096676a Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Fri, 25 Feb 2022 01:09:51 +0800 Subject: [PATCH 23/76] [DLMED] update doc Signed-off-by: Nic Ma --- monai/apps/manifest/config_parser.py | 7 +- monai/apps/manifest/reference_resolver.py | 8 +- monai/apps/manifest/run.py | 7 +- monai/apps/manifest/utils.py | 9 +- tests/test_manifest_run.py | 105 +++++++--------------- 5 files changed, 45 insertions(+), 91 deletions(-) diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py index aa44a4f2c8..9651db262b 100644 --- a/monai/apps/manifest/config_parser.py +++ b/monai/apps/manifest/config_parser.py @@ -56,6 +56,7 @@ class ConfigParser: print(net) # also support to get the parsed `ConfigItem` content, but do not instantiate the components + # changing this parsed config content will not affect the config source in the parser trainer = parser.get_parsed_content("trainer", instantiate=False) print(trainer) @@ -165,8 +166,6 @@ def set(self, config: Any, id: str = ""): def _do_parse(self, config, id: str = ""): """ Recursively parse the nested data in config source, add every item as `ConfigItem` to the resolver. - Note that the configuration content of every `ConfigItem` will be decoupled from the config source - of parser during parsing. Args: config: config source to parse. @@ -213,10 +212,6 @@ def get_parsed_content(self, id: str = "", **kwargs): - If the item is ``ConfigExpression`` and ``eval_expr=True``, the result is the evaluated output. - Else, the result is the configuration content of `ConfigItem`. - Note that the configuration content of every `ConfigItem` is decoupled from the config source of parser - during parsing. - - Args: id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to go one level further into the nested structures. diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 93aebdc482..8b6dcfc7c0 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -160,7 +160,7 @@ def get_resolved_content(self, id: str, **kwargs): def match_refs_pattern(value: str) -> Set[str]: """ Match regular expression for the input string to find the references. - The reference string starts with ``"@"``, like: ``"@XXX#YYY#ZZZ"``. + The reference string starts with ``"@"``, like: ``"@XXX##ZZZ"``. Args: value: input value to match regular expression. @@ -168,7 +168,7 @@ def match_refs_pattern(value: str) -> Set[str]: """ refs: Set[str] = set() # regular expression pattern to match "@XXX" or "@XXX#YYY" - result = re.compile(r"@[<>\w]*[\#<>\w]*").findall(value) + result = re.compile(r"@(\<\w+\>|\w+)(\#(\<\w+\>|\w+))*").findall(value) for item in result: if ConfigExpression.is_expression(value) or value == item: # only check when string starts with "$" or the whole content is "@XXX" @@ -179,7 +179,7 @@ def match_refs_pattern(value: str) -> Set[str]: def update_refs_pattern(value: str, refs: Dict) -> str: """ Match regular expression for the input string to update content with the references. - The reference part starts with ``"@"``, like: ``"@XXX#YYY#ZZZ"``. + The reference part starts with ``"@"``, like: ``"@XXX##ZZZ"``. References dictionary must contain the referring IDs as keys. Args: @@ -188,7 +188,7 @@ def update_refs_pattern(value: str, refs: Dict) -> str: """ # regular expression pattern to match "@XXX" or "@XXX#YYY" - result = re.compile(r"@[<>\w]*[\#<>\w]*").findall(value) + result = re.compile(r"@(\<\w+\>|\w+)(\#(\<\w+\>|\w+))*").findall(value) for item in result: ref_id = item[1:] if ref_id not in refs: diff --git a/monai/apps/manifest/run.py b/monai/apps/manifest/run.py index b38c969d14..7e7e2eedce 100644 --- a/monai/apps/manifest/run.py +++ b/monai/apps/manifest/run.py @@ -11,6 +11,7 @@ import argparse + from monai.apps.manifest.utils import parse_config_files, parse_id_value @@ -28,7 +29,9 @@ def run(): parser.add_argument("--config", "-c", type=str, help="filepath of the config file.", required=True) parser.add_argument("--override", "-o", metavar="ID=VALUE", nargs="*") parser.add_argument( - "--target", "-t", type=str, + "--target", + "-t", + type=str, help=("ID name of the target workflow, it must have the `run` method, follow MONAI `BaseWorkflow`."), required=True, ) @@ -45,5 +48,5 @@ def run(): workflow.run() -if __name__ == '__main__': +if __name__ == "__main__": run() diff --git a/monai/apps/manifest/utils.py b/monai/apps/manifest/utils.py index 36505c6cbd..f062cf6793 100644 --- a/monai/apps/manifest/utils.py +++ b/monai/apps/manifest/utils.py @@ -9,9 +9,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from distutils.util import strtobool import json -from typing import Any, Dict, Tuple +from distutils.util import strtobool +from typing import Any, Dict, Optional, Tuple + import yaml from monai.apps.manifest.config_parser import ConfigParser @@ -62,7 +63,7 @@ def parse_id_value(pair: str) -> Tuple[str, Any]: return id, value -def parse_config_files(config_file: str, meta_file: str, override: Dict = {}) -> ConfigParser: +def parse_config_files(config_file: str, meta_file: str, override: Optional[Dict] = None) -> ConfigParser: """ Read the config file, metadata file and override with specified `id=value` pairs. Put metadata in the config content with key "". @@ -84,7 +85,7 @@ def parse_config_files(config_file: str, meta_file: str, override: Dict = {}) -> parser = ConfigParser(config=config) - if len(override) > 0: + if override is not None: for id, v in override.items(): if isinstance(v, str) and v.startswith(""): v = read_config(v[6:]) diff --git a/tests/test_manifest_run.py b/tests/test_manifest_run.py index 89aceffe2c..6618a55494 100644 --- a/tests/test_manifest_run.py +++ b/tests/test_manifest_run.py @@ -11,14 +11,16 @@ import json import logging -import sys import os -import unittest -import numpy as np +import sys import tempfile -import nibabel as nib +import unittest +import nibabel as nib +import numpy as np +import yaml from parameterized import parameterized + from monai.transforms import LoadImage TEST_CASE_1 = [ @@ -33,93 +35,49 @@ "channels": [16, 32, 64, 128, 256], "strides": [2, 2, 2, 2], "num_res_units": 2, - "norm": "batch" - } + "norm": "batch", + }, }, "network": "$@network_def.to(@device)", "preprocessing": { "": "Compose", "": { "transforms": [ - { - "": "LoadImaged", - "": { - "keys": "image" - } - }, - { - "": "EnsureChannelFirstd", - "": { - "keys": "image" - } - }, - { - "": "ScaleIntensityd", - "": { - "keys": "image" - } - }, - { - "": "EnsureTyped", - "": { - "keys": "image" - } - } + {"": "LoadImaged", "": {"keys": "image"}}, + {"": "EnsureChannelFirstd", "": {"keys": "image"}}, + {"": "ScaleIntensityd", "": {"keys": "image"}}, + {"": "EnsureTyped", "": {"keys": "image"}}, ] - } + }, }, "dataset": { "": "Dataset", - "": { - "data": "@#datalist", # test placeholger with `datalist` - "transform": "@preprocessing" - } + "": {"data": "@#datalist", "transform": "@preprocessing"}, # test placeholger with `datalist` }, "dataloader": { "": "DataLoader", - "": { - "dataset": "@dataset", - "batch_size": 1, - "shuffle": False, - "num_workers": 4 - } + "": {"dataset": "@dataset", "batch_size": 1, "shuffle": False, "num_workers": 4}, }, "inferer": { "": "SlidingWindowInferer", - "": { - "roi_size": [96, 96, 96], - "sw_batch_size": 4, - "overlap": 0.5 - } + "": {"roi_size": [96, 96, 96], "sw_batch_size": 4, "overlap": 0.5}, }, "postprocessing": { "": "Compose", "": { "transforms": [ - { - "": "Activationsd", - "": { - "keys": "pred", - "softmax": True - } - }, - { - "": "AsDiscreted", - "": { - "keys": "pred", - "argmax": True - } - }, + {"": "Activationsd", "": {"keys": "pred", "softmax": True}}, + {"": "AsDiscreted", "": {"keys": "pred", "argmax": True}}, { "": "SaveImaged", "": { "keys": "pred", "meta_keys": "image_meta_dict", - "output_dir": "@#output_dir" # test placeholger with `output_dir` - } - } + "output_dir": "@#output_dir", # test placeholger with `output_dir` + }, + }, ] - } + }, }, "evaluator": { "": "SupervisedEvaluator", @@ -129,9 +87,9 @@ "network": "@network", "inferer": "@inferer", "postprocessing": "@postprocessing", - "amp": False - } - } + "amp": False, + }, + }, }, (128, 128, 128), ] @@ -146,15 +104,12 @@ def test_shape(self, config, expected_shape): filename = os.path.join(tempdir, "image.nii") nib.save(nib.Nifti1Image(test_image, np.eye(4)), filename) - meta = { - "datalist": [{"image": filename}], - "output_dir": tempdir, - "window": (96, 96, 96), - } - metafile = os.path.join(tempdir, "meta.json") + meta = {"datalist": [{"image": filename}], "output_dir": tempdir, "window": (96, 96, 96)} + # test YAML file + metafile = os.path.join(tempdir, "meta.yaml") with open(metafile, "w") as f: - json.dump(meta, f) - + yaml.dump(meta, f) + # test JSON file configfile = os.path.join(tempdir, "config.json") with open(configfile, "w") as f: json.dump(config, f) From ba671519d90609038d2b3c4ec343e9940ec1c461 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Fri, 25 Feb 2022 01:22:28 +0800 Subject: [PATCH 24/76] [DLMED] fix flake8 Signed-off-by: Nic Ma --- monai/apps/manifest/utils.py | 4 ++-- requirements-dev.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/monai/apps/manifest/utils.py b/monai/apps/manifest/utils.py index f062cf6793..0e6e8f08c7 100644 --- a/monai/apps/manifest/utils.py +++ b/monai/apps/manifest/utils.py @@ -11,7 +11,7 @@ import json from distutils.util import strtobool -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional, Tuple, Union import yaml @@ -44,7 +44,7 @@ def parse_id_value(pair: str) -> Tuple[str, Any]: items = pair.split("=") # we remove blanks around id id = items[0].strip() - value = "" + value: Union[str, int, float, bool] = "" if len(items) > 1: # rejoin the rest value = "=".join(items[1:]) diff --git a/requirements-dev.txt b/requirements-dev.txt index eaf363fbe4..121f65752c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -43,3 +43,4 @@ transformers mlflow matplotlib!=3.5.0 tensorboardX +types-PyYAML From e1fcb30bd0dea908c318cab68c1e73cf7d3c2176 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 28 Feb 2022 10:48:02 +0800 Subject: [PATCH 25/76] [DLMED] fix optional import Signed-off-by: Nic Ma --- docs/requirements.txt | 1 + docs/source/installation.md | 5 ++--- environment-dev.yml | 1 + monai/apps/manifest/reference_resolver.py | 4 ++-- monai/apps/manifest/utils.py | 5 +++-- requirements-dev.txt | 1 + setup.cfg | 3 +++ 7 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index dcf5ef5b2a..fed93f7dff 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -25,3 +25,4 @@ mlflow tensorboardX imagecodecs; platform_system == "Linux" tifffile; platform_system == "Linux" +pyyaml diff --git a/docs/source/installation.md b/docs/source/installation.md index 15c372c385..d9ea1d1740 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -190,10 +190,9 @@ Since MONAI v0.2.0, the extras syntax such as `pip install 'monai[nibabel]'` is - The options are ``` -[nibabel, skimage, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, cucim, openslide, pandas, einops, transformers, mlflow, matplotlib, tensorboardX, tifffile, imagecodecs] +[nibabel, skimage, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, cucim, openslide, pandas, einops, transformers, mlflow, matplotlib, tensorboardX, tifffile, imagecodecs, pyyaml] ``` which correspond to `nibabel`, `scikit-image`, `pillow`, `tensorboard`, -`gdown`, `pytorch-ignite`, `torchvision`, `itk`, `tqdm`, `lmdb`, `psutil`, `cucim`, `openslide-python`, `pandas`, `einops`, `transformers`, `mlflow`, `matplotlib`, `tensorboardX`, -`tifffile`, `imagecodecs`, respectively. +`gdown`, `pytorch-ignite`, `torchvision`, `itk`, `tqdm`, `lmdb`, `psutil`, `cucim`, `openslide-python`, `pandas`, `einops`, `transformers`, `mlflow`, `matplotlib`, `tensorboardX`, `tifffile`, `imagecodecs`, `pyyaml`, respectively. - `pip install 'monai[all]'` installs all the optional dependencies. diff --git a/environment-dev.yml b/environment-dev.yml index ae41f21f1f..78cd84cf28 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -42,6 +42,7 @@ dependencies: - transformers - mlflow - tensorboardX + - pyyaml - pip - pip: # pip for itk as conda-forge version only up to v5.1 diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py index 8b6dcfc7c0..40bc2a3d50 100644 --- a/monai/apps/manifest/reference_resolver.py +++ b/monai/apps/manifest/reference_resolver.py @@ -168,7 +168,7 @@ def match_refs_pattern(value: str) -> Set[str]: """ refs: Set[str] = set() # regular expression pattern to match "@XXX" or "@XXX#YYY" - result = re.compile(r"@(\<\w+\>|\w+)(\#(\<\w+\>|\w+))*").findall(value) + result = re.compile(r"@(?:(?:<\w*>)|(?:\w*))(?:(?:#<\w*>)|(?:#\w*))*").findall(value) for item in result: if ConfigExpression.is_expression(value) or value == item: # only check when string starts with "$" or the whole content is "@XXX" @@ -188,7 +188,7 @@ def update_refs_pattern(value: str, refs: Dict) -> str: """ # regular expression pattern to match "@XXX" or "@XXX#YYY" - result = re.compile(r"@(\<\w+\>|\w+)(\#(\<\w+\>|\w+))*").findall(value) + result = re.compile(r"@(?:(?:<\w*>)|(?:\w*))(?:(?:#<\w*>)|(?:#\w*))*").findall(value) for item in result: ref_id = item[1:] if ref_id not in refs: diff --git a/monai/apps/manifest/utils.py b/monai/apps/manifest/utils.py index 0e6e8f08c7..66df4b7edc 100644 --- a/monai/apps/manifest/utils.py +++ b/monai/apps/manifest/utils.py @@ -13,9 +13,10 @@ from distutils.util import strtobool from typing import Any, Dict, Optional, Tuple, Union -import yaml - from monai.apps.manifest.config_parser import ConfigParser +from monai.utils import optional_import + +yaml, _ = optional_import("yaml") def read_config(filepath: str): diff --git a/requirements-dev.txt b/requirements-dev.txt index 121f65752c..7beb088d2b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -44,3 +44,4 @@ mlflow matplotlib!=3.5.0 tensorboardX types-PyYAML +pyyaml diff --git a/setup.cfg b/setup.cfg index a9cfa09ccc..5a77c35489 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,6 +50,7 @@ all = mlflow matplotlib tensorboardX + pyyaml nibabel = nibabel skimage = @@ -92,6 +93,8 @@ matplotlib = matplotlib tensorboardX = tensorboardX +pyyaml = + pyyaml [flake8] select = B,C,E,F,N,P,T4,W,B9 From babea2b93057929c625e9beacac7e226443e57c2 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 28 Feb 2022 11:25:55 +0800 Subject: [PATCH 26/76] [DLMED] refine APIs Signed-off-by: Nic Ma --- monai/apps/__init__.py | 2 +- monai/apps/manifest/__init__.py | 2 +- monai/apps/manifest/run.py | 6 ++-- monai/apps/manifest/utils.py | 54 ++++++++++++++++++++++----------- 4 files changed, 42 insertions(+), 22 deletions(-) diff --git a/monai/apps/__init__.py b/monai/apps/__init__.py index c1747479fc..c14e58a420 100644 --- a/monai/apps/__init__.py +++ b/monai/apps/__init__.py @@ -17,9 +17,9 @@ ConfigItem, ConfigParser, ReferenceResolver, + load_config_file, parse_config_files, parse_id_value, - read_config, ) from .mmars import MODEL_DESC, RemoteMMARKeys, download_mmar, get_model_spec, load_from_mmar from .utils import SUPPORTED_HASH_TYPES, check_hash, download_and_extract, download_url, extractall, get_logger, logger diff --git a/monai/apps/manifest/__init__.py b/monai/apps/manifest/__init__.py index 97f23b919c..44b3a340bf 100644 --- a/monai/apps/manifest/__init__.py +++ b/monai/apps/manifest/__init__.py @@ -12,4 +12,4 @@ from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem from .config_parser import ConfigParser from .reference_resolver import ReferenceResolver -from .utils import parse_config_files, parse_id_value, read_config +from .utils import load_config_file, parse_config_files, parse_id_value diff --git a/monai/apps/manifest/run.py b/monai/apps/manifest/run.py index 7e7e2eedce..b973ba9a66 100644 --- a/monai/apps/manifest/run.py +++ b/monai/apps/manifest/run.py @@ -17,9 +17,9 @@ def run(): """ - Specify the metadata file, config file to run a standard training or evaluation program. - It's used to execute most of the supervised training or evaluation cases. - It supports to override the config content with specified `id` and `value`. + Specify a metadata file and a config file to run a regular training or evaluation program. + It's used to execute most of the supervised training, evaluation or inference cases. + It supports to override the config content with specified `id` and `value` pairs. The `override` arg can also be used to provide default value for placeholders. For example: put a placeholder `"data": "@runtime_value"` in the config, then define `runtime_value` in `override`. diff --git a/monai/apps/manifest/utils.py b/monai/apps/manifest/utils.py index 66df4b7edc..b8f0a9dc1d 100644 --- a/monai/apps/manifest/utils.py +++ b/monai/apps/manifest/utils.py @@ -11,31 +11,40 @@ import json from distutils.util import strtobool -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Sequence, Tuple, Union from monai.apps.manifest.config_parser import ConfigParser -from monai.utils import optional_import +from monai.utils import ensure_tuple, optional_import yaml, _ = optional_import("yaml") -def read_config(filepath: str): +def load_config_file(filepath: str, **kwargs): """ - Read config file with specified file path. + Load config file with specified file path. Suppprt JSON and YAML formats. + Args: + filepath: path of target file to load, supported postfixes: `.json`, `yml`, `yaml`. + kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. + for more details, please check: + https://docs.python.org/3/library/json.html#json.load. + https://pyyaml.org/wiki/PyYAMLDocumentation. + """ with open(filepath) as f: if filepath.lower().endswith(".json"): - return json.load(f) + return json.load(f, **kwargs) if filepath.lower().endswith((".yml", ".yaml")): - return yaml.load(f, Loader=yaml.FullLoader) + if "Loader" not in kwargs: + kwargs["Loader"] = yaml.FullLoader + return yaml.load(f, **kwargs) raise ValueError("only support JSON or YAML config file so far.") def parse_id_value(pair: str) -> Tuple[str, Any]: """ - Parse the "id=value" pair to `id` and `value`. + Parse the "id=value" pair string to `id` and `value`. Will try to convert the correct data type of `value` from string. Args: @@ -43,7 +52,7 @@ def parse_id_value(pair: str) -> Tuple[str, Any]: """ items = pair.split("=") - # we remove blanks around id + # remove blanks around id id = items[0].strip() value: Union[str, int, float, bool] = "" if len(items) > 1: @@ -64,7 +73,9 @@ def parse_id_value(pair: str) -> Tuple[str, Any]: return id, value -def parse_config_files(config_file: str, meta_file: str, override: Optional[Dict] = None) -> ConfigParser: +def parse_config_files( + config_file: Union[str, Sequence[str]], meta_file: Union[str, Sequence[str]], override: Optional[Dict] = None +) -> ConfigParser: """ Read the config file, metadata file and override with specified `id=value` pairs. Put metadata in the config content with key "". @@ -73,23 +84,32 @@ def parse_config_files(config_file: str, meta_file: str, override: Optional[Dict and use the content as `value`. Args: - config_file: filepath of the config file. - meta_file: filepath of the metadata file. + config_file: filepath of the config file, the config content must be a dictionary, + if providing a list of files, wil merge the content of them. + meta_file: filepath of the metadata file, the config content must be a dictionary, + if providing a list of files, wil merge the content of them. override: dict of `{id: value}` pairs to override or add the config content. """ - config = read_config(config_file) - if not isinstance(config, dict): - raise ValueError("input config file must be a dictionary.") - - config[""] = read_config(meta_file) + config: Dict = {"": {}} + for f in ensure_tuple(config_file): + content = load_config_file(f) + if not isinstance(content, dict): + raise ValueError("input config content must be a dictionary.") + config.update(content) + + for f in ensure_tuple(meta_file): + content = load_config_file(f) + if not isinstance(content, dict): + raise ValueError("meta data content must be a dictionary.") + config[""].update(content) parser = ConfigParser(config=config) if override is not None: for id, v in override.items(): if isinstance(v, str) and v.startswith(""): - v = read_config(v[6:]) + v = load_config_file(v[6:]) parser[id] = v return parser From 3aac99fb740a050cf725037a694976f9e9b4f300 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 28 Feb 2022 23:08:15 +0800 Subject: [PATCH 27/76] [DLMED] fix typo Signed-off-by: Nic Ma --- monai/apps/manifest/run.py | 4 ++++ tests/test_manifest_run.py | 5 +---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/monai/apps/manifest/run.py b/monai/apps/manifest/run.py index b973ba9a66..fd84f50ceb 100644 --- a/monai/apps/manifest/run.py +++ b/monai/apps/manifest/run.py @@ -11,6 +11,8 @@ import argparse +import logging +import sys from monai.apps.manifest.utils import parse_config_files, parse_id_value @@ -37,6 +39,8 @@ def run(): ) args = parser.parse_args() + logging.basicConfig(stream=sys.stdout, level=logging.INFO) + override = {} if args.override is not None: for pair in args.override: diff --git a/tests/test_manifest_run.py b/tests/test_manifest_run.py index 6618a55494..a38d72b131 100644 --- a/tests/test_manifest_run.py +++ b/tests/test_manifest_run.py @@ -10,9 +10,7 @@ # limitations under the License. import json -import logging import os -import sys import tempfile import unittest @@ -95,10 +93,9 @@ ] -class TestChannelPad(unittest.TestCase): +class TestManifestRun(unittest.TestCase): @parameterized.expand([TEST_CASE_1]) def test_shape(self, config, expected_shape): - logging.basicConfig(stream=sys.stdout, level=logging.INFO) test_image = np.random.rand(128, 128, 128) with tempfile.TemporaryDirectory() as tempdir: filename = os.path.join(tempdir, "image.nii") From dc10f5de42328dac86021505b8528662b51f600d Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 1 Mar 2022 12:21:47 +0800 Subject: [PATCH 28/76] [DLMED] add support to only use part of the file to override Signed-off-by: Nic Ma --- monai/apps/__init__.py | 1 + monai/apps/manifest/__init__.py | 2 +- monai/apps/manifest/utils.py | 28 +++++++++++++++++++++++++++- tests/test_manifest_run.py | 17 ++++++++++++++--- 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/monai/apps/__init__.py b/monai/apps/__init__.py index c14e58a420..fca7ea23e6 100644 --- a/monai/apps/__init__.py +++ b/monai/apps/__init__.py @@ -18,6 +18,7 @@ ConfigParser, ReferenceResolver, load_config_file, + load_config_file_content, parse_config_files, parse_id_value, ) diff --git a/monai/apps/manifest/__init__.py b/monai/apps/manifest/__init__.py index 44b3a340bf..d3b04fc90b 100644 --- a/monai/apps/manifest/__init__.py +++ b/monai/apps/manifest/__init__.py @@ -12,4 +12,4 @@ from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem from .config_parser import ConfigParser from .reference_resolver import ReferenceResolver -from .utils import load_config_file, parse_config_files, parse_id_value +from .utils import load_config_file, load_config_file_content, parse_config_files, parse_id_value diff --git a/monai/apps/manifest/utils.py b/monai/apps/manifest/utils.py index b8f0a9dc1d..9aa4c1dd85 100644 --- a/monai/apps/manifest/utils.py +++ b/monai/apps/manifest/utils.py @@ -10,6 +10,7 @@ # limitations under the License. import json +import re from distutils.util import strtobool from typing import Any, Dict, Optional, Sequence, Tuple, Union @@ -42,6 +43,31 @@ def load_config_file(filepath: str, **kwargs): raise ValueError("only support JSON or YAML config file so far.") +def load_config_file_content(path: str, **kwargs): + """ + Load part of the content from a config file with specified `id` in the path. + Suppprt JSON and YAML formats file. + + Args: + path: path of target file to load, it can only load part of it appending target `id` + in the path with "#" mark. for example: `/data/config.json`, `/data/config.json#net#`. + kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. + for more details, please check: + https://docs.python.org/3/library/json.html#json.load. + https://pyyaml.org/wiki/PyYAMLDocumentation. + + """ + pattern = r"(json|yaml|yml)" + result = re.findall(pattern, path, re.IGNORECASE) + if len(result) != 1: + raise ValueError(f"path should only contain 1 file, but got: {path}.") + + # split the path into filepath and target id of the content + paths = path.split(result[0]) + parser = ConfigParser(config=load_config_file(paths[0] + result[0], **kwargs)) + return parser[paths[1][1:] if paths[1] != "" else ""] + + def parse_id_value(pair: str) -> Tuple[str, Any]: """ Parse the "id=value" pair string to `id` and `value`. @@ -109,7 +135,7 @@ def parse_config_files( if override is not None: for id, v in override.items(): if isinstance(v, str) and v.startswith(""): - v = load_config_file(v[6:]) + v = load_config_file_content(v[6:]) parser[id] = v return parser diff --git a/tests/test_manifest_run.py b/tests/test_manifest_run.py index a38d72b131..ad27a0ae46 100644 --- a/tests/test_manifest_run.py +++ b/tests/test_manifest_run.py @@ -36,7 +36,7 @@ "norm": "batch", }, }, - "network": "$@network_def.to(@device)", + "network": "will be overrided", "preprocessing": { "": "Compose", "": { @@ -49,7 +49,7 @@ }, }, "dataset": { - "": "Dataset", + "": "will be overrided", "": {"data": "@#datalist", "transform": "@preprocessing"}, # test placeholger with `datalist` }, "dataloader": { @@ -111,9 +111,20 @@ def test_shape(self, config, expected_shape): with open(configfile, "w") as f: json.dump(config, f) + # test override with file, up case postfix + overridefile1 = os.path.join(tempdir, "override1.JSON") + with open(overridefile1, "w") as f: + # test override with part of the overriding file + json.dump({"move_net": "$@network_def.to(@device)"}, f) + overridefile2 = os.path.join(tempdir, "override2.JSON") + with open(overridefile2, "w") as f: + # test override with the whole overriding file + json.dump("Dataset", f) + os.system( f"python -m monai.apps.manifest.run -m {metafile} -c {configfile}" - f" -o 'evaluator##amp'=False -t evaluator" + f" -o 'evaluator##amp'=False 'network'='{overridefile1}#move_net'" + f" 'dataset#'='{overridefile2}' -t evaluator" ) saved = LoadImage(image_only=True)(os.path.join(tempdir, "image", "image_trans.nii.gz")) From 053bd6bc20bcafcf5e9c896a02bf6f9ac631dee5 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 1 Mar 2022 15:53:10 +0800 Subject: [PATCH 29/76] [DLMED] add more tests Signed-off-by: Nic Ma --- tests/test_manifest_run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_manifest_run.py b/tests/test_manifest_run.py index ad27a0ae46..09f4cb7a5c 100644 --- a/tests/test_manifest_run.py +++ b/tests/test_manifest_run.py @@ -121,11 +121,12 @@ def test_shape(self, config, expected_shape): # test override with the whole overriding file json.dump("Dataset", f) - os.system( + ret = os.system( f"python -m monai.apps.manifest.run -m {metafile} -c {configfile}" f" -o 'evaluator##amp'=False 'network'='{overridefile1}#move_net'" f" 'dataset#'='{overridefile2}' -t evaluator" ) + self.assertEqual(ret, 0) saved = LoadImage(image_only=True)(os.path.join(tempdir, "image", "image_trans.nii.gz")) self.assertTupleEqual(saved.shape, expected_shape) From 8b390e6f5737ab3f1ebfcae8f7737348bd483a61 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 1 Mar 2022 18:06:03 +0800 Subject: [PATCH 30/76] [DLMED] update to latest Signed-off-by: Nic Ma --- docs/source/apps.rst | 27 +- monai/apps/__init__.py | 13 +- monai/apps/manifest/__init__.py | 3 - monai/apps/manifest/config_item.py | 338 ---------------------- monai/apps/manifest/config_parser.py | 227 --------------- monai/apps/manifest/reference_resolver.py | 255 ---------------- monai/apps/manifest/run.py | 1 - 7 files changed, 3 insertions(+), 861 deletions(-) delete mode 100644 monai/apps/manifest/config_item.py delete mode 100644 monai/apps/manifest/config_parser.py delete mode 100644 monai/apps/manifest/reference_resolver.py diff --git a/docs/source/apps.rst b/docs/source/apps.rst index 8c0214d584..239ae9eb17 100644 --- a/docs/source/apps.rst +++ b/docs/source/apps.rst @@ -19,8 +19,8 @@ Applications :members: -Clara MMARs ------------ +`Clara MMARs` +------------- .. autofunction:: download_mmar .. autofunction:: load_from_mmar @@ -29,29 +29,6 @@ Clara MMARs :annotation: -Model Manifest --------------- - -.. autoclass:: ComponentLocator - :members: - -.. autoclass:: ConfigComponent - :members: - -.. autoclass:: ConfigExpression - :members: - -.. autoclass:: ConfigItem - :members: - -.. autoclass:: ConfigParser - :members: - :special-members: __getitem__, __setitem__ - -.. autoclass:: ReferenceResolver - :members: - - `Utilities` ----------- diff --git a/monai/apps/__init__.py b/monai/apps/__init__.py index fca7ea23e6..cf48405bbd 100644 --- a/monai/apps/__init__.py +++ b/monai/apps/__init__.py @@ -10,17 +10,6 @@ # limitations under the License. from .datasets import CrossValidation, DecathlonDataset, MedNISTDataset -from .manifest import ( - ComponentLocator, - ConfigComponent, - ConfigExpression, - ConfigItem, - ConfigParser, - ReferenceResolver, - load_config_file, - load_config_file_content, - parse_config_files, - parse_id_value, -) +from .manifest import load_config_file, load_config_file_content, parse_config_files, parse_id_value from .mmars import MODEL_DESC, RemoteMMARKeys, download_mmar, get_model_spec, load_from_mmar from .utils import SUPPORTED_HASH_TYPES, check_hash, download_and_extract, download_url, extractall, get_logger, logger diff --git a/monai/apps/manifest/__init__.py b/monai/apps/manifest/__init__.py index d3b04fc90b..5321661436 100644 --- a/monai/apps/manifest/__init__.py +++ b/monai/apps/manifest/__init__.py @@ -9,7 +9,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem -from .config_parser import ConfigParser -from .reference_resolver import ReferenceResolver from .utils import load_config_file, load_config_file_content, parse_config_files, parse_id_value diff --git a/monai/apps/manifest/config_item.py b/monai/apps/manifest/config_item.py deleted file mode 100644 index a6406795cf..0000000000 --- a/monai/apps/manifest/config_item.py +++ /dev/null @@ -1,338 +0,0 @@ -# Copyright (c) MONAI Consortium -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import inspect -import sys -import warnings -from abc import ABC, abstractmethod -from importlib import import_module -from typing import Any, Dict, List, Mapping, Optional, Sequence, Union - -from monai.utils import ensure_tuple, instantiate - -__all__ = ["ComponentLocator", "ConfigItem", "ConfigExpression", "ConfigComponent"] - - -class Instantiable(ABC): - """ - Base class for an instantiable object. - """ - - @abstractmethod - def is_disabled(self, *args: Any, **kwargs: Any) -> bool: - """ - Return a boolean flag to indicate whether the object should be instantiated. - """ - raise NotImplementedError(f"subclass {self.__class__.__name__} must implement this method.") - - @abstractmethod - def instantiate(self, *args: Any, **kwargs: Any) -> object: - """ - Instantiate the target component and return the instance. - """ - raise NotImplementedError(f"subclass {self.__class__.__name__} must implement this method.") - - -class ComponentLocator: - """ - Scan all the available classes and functions in the MONAI package and map them with the module paths in a table. - It's used to locate the module path for provided component name. - - Args: - excludes: if any string of the `excludes` exists in the full module name, don't import this module. - - """ - - MOD_START = "monai" - - def __init__(self, excludes: Optional[Union[Sequence[str], str]] = None): - self.excludes = [] if excludes is None else ensure_tuple(excludes) - self._components_table: Optional[Dict[str, List]] = None - - def _find_module_names(self) -> List[str]: - """ - Find all the modules start with MOD_START and don't contain any of `excludes`. - - """ - return [ - m for m in sys.modules.keys() if m.startswith(self.MOD_START) and all(s not in m for s in self.excludes) - ] - - def _find_classes_or_functions(self, modnames: Union[Sequence[str], str]) -> Dict[str, List]: - """ - Find all the classes and functions in the modules with specified `modnames`. - - Args: - modnames: names of the target modules to find all the classes and functions. - - """ - table: Dict[str, List] = {} - # all the MONAI modules are already loaded by `load_submodules` - for modname in ensure_tuple(modnames): - try: - # scan all the classes and functions in the module - module = import_module(modname) - for name, obj in inspect.getmembers(module): - if (inspect.isclass(obj) or inspect.isfunction(obj)) and obj.__module__ == modname: - if name not in table: - table[name] = [] - table[name].append(modname) - except ModuleNotFoundError: - pass - return table - - def get_component_module_name(self, name: str) -> Optional[Union[List[str], str]]: - """ - Get the full module name of the class or function with specified ``name``. - If target component name exists in multiple packages or modules, return a list of full module names. - - Args: - name: name of the expected class or function. - - """ - if not isinstance(name, str): - raise ValueError(f"`name` must be a valid string, but got: {name}.") - if self._components_table is None: - # init component and module mapping table - self._components_table = self._find_classes_or_functions(self._find_module_names()) - - mods: Optional[Union[List[str], str]] = self._components_table.get(name, None) - if isinstance(mods, list) and len(mods) == 1: - mods = mods[0] - return mods - - -class ConfigItem: - """ - Basic data structure to represent a configuration item. - - A `ConfigItem` instance can optionally have a string id, so that other items can refer to it. - It has a build-in `config` property to store the configuration object. - - Args: - config: content of a config item, can be objects of any types, - a configuration resolver may interpret the content to generate a configuration object. - id: name of the current config item, defaults to empty string. - - """ - - def __init__(self, config: Any, id: str = "") -> None: - self.config = config - self.id = id - - def get_id(self) -> str: - """ - Get the ID name of current config item, useful to identify config items during parsing. - - """ - return self.id - - def update_config(self, config: Any): - """ - Replace the content of `self.config` with new `config`. - A typical usage is to modify the initial config content at runtime. - - Args: - config: content of a `ConfigItem`. - - """ - self.config = config - - def get_config(self): - """ - Get the config content of current config item. - - """ - return self.config - - def __repr__(self) -> str: - return str(self.config) - - -class ConfigComponent(ConfigItem, Instantiable): - """ - Subclass of :py:class:`monai.apps.ConfigItem`, this class uses a dictionary with string keys to - represent a component of `class` or `function` and supports instantiation. - - Currently, four special keys (strings surrounded by ``<>``) are defined and interpreted beyond the regular literals: - - - class or function identifier of the python module, specified by one of the two keys. - - ``""``: indicates build-in python classes or functions such as "LoadImageDict". - - ``""``: full module name, such as "monai.transforms.LoadImageDict". - - ``""``: input arguments to the python module. - - ``""``: a flag to indicate whether to skip the instantiation. - - .. code-block:: python - - locator = ComponentLocator(excludes=["modules_to_exclude"]) - config = { - "": "LoadImaged", - "": { - "keys": ["image", "label"] - } - } - - configer = ConfigComponent(config, id="test", locator=locator) - image_loader = configer.instantiate() - print(image_loader) # - - Args: - config: content of a config item. - id: name of the current config item, defaults to empty string. - locator: a ``ComponentLocator`` to convert a module name string into the actual python module. - if `None`, a ``ComponentLocator(excludes=excludes)`` will be used. - excludes: if ``locator`` is None, create a new ``ComponentLocator`` with ``excludes``. - See also: :py:class:`monai.apps.manifest.ComponentLocator`. - - """ - - def __init__( - self, - config: Any, - id: str = "", - locator: Optional[ComponentLocator] = None, - excludes: Optional[Union[Sequence[str], str]] = None, - ) -> None: - super().__init__(config=config, id=id) - self.locator = ComponentLocator(excludes=excludes) if locator is None else locator - - @staticmethod - def is_instantiable(config: Any) -> bool: - """ - Check whether this config represents a `class` or `function` that is to be instantiated. - - Args: - config: input config content to check. - - """ - return isinstance(config, Mapping) and ("" in config or "" in config) - - def resolve_module_name(self): - """ - Resolve the target module name from current config content. - The config content must have ``""`` or ``""``. - When both are specified, ``""`` will be used. - - """ - config = dict(self.get_config()) - path = config.get("") - if path is not None: - if not isinstance(path, str): - raise ValueError(f"'' must be a string, but got: {path}.") - if "" in config: - warnings.warn(f"both '' and '', default to use '': {path}.") - return path - - name = config.get("") - if not isinstance(name, str): - raise ValueError("must provide a string for `` or `` of target component to instantiate.") - - module = self.locator.get_component_module_name(name) - if module is None: - raise ModuleNotFoundError(f"can not find component '{name}' in {self.locator.MOD_START} modules.") - if isinstance(module, list): - warnings.warn( - f"there are more than 1 component have name `{name}`: {module}, use the first one `{module[0]}." - f" if want to use others, please set its module path in `` directly." - ) - module = module[0] - return f"{module}.{name}" - - def resolve_args(self): - """ - Utility function used in `instantiate()` to resolve the arguments from current config content. - - """ - return self.get_config().get("", {}) - - def is_disabled(self) -> bool: # type: ignore - """ - Utility function used in `instantiate()` to check whether to skip the instantiation. - - """ - _is_disabled = self.get_config().get("", False) - return _is_disabled.lower().strip() == "true" if isinstance(_is_disabled, str) else bool(_is_disabled) - - def instantiate(self, **kwargs) -> object: # type: ignore - """ - Instantiate component based on ``self.config`` content. - The target component must be a `class` or a `function`, otherwise, return `None`. - - Args: - kwargs: args to override / add the config args when instantiation. - - """ - if not self.is_instantiable(self.get_config()) or self.is_disabled(): - # if not a class or function or marked as `disabled`, skip parsing and return `None` - return None - - modname = self.resolve_module_name() - args = self.resolve_args() - args.update(kwargs) - return instantiate(modname, **args) - - -class ConfigExpression(ConfigItem): - """ - Subclass of :py:class:`monai.apps.ConfigItem`, the `ConfigItem` represents an executable expression - (execute based on ``eval()``). - - See also: - - - https://docs.python.org/3/library/functions.html#eval. - - For example: - - .. code-block:: python - - import monai - from monai.apps.manifest import ConfigExpression - - config = "$monai.__version__" - expression = ConfigExpression(config, id="test", globals={"monai": monai}) - print(expression.execute()) - - Args: - config: content of a config item. - id: name of current config item, defaults to empty string. - globals: additional global context to evaluate the string. - - """ - - def __init__(self, config: Any, id: str = "", globals: Optional[Dict] = None) -> None: - super().__init__(config=config, id=id) - self.globals = globals - - def evaluate(self, locals: Optional[Dict] = None): - """ - Execute the current config content and return the result if it is expression, based on Python `eval()`. - For more details: https://docs.python.org/3/library/functions.html#eval. - - Args: - locals: besides ``globals``, may also have some local symbols used in the expression at runtime. - - """ - value = self.get_config() - if not ConfigExpression.is_expression(value): - return None - return eval(value[1:], self.globals, locals) - - @staticmethod - def is_expression(config: Union[Dict, List, str]) -> bool: - """ - Check whether the config is an executable expression string. - Currently, a string starts with ``"$"`` character is interpreted as an expression. - - Args: - config: input config content to check. - - """ - return isinstance(config, str) and config.startswith("$") diff --git a/monai/apps/manifest/config_parser.py b/monai/apps/manifest/config_parser.py deleted file mode 100644 index 9651db262b..0000000000 --- a/monai/apps/manifest/config_parser.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright (c) MONAI Consortium -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import importlib -from copy import deepcopy -from typing import Any, Dict, Optional, Sequence, Union - -from monai.apps.manifest.config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem -from monai.apps.manifest.reference_resolver import ReferenceResolver - - -class ConfigParser: - """ - The primary configuration parser. It traverses a structured config (in the form of nested Python dict or list), - creates ``ConfigItem``, and assign unique IDs according to the structures. - - This class provides convenient access to the set of ``ConfigItem`` of the config by ID. - A typical workflow of config parsing is as follows: - - - Initialize ``ConfigParser`` with the ``config`` source. - - Call ``get_parsed_content()`` to get expected component with `id`. - - .. code-block:: python - - from monai.apps import ConfigParser - - - config = { - "my_dims": 2, - "dims_1": "$@my_dims + 1", - "my_xform": {"": "LoadImage"}, - "my_net": {"": "BasicUNet", - "": {"spatial_dims": "@dims_1", "in_channels": 1, "out_channels": 4}}, - "trainer": {"": "SupervisedTrainer", - "": {"network": "@my_net", "preprocessing": "@my_xform"}} - } - # in the example $@my_dims + 1 is an expression, which adds 1 to the value of @my_dims - parser = ConfigParser(config) - - # get/set the configuration source content - print(parser["my_net"][""]["in_channels"]) # original input channels 1 - parser["my_net"][""]["in_channels"] = 4 # change input channels to 4 - print(parser["my_net"][""]["in_channels"]) - - # instantiate the network component - parser.parse(True) - net = parser.get_parsed_content("my_net", instantiate=True) - print(net) - - # also support to get the parsed `ConfigItem` content, but do not instantiate the components - # changing this parsed config content will not affect the config source in the parser - trainer = parser.get_parsed_content("trainer", instantiate=False) - print(trainer) - - Args: - config: input config source to parse. - excludes: when importing modules to instantiate components, - excluding components from modules specified in ``excludes``. - globals: pre-import packages as global variables to ``ConfigExpression``, - so that expressions, for example, ``"$monai.data.list_data_collate"`` can use ``monai`` modules. - The current supported globals and alias names are - ``{"monai": "monai", "torch": "torch", "np": "numpy", "numpy": "numpy"}``. - These are MONAI's minimal dependencies. - - See also: - - - :py:class:`monai.apps.ConfigItem` - - """ - - def __init__( - self, - config: Any, - excludes: Optional[Union[Sequence[str], str]] = None, - globals: Optional[Dict[str, Any]] = None, - ): - self.config = None - self.globals: Dict[str, Any] = {} - globals = {"monai": "monai", "torch": "torch", "np": "numpy", "numpy": "numpy"} if globals is None else globals - if globals is not None: - for k, v in globals.items(): - self.globals[k] = importlib.import_module(v) if isinstance(v, str) else v - - self.locator = ComponentLocator(excludes=excludes) - self.reference_resolver = ReferenceResolver() - self.set(config=config) - - def __repr__(self): - return f"{self.config}" - - def __getitem__(self, id: Union[str, int]): - """ - Get the config by id. - - Args: - id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to - go one level further into the nested structures. - Use digits indexing from "0" for list or other strings for dict. - For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. - - """ - if id == "": - return self.config - config = self.config - for k in str(id).split("#"): - if not isinstance(config, (dict, list)): - raise ValueError(f"config must be dict or list for key `{k}`, but got {type(config)}: {config}.") - indexing = k if isinstance(config, dict) else int(k) - config = config[indexing] - return config - - def __setitem__(self, id: Union[str, int], config: Any): - """ - Set config by ``id``. - - Args: - id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to - go one level further into the nested structures. - Use digits indexing from "0" for list or other strings for dict. - For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. - config: config to set at location ``id``. - - """ - if id == "": - self.config = config - self.reference_resolver.reset() - return - keys = str(id).split("#") - # get the last second config item and replace it - last_id = "#".join(keys[:-1]) - conf_ = self[last_id] - indexing = keys[-1] if isinstance(conf_, dict) else int(keys[-1]) - conf_[indexing] = config - self.reference_resolver.reset() - return - - def get(self, id: str = "", default: Optional[Any] = None): - """ - Get the config by id. - - Args: - id: id to specify the expected position. See also :py:meth:`__getitem__`. - default: default value to return if the specified ``id`` is invalid. - - """ - try: - return self[id] - except KeyError: - return default - - def set(self, config: Any, id: str = ""): - """ - Set config by ``id``. See also :py:meth:`__setitem__`. - - """ - self[id] = config - - def _do_parse(self, config, id: str = ""): - """ - Recursively parse the nested data in config source, add every item as `ConfigItem` to the resolver. - - Args: - config: config source to parse. - id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to - go one level further into the nested structures. - Use digits indexing from "0" for list or other strings for dict. - For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. - - """ - if isinstance(config, (dict, list)): - subs = enumerate(config) if isinstance(config, list) else config.items() - for k, v in subs: - sub_id = f"{id}#{k}" if id != "" else k - self._do_parse(config=v, id=sub_id) - - # copy every config item to make them independent and add them to the resolver - item_conf = deepcopy(config) - if ConfigComponent.is_instantiable(item_conf): - self.reference_resolver.add_item(ConfigComponent(config=item_conf, id=id, locator=self.locator)) - elif ConfigExpression.is_expression(item_conf): - self.reference_resolver.add_item(ConfigExpression(config=item_conf, id=id, globals=self.globals)) - else: - self.reference_resolver.add_item(ConfigItem(config=item_conf, id=id)) - - def parse(self, reset: bool = True): - """ - Recursively parse the config source, add every item as ``ConfigItem`` to the resolver. - Note that the configuration content in every `ConfigItem` will be decoupled from the - configuration source of parser during parsing. - - Args: - reset: whether to reset the ``reference_resolver`` before parsing. Defaults to `True`. - - """ - if reset: - self.reference_resolver.reset() - self._do_parse(config=self.config) - - def get_parsed_content(self, id: str = "", **kwargs): - """ - Get the parsed result of ``ConfigItem`` with the specified ``id``. - - - If the item is ``ConfigComponent`` and ``instantiate=True``, the result is the instance. - - If the item is ``ConfigExpression`` and ``eval_expr=True``, the result is the evaluated output. - - Else, the result is the configuration content of `ConfigItem`. - - Args: - id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to - go one level further into the nested structures. - Use digits indexing from "0" for list or other strings for dict. - For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. - kwargs: additional keyword arguments to be passed to ``_resolve_one_item``. - Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. - - """ - if not self.reference_resolver.is_resolved(): - # not parsed the config source yet, parse it - self.parse() - return self.reference_resolver.get_resolved_content(id=id, **kwargs) diff --git a/monai/apps/manifest/reference_resolver.py b/monai/apps/manifest/reference_resolver.py deleted file mode 100644 index 40bc2a3d50..0000000000 --- a/monai/apps/manifest/reference_resolver.py +++ /dev/null @@ -1,255 +0,0 @@ -# Copyright (c) MONAI Consortium -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -from typing import Any, Dict, Optional, Sequence, Set - -from monai.apps.manifest.config_item import ConfigComponent, ConfigExpression, ConfigItem -from monai.utils import look_up_option - - -class ReferenceResolver: - """ - Utility class to manage a set of ``ConfigItem`` and resolve the references between them. - - This class maintains a set of ``ConfigItem`` objects and their associated IDs. - The IDs must be unique within this set. A string in ``ConfigItem`` - starting with ``@`` will be treated as a reference to other ``ConfigItem`` objects by ID. - Since ``ConfigItem`` may have a nested dictionary or list structure, - the reference string may also contain a ``#`` character to refer to a substructure by - key indexing for a dictionary or integer indexing for a list. - - In this class, resolving references is essentially substitution of the reference strings with the - corresponding python objects. A typical workflow of resolving references is as follows: - - - Add multiple ``ConfigItem`` objects to the ``ReferenceResolver`` by ``add_item()``. - - Call ``get_resolved_content()`` to automatically resolve the references. This is done (recursively) by: - - Convert the items to objects, for those do not have references to other items. - - If it is instantiable, instantiate it and cache the class instance in ``resolved_content``. - - If it is an expression, evaluate it and save the value in ``resolved_content``. - - Substitute the reference strings with the corresponding objects. - - Args: - items: ``ConfigItem``s to resolve, this could be added later with ``add_item()``. - - """ - - def __init__(self, items: Optional[Sequence[ConfigItem]] = None): - # save the items in a dictionary with the `ConfigItem.id` as key - self.items: Dict[str, Any] = {} if items is None else {i.get_id(): i for i in items} - self.resolved_content: Dict[str, Any] = {} - - def reset(self): - """ - Clear all the added `ConfigItem` and all the resolved content. - - """ - self.items = {} - self.resolved_content = {} - - def is_resolved(self) -> bool: - return bool(self.resolved_content) - - def add_item(self, item: ConfigItem): - """ - Add a ``ConfigItem`` to the resolver. - - Args: - item: a ``ConfigItem``. - - """ - id = item.get_id() - if id in self.items: - return - self.items[id] = item - - def get_item(self, id: str, resolve: bool = False, **kwargs): - """ - Get the ``ConfigItem`` by id. - - If ``resolve=True``, the returned item will be resolved, that is, - all the reference strings are substituted by the corresponding ``ConfigItem`` objects. - - Args: - id: id of the expected config item. - resolve: whether to resolve the item if it is not resolved, default to False. - kwargs: keyword arguments to pass to ``_resolve_one_item()``. - Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. - - """ - if resolve and id not in self.resolved_content: - self._resolve_one_item(id=id, **kwargs) - return self.items.get(id) - - def _resolve_one_item(self, id: str, waiting_list: Optional[Set[str]] = None, **kwargs): - """ - Resolve one ``ConfigItem`` of ``id``, cache the resolved result in ``resolved_content``. - If it has unresolved references, recursively resolve the referring items first. - - Args: - id: id name of ``ConfigItem`` to be resolved. - waiting_list: set of ids pending to be resolved. - It's used to detect circular references such as: - `{"name": "A", "dep": "@B"}` and `{"name": "B", "dep": "@A"}`. - kwargs: keyword arguments to pass to ``_resolve_one_item()``. - Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. - - """ - try: - item = look_up_option(id, self.items, print_all_options=False) - except ValueError as err: - raise KeyError(f"id='{id}' is not found in the config resolver.") from err - item_config = item.get_config() - - if waiting_list is None: - waiting_list = set() - waiting_list.add(id) - - ref_ids = self.find_refs_in_config(config=item_config, id=id) - for d in ref_ids: - # if current item has reference already in the waiting list, that's circular references - if d in waiting_list: - raise ValueError(f"detected circular references for id='{d}' in the config content.") - # check whether the component has any unresolved references - if d not in self.resolved_content: - # this referring item is not resolved - try: - look_up_option(d, self.items, print_all_options=False) - except ValueError as err: - raise ValueError(f"the referring item `@{d}` is not defined in the config content.") from err - # recursively resolve the reference first - self._resolve_one_item(id=d, waiting_list=waiting_list, **kwargs) - waiting_list.discard(d) - - # all references are resolved, then try to resolve current config item - new_config = self.update_config_with_refs(config=item_config, id=id, refs=self.resolved_content) - item.update_config(config=new_config) - # save the resolved result into `resolved_content` to recursively resolve others - if isinstance(item, ConfigComponent): - self.resolved_content[id] = item.instantiate() if kwargs.get("instantiate", True) else item - elif isinstance(item, ConfigExpression): - self.resolved_content[id] = ( - item.evaluate(locals={"refs": self.resolved_content}) if kwargs.get("eval_expr", True) else item - ) - else: - self.resolved_content[id] = new_config - - def get_resolved_content(self, id: str, **kwargs): - """ - Get the resolved ``ConfigItem`` by id. If there are unresolved references, try to resolve them first. - - Args: - id: id name of the expected item. - kwargs: additional keyword arguments to be passed to ``_resolve_one_item``. - Currently support ``instantiate`` and ``eval_expr``. Both are defaulting to True. - - """ - if id not in self.resolved_content: - self._resolve_one_item(id=id, **kwargs) - return self.resolved_content[id] - - @staticmethod - def match_refs_pattern(value: str) -> Set[str]: - """ - Match regular expression for the input string to find the references. - The reference string starts with ``"@"``, like: ``"@XXX##ZZZ"``. - - Args: - value: input value to match regular expression. - - """ - refs: Set[str] = set() - # regular expression pattern to match "@XXX" or "@XXX#YYY" - result = re.compile(r"@(?:(?:<\w*>)|(?:\w*))(?:(?:#<\w*>)|(?:#\w*))*").findall(value) - for item in result: - if ConfigExpression.is_expression(value) or value == item: - # only check when string starts with "$" or the whole content is "@XXX" - refs.add(item[1:]) - return refs - - @staticmethod - def update_refs_pattern(value: str, refs: Dict) -> str: - """ - Match regular expression for the input string to update content with the references. - The reference part starts with ``"@"``, like: ``"@XXX##ZZZ"``. - References dictionary must contain the referring IDs as keys. - - Args: - value: input value to match regular expression. - refs: all the referring components with ids as keys, default to `None`. - - """ - # regular expression pattern to match "@XXX" or "@XXX#YYY" - result = re.compile(r"@(?:(?:<\w*>)|(?:\w*))(?:(?:#<\w*>)|(?:#\w*))*").findall(value) - for item in result: - ref_id = item[1:] - if ref_id not in refs: - raise KeyError(f"can not find expected ID '{ref_id}' in the references.") - if ConfigExpression.is_expression(value): - # replace with local code, will be used in the `evaluate` logic with `locals={"refs": ...}` - value = value.replace(item, f"refs['{ref_id}']") - elif value == item: - # the whole content is "@XXX", it will avoid the case that regular string contains "@" - value = refs[ref_id] - return value - - @staticmethod - def find_refs_in_config(config, id: str, refs: Optional[Set[str]] = None) -> Set[str]: - """ - Recursively search all the content of input config item to get the ids of references. - References mean: the IDs of other config items (``"@XXX"`` in this config item), or the - sub-item in the config is `instantiable`, or the sub-item in the config is `expression`. - For `dict` and `list`, recursively check the sub-items. - - Args: - config: input config content to search. - id: ID name for the input config item. - refs: list of the ID name of found references, default to `None`. - - """ - refs_: Set[str] = refs or set() - if isinstance(config, str): - return refs_.union(ReferenceResolver.match_refs_pattern(value=config)) - if not isinstance(config, (list, dict)): - return refs_ - for k, v in config.items() if isinstance(config, dict) else enumerate(config): - sub_id = f"{id}#{k}" if id != "" else f"{k}" - if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): - refs_.add(sub_id) - refs_ = ReferenceResolver.find_refs_in_config(v, sub_id, refs_) - return refs_ - - @staticmethod - def update_config_with_refs(config, id: str, refs: Optional[Dict] = None): - """ - With all the references in ``refs``, update the input config content with references - and return the new config. - - Args: - config: input config content to update. - id: ID name for the input config. - refs: all the referring content with ids, default to `None`. - - """ - refs_: Dict = refs or {} - if isinstance(config, str): - return ReferenceResolver.update_refs_pattern(config, refs_) - if not isinstance(config, (list, dict)): - return config - ret = type(config)() - for idx, v in config.items() if isinstance(config, dict) else enumerate(config): - sub_id = f"{id}#{idx}" if id != "" else f"{idx}" - if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): - updated = refs_[sub_id] - else: - updated = ReferenceResolver.update_config_with_refs(v, sub_id, refs_) - ret.update({idx: updated}) if isinstance(ret, dict) else ret.append(updated) - return ret diff --git a/monai/apps/manifest/run.py b/monai/apps/manifest/run.py index fd84f50ceb..4d658a549c 100644 --- a/monai/apps/manifest/run.py +++ b/monai/apps/manifest/run.py @@ -9,7 +9,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - import argparse import logging import sys From c5a2c3313b885d376c636106bf98eb2dda05b930 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Wed, 2 Mar 2022 19:08:56 +0800 Subject: [PATCH 31/76] [DLMED] update according to comments Signed-off-by: Nic Ma --- docs/source/bundle.rst | 10 ++++++- environment-dev.yml | 1 + monai/bundle/__init__.py | 10 ++++++- monai/bundle/run.py | 55 -------------------------------------- monai/bundle/scripts.py | 57 ++++++++++++++++++++++++++++++++++++++++ monai/bundle/utils.py | 45 ++++++++++++++++++++++++------- requirements-dev.txt | 1 + setup.cfg | 3 +++ tests/test_bundle_run.py | 27 ++++++++++++------- 9 files changed, 134 insertions(+), 75 deletions(-) delete mode 100644 monai/bundle/run.py create mode 100644 monai/bundle/scripts.py diff --git a/docs/source/bundle.rst b/docs/source/bundle.rst index a3c2178995..1f34c9ea21 100644 --- a/docs/source/bundle.rst +++ b/docs/source/bundle.rst @@ -33,12 +33,20 @@ Model Bundle .. autoclass:: ConfigParser :members: +`Scripts` +--------- +.. autofunction:: run + `Utilities` ----------- .. autofunction:: load_config_file .. autofunction:: load_config_file_content -.. autofunction:: parse_config_files +.. autofunction:: update_default_args .. autofunction:: parse_id_value + +.. autofunction:: id_value_str_to_dict + +.. autofunction:: parse_config_file diff --git a/environment-dev.yml b/environment-dev.yml index 78cd84cf28..4491f87ceb 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -43,6 +43,7 @@ dependencies: - mlflow - tensorboardX - pyyaml + - fire - pip - pip: # pip for itk as conda-forge version only up to v5.1 diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py index 9465a7a91d..adc7fac20b 100644 --- a/monai/bundle/__init__.py +++ b/monai/bundle/__init__.py @@ -12,4 +12,12 @@ from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, Instantiable from .config_parser import ConfigParser from .reference_resolver import ReferenceResolver -from .utils import load_config_file, load_config_file_content, parse_config_files, parse_id_value +from .scripts import run +from .utils import ( + id_value_str_to_dict, + load_config_file, + load_config_file_content, + parse_config_file, + parse_id_value, + update_default_args, +) diff --git a/monai/bundle/run.py b/monai/bundle/run.py deleted file mode 100644 index bff1fe2928..0000000000 --- a/monai/bundle/run.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) MONAI Consortium -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import logging -import sys - -from monai.bundle.utils import parse_config_files, parse_id_value - - -def run(): - """ - Specify a metadata file and a config file to run a regular training or evaluation program. - It's used to execute most of the supervised training, evaluation or inference cases. - It supports to override the config content with specified `id` and `value` pairs. - The `override` arg can also be used to provide default value for placeholders. For example: - put a placeholder `"data": "@runtime_value"` in the config, then define `runtime_value` in `override`. - - """ - parser = argparse.ArgumentParser() - parser.add_argument("--metadata", "-m", type=str, help="filepath of the metadata file.", required=True) - parser.add_argument("--config", "-c", type=str, help="filepath of the config file.", required=True) - parser.add_argument("--override", "-o", metavar="ID=VALUE", nargs="*") - parser.add_argument( - "--target", - "-t", - type=str, - help=("ID name of the target workflow, it must have the `run` method, follow MONAI `BaseWorkflow`."), - required=True, - ) - - args = parser.parse_args() - logging.basicConfig(stream=sys.stdout, level=logging.INFO) - - override = {} - if args.override is not None: - for pair in args.override: - id, v = parse_id_value(pair) - override[id] = v - config_parser = parse_config_files(config_file=args.config, meta_file=args.metadata, override=override) - - workflow = config_parser.get_parsed_content(id=args.target) - workflow.run() - - -if __name__ == "__main__": - run() diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py new file mode 100644 index 0000000000..2439d536cc --- /dev/null +++ b/monai/bundle/scripts.py @@ -0,0 +1,57 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional, Sequence, Union + +from monai.bundle.utils import id_value_str_to_dict, parse_config_file, update_default_args + + +def run( + meta_file: Union[str, Sequence[str]] = None, + config_file: Union[str, Sequence[str]] = None, + override: Optional[Union[Dict, str]] = None, + target: Optional[str] = None, + args_file: Optional[str] = None, +): + """ + Specify metadata file and config file to run a regular training or evaluation program. + It's used to execute most of the supervised training, evaluation or inference cases. + + Args: + meta_file: filepath of the metadata file, if None, must provide it in `arg_file`. + if providing a list of files, wil merge the content of them. + config_file: filepath of the config file, if None, must provide it in `arg_file`. + if providing a list of files, wil merge the content of them. + override: override above config content with specified `id` and `value` pairs. + it can also be used to provide default value for placeholders. for example: + put a placeholder `"data": "@runtime_value"` in the config, then define + `runtime_value` in `override`. + it also supports a string representing a dict, like: "{'AA#BB': 123}", usually from command line. + target: ID name of the target workflow, it must have the `run` method, follow MONAI `BaseWorkflow`. + if None, must provide it in `arg_file`. + args_file: to avoid providing same args every time running the program, it supports + to put the args as a dictionary in a JSON or YAML file. + + """ + + if isinstance(override, str): + # if override is a string representing a dict (usually from command line), convert it to dict + override = id_value_str_to_dict(override) + args = update_default_args( + args=args_file, meta_file=meta_file, config_file=config_file, override=override, target=target + ) + + config_parser = parse_config_file( + config_file=args.get("config_file"), meta_file=args.get("meta_file"), override=args.get("override") + ) + # get expected workflow to run + workflow = config_parser.get_parsed_content(id=args.get("target")) + workflow.run() diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index b585f0d71a..3d192448c9 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -37,9 +37,7 @@ def load_config_file(filepath: str, **kwargs): if filepath.lower().endswith(".json"): return json.load(f, **kwargs) if filepath.lower().endswith((".yml", ".yaml")): - if "Loader" not in kwargs: - kwargs["Loader"] = yaml.FullLoader - return yaml.load(f, **kwargs) + return yaml.safe_load(f, **kwargs) raise ValueError("only support JSON or YAML config file so far.") @@ -68,22 +66,42 @@ def load_config_file_content(path: str, **kwargs): return parser[paths[1][1:] if paths[1] != "" else ""] +def update_default_args(args: Optional[Union[str, Dict]] = None, **kwargs) -> Dict: + """ + Update the `args` with the input `kwargs`. + For dict data, recursively update the content based on the keys. + + Args: + args: source args to update. + kwargs: destination args to update. + + """ + args = {} if args is None else args + if isinstance(args, str): + args = load_config_file_content(args) + + # recursively update the default args with new args + for k, v in kwargs.items(): + args[k] = update_default_args(args[k], **v) if isinstance(v, dict) and isinstance(args.get(k), dict) else v + return args + + def parse_id_value(pair: str) -> Tuple[str, Any]: """ - Parse the "id=value" pair string to `id` and `value`. + Parse the "id:value" pair string to `id` and `value`. Will try to convert the correct data type of `value` from string. Args: - pair (str): input "id=value" pair to parse. + pair (str): input "id:value" pair to parse. """ - items = pair.split("=") + items = pair.split(":") # remove blanks around id id = items[0].strip() value: Union[str, int, float, bool] = "" if len(items) > 1: - # rejoin the rest - value = "=".join(items[1:]) + # rejoin the rest, and remove blanks around value + value = ":".join(items[1:]).strip() # try to convert the correct data type try: @@ -99,7 +117,16 @@ def parse_id_value(pair: str) -> Tuple[str, Any]: return id, value -def parse_config_files( +def id_value_str_to_dict(pairs: str) -> Dict[str, Any]: + """ + Utility to convert a string which represents a dict of `id:value` pairs to a python dict. + Will try to convert the correct data type of `value` from string. + + """ + return dict(map(parse_id_value, pairs[1:-1].split(","))) + + +def parse_config_file( config_file: Union[str, Sequence[str]], meta_file: Union[str, Sequence[str]], override: Optional[Dict] = None ) -> ConfigParser: """ diff --git a/requirements-dev.txt b/requirements-dev.txt index 7beb088d2b..2b3786d1f3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -45,3 +45,4 @@ matplotlib!=3.5.0 tensorboardX types-PyYAML pyyaml +fire diff --git a/setup.cfg b/setup.cfg index 5a77c35489..8ee71553fd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,6 +51,7 @@ all = matplotlib tensorboardX pyyaml + fire nibabel = nibabel skimage = @@ -95,6 +96,8 @@ tensorboardX = tensorboardX pyyaml = pyyaml +fire = + fire [flake8] select = B,C,E,F,N,P,T4,W,B9 diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 12a82662c2..77f2067317 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -11,6 +11,8 @@ import json import os +import subprocess +import sys import tempfile import unittest @@ -101,14 +103,19 @@ def test_shape(self, config, expected_shape): filename = os.path.join(tempdir, "image.nii") nib.save(nib.Nifti1Image(test_image, np.eye(4)), filename) + # generate default args in a file + def_args = os.path.join(tempdir, "def_args.json") + with open(def_args, "w") as f: + json.dump({"config_file": "will be overrided by `config_file` arg"}, f) + meta = {"datalist": [{"image": filename}], "output_dir": tempdir, "window": (96, 96, 96)} # test YAML file - metafile = os.path.join(tempdir, "meta.yaml") - with open(metafile, "w") as f: - yaml.dump(meta, f) + meta_file = os.path.join(tempdir, "meta.yaml") + with open(meta_file, "w") as f: + yaml.safe_dump(meta, f) # test JSON file - configfile = os.path.join(tempdir, "config.json") - with open(configfile, "w") as f: + config_file = os.path.join(tempdir, "config.json") + with open(config_file, "w") as f: json.dump(config, f) # test override with file, up case postfix @@ -121,10 +128,12 @@ def test_shape(self, config, expected_shape): # test override with the whole overriding file json.dump("Dataset", f) - ret = os.system( - f"python -m monai.bundle.run -m {metafile} -c {configfile}" - f" -o 'evaluator##amp'=False 'network'='{overridefile1}#move_net'" - f" 'dataset#'='{overridefile2}' -t evaluator" + # here test the script with `google fire` tool as CLI + ret = subprocess.check_call( + f"{sys.executable} -m fire monai.bundle run --meta_file={meta_file} --config_file={config_file}" + f" --override='{{evaluator##amp: False, network: {overridefile1}#move_net," + f" dataset#: {overridefile2}}}' --target=evaluator", + shell=True, ) self.assertEqual(ret, 0) From ac83c02b99ecd571e52b35a9f264810723060a07 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Wed, 2 Mar 2022 19:24:38 +0800 Subject: [PATCH 32/76] [DLMED] fix flake8 Signed-off-by: Nic Ma --- monai/bundle/scripts.py | 8 ++++---- monai/bundle/utils.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index 2439d536cc..eec0c42350 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -15,8 +15,8 @@ def run( - meta_file: Union[str, Sequence[str]] = None, - config_file: Union[str, Sequence[str]] = None, + meta_file: Optional[Union[str, Sequence[str]]] = None, + config_file: Optional[Union[str, Sequence[str]]] = None, override: Optional[Union[Dict, str]] = None, target: Optional[str] = None, args_file: Optional[str] = None, @@ -50,8 +50,8 @@ def run( ) config_parser = parse_config_file( - config_file=args.get("config_file"), meta_file=args.get("meta_file"), override=args.get("override") + config_file=args["config_file"], meta_file=args["meta_file"], override=args["override"] ) # get expected workflow to run - workflow = config_parser.get_parsed_content(id=args.get("target")) + workflow = config_parser.get_parsed_content(id=args["target"]) workflow.run() diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 3d192448c9..5578f557a1 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -76,14 +76,14 @@ def update_default_args(args: Optional[Union[str, Dict]] = None, **kwargs) -> Di kwargs: destination args to update. """ - args = {} if args is None else args + args_: Dict = args if isinstance(args, dict) is None else {} # type: ignore if isinstance(args, str): - args = load_config_file_content(args) + args_ = load_config_file_content(args) # recursively update the default args with new args for k, v in kwargs.items(): - args[k] = update_default_args(args[k], **v) if isinstance(v, dict) and isinstance(args.get(k), dict) else v - return args + args_[k] = update_default_args(args_[k], **v) if isinstance(v, dict) and isinstance(args_.get(k), dict) else v + return args_ def parse_id_value(pair: str) -> Tuple[str, Any]: From e3f9639f00c4d3e6af766e1b021e3c0c581ab1c1 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Wed, 2 Mar 2022 20:48:43 +0800 Subject: [PATCH 33/76] [DLMED] update according to comments Signed-off-by: Nic Ma --- docs/requirements.txt | 1 + docs/source/installation.md | 4 ++-- monai/bundle/scripts.py | 7 +++++++ tests/test_bundle_run.py | 16 +++++++++++++--- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index fed93f7dff..c2d0f22dcb 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -26,3 +26,4 @@ tensorboardX imagecodecs; platform_system == "Linux" tifffile; platform_system == "Linux" pyyaml +fire diff --git a/docs/source/installation.md b/docs/source/installation.md index d9ea1d1740..29cf1eab66 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -190,9 +190,9 @@ Since MONAI v0.2.0, the extras syntax such as `pip install 'monai[nibabel]'` is - The options are ``` -[nibabel, skimage, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, cucim, openslide, pandas, einops, transformers, mlflow, matplotlib, tensorboardX, tifffile, imagecodecs, pyyaml] +[nibabel, skimage, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, cucim, openslide, pandas, einops, transformers, mlflow, matplotlib, tensorboardX, tifffile, imagecodecs, pyyaml, fire] ``` which correspond to `nibabel`, `scikit-image`, `pillow`, `tensorboard`, -`gdown`, `pytorch-ignite`, `torchvision`, `itk`, `tqdm`, `lmdb`, `psutil`, `cucim`, `openslide-python`, `pandas`, `einops`, `transformers`, `mlflow`, `matplotlib`, `tensorboardX`, `tifffile`, `imagecodecs`, `pyyaml`, respectively. +`gdown`, `pytorch-ignite`, `torchvision`, `itk`, `tqdm`, `lmdb`, `psutil`, `cucim`, `openslide-python`, `pandas`, `einops`, `transformers`, `mlflow`, `matplotlib`, `tensorboardX`, `tifffile`, `imagecodecs`, `pyyaml`, `fire`, respectively. - `pip install 'monai[all]'` installs all the optional dependencies. diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index eec0c42350..213589e174 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -12,6 +12,9 @@ from typing import Dict, Optional, Sequence, Union from monai.bundle.utils import id_value_str_to_dict, parse_config_file, update_default_args +from monai.utils import optional_import + +fire, _ = optional_import("fire") def run( @@ -55,3 +58,7 @@ def run( # get expected workflow to run workflow = config_parser.get_parsed_content(id=args["target"]) workflow.run() + + +if __name__ == "__main__": + fire.Fire() diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 77f2067317..ae016f7503 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -128,6 +128,8 @@ def test_shape(self, config, expected_shape): # test override with the whole overriding file json.dump("Dataset", f) + saver = LoadImage(image_only=True) + # here test the script with `google fire` tool as CLI ret = subprocess.check_call( f"{sys.executable} -m fire monai.bundle run --meta_file={meta_file} --config_file={config_file}" @@ -136,9 +138,17 @@ def test_shape(self, config, expected_shape): shell=True, ) self.assertEqual(ret, 0) - - saved = LoadImage(image_only=True)(os.path.join(tempdir, "image", "image_trans.nii.gz")) - self.assertTupleEqual(saved.shape, expected_shape) + self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_trans.nii.gz")).shape, expected_shape) + # test with `monai.bundle.scripts` as CLI entry directly + ret = subprocess.check_call( + f"{sys.executable} -m monai.bundle.scripts run --meta_file={meta_file} --config_file={config_file}" + f" --override='{{postprocessing##transforms#2##output_postfix: seg," + f" network: {overridefile1}#move_net," + f" dataset#: {overridefile2}}}' --target=evaluator", + shell=True, + ) + self.assertEqual(ret, 0) + self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_seg.nii.gz")).shape, expected_shape) if __name__ == "__main__": From bbdfc05e0b40782e8017a6a220fc2b5065a21971 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Wed, 2 Mar 2022 22:38:45 +0800 Subject: [PATCH 34/76] [DLMED] update `fire` supported dict arg Signed-off-by: Nic Ma --- tests/test_bundle_run.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index ae016f7503..c2292fc98f 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -132,16 +132,30 @@ def test_shape(self, config, expected_shape): # here test the script with `google fire` tool as CLI ret = subprocess.check_call( - f"{sys.executable} -m fire monai.bundle run --meta_file={meta_file} --config_file={config_file}" - f" --override='{{evaluator##amp: False, network: {overridefile1}#move_net," - f" dataset#: {overridefile2}}}' --target=evaluator", - shell=True, + [ + f"{sys.executable}", + "-m", + "fire", + "monai.bundle", + "run", + "--meta_file", + f"{meta_file}", + "--config_file", + f"{config_file}", + "--override", + # `fire` can parse below string as a dictionary + f"{{'evaluator##amp':False,'network':'{overridefile1}#move_net', \ + 'dataset#':'{overridefile2}'}}", + "--target", + "evaluator", + ] ) self.assertEqual(ret, 0) self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_trans.nii.gz")).shape, expected_shape) # test with `monai.bundle.scripts` as CLI entry directly ret = subprocess.check_call( f"{sys.executable} -m monai.bundle.scripts run --meta_file={meta_file} --config_file={config_file}" + # `fire` can not parse below string as a dictionary, will pass to `run` as a string representing a dict f" --override='{{postprocessing##transforms#2##output_postfix: seg," f" network: {overridefile1}#move_net," f" dataset#: {overridefile2}}}' --target=evaluator", From 248f300c28e7eafa637cca2a1bad5ba9545967d4 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Wed, 2 Mar 2022 23:07:54 +0800 Subject: [PATCH 35/76] [DLMED] add examples to doc-string Signed-off-by: Nic Ma --- monai/bundle/scripts.py | 24 ++++++++++++++++++++++++ monai/bundle/utils.py | 3 ++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index 213589e174..e8afa613ac 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -28,6 +28,30 @@ def run( Specify metadata file and config file to run a regular training or evaluation program. It's used to execute most of the supervised training, evaluation or inference cases. + Typical usage examples: + + 1. Execute the `run` API with other CLI tools, take `fire` for example: + `python -m fire monai.bundle run --meta_file= --config_file= --target=trainer` + + 2. Execute this module as CLI entry based on `fire`: + `python -m monai.bundle.scripts run --meta_file= --config_file= --target=trainer` + + 3. Override some config values at runtime, set `override` as a dict: + `python -m monai.bundle.scripts run --override={"'net##ndims'": 2} ...` + + 4. Override some config values at runtime, set `override` as a string: + `python -m monai.bundle.scripts run --override="{net##ndims: 2}" ...` + + 5. Override some config values with another config file: + `python -m monai.bundle.scripts run --override={"'net#'": "'/data/other.json'"} ...` + + 6. Override some config values with part content of another config file: + `python -m monai.bundle.scripts run --override={"'net#'": "'/data/other.json#net_arg'"} ...` + + 7. Set default args of `run` in a JSON / YAML file, help to record and simplify the command line. + Other args still can override the default args at runtime: + `python -m monai.bundle.scripts run --args_file="'/data/args.json'" --config_file=` + Args: meta_file: filepath of the metadata file, if None, must provide it in `arg_file`. if providing a list of files, wil merge the content of them. diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 5578f557a1..1e8691e0c9 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -119,7 +119,8 @@ def parse_id_value(pair: str) -> Tuple[str, Any]: def id_value_str_to_dict(pairs: str) -> Dict[str, Any]: """ - Utility to convert a string which represents a dict of `id:value` pairs to a python dict. + Utility to convert a string which represents a dict of `id:value` pairs to a python dict. For example: + `"{postprocessing##postfix: output, network: other.json#net_args}"` Will try to convert the correct data type of `value` from string. """ From fa67a39fb9d824f40ac4f762e4d7e381d6ae94bb Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 3 Mar 2022 00:38:31 +0800 Subject: [PATCH 36/76] [DLMED] skip windows test for paths Signed-off-by: Nic Ma --- tests/test_bundle_run.py | 46 +++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index c2292fc98f..0b14347804 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -130,28 +130,6 @@ def test_shape(self, config, expected_shape): saver = LoadImage(image_only=True) - # here test the script with `google fire` tool as CLI - ret = subprocess.check_call( - [ - f"{sys.executable}", - "-m", - "fire", - "monai.bundle", - "run", - "--meta_file", - f"{meta_file}", - "--config_file", - f"{config_file}", - "--override", - # `fire` can parse below string as a dictionary - f"{{'evaluator##amp':False,'network':'{overridefile1}#move_net', \ - 'dataset#':'{overridefile2}'}}", - "--target", - "evaluator", - ] - ) - self.assertEqual(ret, 0) - self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_trans.nii.gz")).shape, expected_shape) # test with `monai.bundle.scripts` as CLI entry directly ret = subprocess.check_call( f"{sys.executable} -m monai.bundle.scripts run --meta_file={meta_file} --config_file={config_file}" @@ -164,6 +142,30 @@ def test_shape(self, config, expected_shape): self.assertEqual(ret, 0) self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_seg.nii.gz")).shape, expected_shape) + if sys.platform != "win32": + # here test the script with `google fire` tool as CLI + ret = subprocess.check_call( + [ + f"{sys.executable}", + "-m", + "fire", + "monai.bundle", + "run", + "--meta_file", + f"{meta_file}", + "--config_file", + f"{config_file}", + "--override", + # `fire` can parse below string as a dictionary in Linux env + f"{{'evaluator##amp':False,'network':'{overridefile1}#move_net', \ + 'dataset#':'{overridefile2}'}}", + "--target", + "evaluator", + ] + ) + self.assertEqual(ret, 0) + self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_trans.nii.gz")).shape, expected_shape) + if __name__ == "__main__": unittest.main() From 0abb8e5a5aecdf5409a1551f3f2d5d6e8d8ce802 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 3 Mar 2022 07:59:48 +0800 Subject: [PATCH 37/76] [DLMED] skip windows Signed-off-by: Nic Ma --- tests/test_bundle_run.py | 47 ++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 0b14347804..746b83c49f 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -22,6 +22,7 @@ from parameterized import parameterized from monai.transforms import LoadImage +from tests.utils import skip_if_windows TEST_CASE_1 = [ { @@ -96,6 +97,7 @@ class TestBundleRun(unittest.TestCase): + @skip_if_windows @parameterized.expand([TEST_CASE_1]) def test_shape(self, config, expected_shape): test_image = np.random.rand(128, 128, 128) @@ -142,29 +144,28 @@ def test_shape(self, config, expected_shape): self.assertEqual(ret, 0) self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_seg.nii.gz")).shape, expected_shape) - if sys.platform != "win32": - # here test the script with `google fire` tool as CLI - ret = subprocess.check_call( - [ - f"{sys.executable}", - "-m", - "fire", - "monai.bundle", - "run", - "--meta_file", - f"{meta_file}", - "--config_file", - f"{config_file}", - "--override", - # `fire` can parse below string as a dictionary in Linux env - f"{{'evaluator##amp':False,'network':'{overridefile1}#move_net', \ - 'dataset#':'{overridefile2}'}}", - "--target", - "evaluator", - ] - ) - self.assertEqual(ret, 0) - self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_trans.nii.gz")).shape, expected_shape) + # here test the script with `google fire` tool as CLI + ret = subprocess.check_call( + [ + f"{sys.executable}", + "-m", + "fire", + "monai.bundle", + "run", + "--meta_file", + f"{meta_file}", + "--config_file", + f"{config_file}", + "--override", + # `fire` can parse below string as a dictionary + f"{{'evaluator##amp':False,'network':'{overridefile1}#move_net', \ + 'dataset#':'{overridefile2}'}}", + "--target", + "evaluator", + ] + ) + self.assertEqual(ret, 0) + self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_trans.nii.gz")).shape, expected_shape) if __name__ == "__main__": From c1fefc8f65671fa7b7f4121a067d0ca7d6b28bf2 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 3 Mar 2022 18:44:45 +0800 Subject: [PATCH 38/76] [DLMED] update windows Signed-off-by: Nic Ma --- tests/test_bundle_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 746b83c49f..b83b35f018 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -96,8 +96,8 @@ ] +@skip_if_windows class TestBundleRun(unittest.TestCase): - @skip_if_windows @parameterized.expand([TEST_CASE_1]) def test_shape(self, config, expected_shape): test_image = np.random.rand(128, 128, 128) From 2ab3847961763ce50e59796ebbcb2ba9e1d88b42 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 3 Mar 2022 19:29:28 +0800 Subject: [PATCH 39/76] [DLMED] update command Signed-off-by: Nic Ma --- tests/test_bundle_run.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index b83b35f018..be642d0c94 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -22,7 +22,6 @@ from parameterized import parameterized from monai.transforms import LoadImage -from tests.utils import skip_if_windows TEST_CASE_1 = [ { @@ -96,7 +95,6 @@ ] -@skip_if_windows class TestBundleRun(unittest.TestCase): @parameterized.expand([TEST_CASE_1]) def test_shape(self, config, expected_shape): @@ -134,12 +132,22 @@ def test_shape(self, config, expected_shape): # test with `monai.bundle.scripts` as CLI entry directly ret = subprocess.check_call( - f"{sys.executable} -m monai.bundle.scripts run --meta_file={meta_file} --config_file={config_file}" - # `fire` can not parse below string as a dictionary, will pass to `run` as a string representing a dict - f" --override='{{postprocessing##transforms#2##output_postfix: seg," - f" network: {overridefile1}#move_net," - f" dataset#: {overridefile2}}}' --target=evaluator", - shell=True, + [ + f"{sys.executable}", + "-m", + "monai.bundle.scripts", + "run", + "--meta_file", + f"{meta_file}", + "--config_file", + f"{config_file}", + # `fire` can not parse below string, will pass to `run` as a string representing a dict + "--override", + f"{{postprocessing##transforms#2##output_postfix: seg, \ + network: {overridefile1}#move_net, dataset#: {overridefile2}}}", + "--target", + "evaluator", + ] ) self.assertEqual(ret, 0) self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_seg.nii.gz")).shape, expected_shape) From a3ce5ac362425bc06d4fa70d29fbb9cc0af45180 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 3 Mar 2022 20:15:40 +0800 Subject: [PATCH 40/76] [DLMED] fix windows test Signed-off-by: Nic Ma --- tests/test_bundle_run.py | 45 ++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index be642d0c94..1a7868168f 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -152,28 +152,29 @@ def test_shape(self, config, expected_shape): self.assertEqual(ret, 0) self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_seg.nii.gz")).shape, expected_shape) - # here test the script with `google fire` tool as CLI - ret = subprocess.check_call( - [ - f"{sys.executable}", - "-m", - "fire", - "monai.bundle", - "run", - "--meta_file", - f"{meta_file}", - "--config_file", - f"{config_file}", - "--override", - # `fire` can parse below string as a dictionary - f"{{'evaluator##amp':False,'network':'{overridefile1}#move_net', \ - 'dataset#':'{overridefile2}'}}", - "--target", - "evaluator", - ] - ) - self.assertEqual(ret, 0) - self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_trans.nii.gz")).shape, expected_shape) + if sys.platform != "win32": + # here test the script with `google fire` tool as CLI + ret = subprocess.check_call( + [ + f"{sys.executable}", + "-m", + "fire", + "monai.bundle", + "run", + "--meta_file", + f"{meta_file}", + "--config_file", + f"{config_file}", + "--override", + # `fire` can parse below string as a dictionary + f"{{'evaluator##amp':False,'network':'{overridefile1}#move_net', \ + 'dataset#':'{overridefile2}'}}", + "--target", + "evaluator", + ] + ) + self.assertEqual(ret, 0) + self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_trans.nii.gz")).shape, expected_shape) if __name__ == "__main__": From 605d6302f79697632eef48141dd7c0e4060422cb Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 7 Mar 2022 11:46:10 +0800 Subject: [PATCH 41/76] [DLMED] construct class Signed-off-by: Nic Ma --- monai/bundle/__init__.py | 4 +- monai/bundle/script.py | 197 +++++++++++++++++++++++++++++++++++++++ monai/bundle/scripts.py | 88 ----------------- monai/bundle/utils.py | 77 ++------------- tests/test_bundle_run.py | 6 +- 5 files changed, 209 insertions(+), 163 deletions(-) create mode 100644 monai/bundle/script.py delete mode 100644 monai/bundle/scripts.py diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py index adc7fac20b..c9e4ff43f1 100644 --- a/monai/bundle/__init__.py +++ b/monai/bundle/__init__.py @@ -12,12 +12,10 @@ from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, Instantiable from .config_parser import ConfigParser from .reference_resolver import ReferenceResolver -from .scripts import run +from .script import Script, Run from .utils import ( id_value_str_to_dict, load_config_file, load_config_file_content, - parse_config_file, parse_id_value, - update_default_args, ) diff --git a/monai/bundle/script.py b/monai/bundle/script.py new file mode 100644 index 0000000000..ad7a0f456d --- /dev/null +++ b/monai/bundle/script.py @@ -0,0 +1,197 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from typing import Any, Dict, Optional, Sequence, Union + +from monai.bundle.config_parser import ConfigParser +from monai.bundle.utils import id_value_str_to_dict, load_config_file, load_config_file_content +from monai.utils import ensure_tuple, optional_import + +fire, _ = optional_import("fire") + + +class Script(ABC): + """ + Base class for typical config based scripts in the bundle. + Typical usage examples: + + 1. Execute the `run` API with other CLI tools, take `fire` for example: + `python -m fire monai.bundle run --meta_file= --config_file= --target=trainer` + + 2. Execute this module as CLI entry based on `fire`: + `python -m monai.bundle.scripts run --meta_file= --config_file= --target=trainer` + + 3. Override some config values at runtime, set `override` as a dict: + `python -m monai.bundle.scripts run --override={"'net##ndims'": 2} ...` + + 4. Override some config values at runtime, set `override` as a string: + `python -m monai.bundle.scripts run --override="{net##ndims: 2}" ...` + + 5. Override some config values with another config file: + `python -m monai.bundle.scripts run --override={"'net#'": "'/data/other.json'"} ...` + + 6. Override some config values with part content of another config file: + `python -m monai.bundle.scripts run --override={"'net#'": "'/data/other.json#net_arg'"} ...` + + 7. Set default args of `run` in a JSON / YAML file, help to record and simplify the command line. + Other args still can override the default args at runtime: + `python -m monai.bundle.scripts run --args_file="'/data/args.json'" --config_file=` + + Args: + meta_file: filepath of the metadata file, if None, must provide it in `arg_file`. + if providing a list of files, wil merge the content of them. + config_file: filepath of the config file, if None, must provide it in `arg_file`. + if providing a list of files, wil merge the content of them. + override: override above config content with specified `id` and `value` pairs. + it can also be used to provide default value for placeholders. for example: + put a placeholder `"data": "@runtime_value"` in the config, then define + `runtime_value` in `override`. + it also supports a string representing a dict, like: "{'AA#BB': 123}", usually from command line. + args_file: to avoid providing same args every time running the program, it supports + to put the args as a dictionary in a JSON or YAML file. + kwargs: other args that can set in `args_file` and override at runtime. + + """ + def __init__( + self, + meta_file: Optional[Union[str, Sequence[str]]] = None, + config_file: Optional[Union[str, Sequence[str]]] = None, + override: Optional[Union[Dict, str]] = None, + args_file: Optional[str] = None, + **kwargs, + ): + if isinstance(override, str): + # if override is a string representing a dict (usually from command line), convert it to dict + override = id_value_str_to_dict(override) + self.args = self._update_default_args( + args=args_file, meta_file=meta_file, config_file=config_file, override=override, **kwargs, + ) + self.parser = self.parse_config_file( + config_file=self.args["config_file"], meta_file=self.args["meta_file"], override=self.args["override"] + ) + + def _update_default_args(self, args: Optional[Union[str, Dict]] = None, **kwargs) -> Dict: + """ + Update the `args` with the input `kwargs`. + For dict data, recursively update the content based on the keys. + + Args: + args: source args to update. + kwargs: destination args to update. + + """ + args_: Dict = args if isinstance(args, dict) is None else {} # type: ignore + if isinstance(args, str): + args_ = load_config_file_content(args) + + # recursively update the default args with new args + for k, v in kwargs.items(): + if isinstance(v, dict) and isinstance(args_.get(k), dict): + args_[k] = self._update_default_args(args_[k], **v) + else: + args_[k] = v + return args_ + + def parse_config_file( + self, + config_file: Union[str, Sequence[str]], + meta_file: Union[str, Sequence[str]], + override: Optional[Dict] = None, + ) -> ConfigParser: + """ + Read the config file, metadata file and override with specified `id=value` pairs. + Put metadata in the config content with key "". + The `id` in `override` identifies target position to override with the `value`. + If `value` starts with "", it will automatically read the `file` + and use the content as `value`. + + Args: + config_file: filepath of the config file, the config content must be a dictionary, + if providing a list of files, wil merge the content of them. + meta_file: filepath of the metadata file, the config content must be a dictionary, + if providing a list of files, wil merge the content of them. + override: dict of `{id: value}` pairs to override or add the config content. + + """ + config: Dict = {"": {}} + for f in ensure_tuple(config_file): + content = load_config_file(f) + if not isinstance(content, dict): + raise ValueError("input config content must be a dictionary.") + config.update(content) + + for f in ensure_tuple(meta_file): + content = load_config_file(f) + if not isinstance(content, dict): + raise ValueError("meta data content must be a dictionary.") + config[""].update(content) + + parser = ConfigParser(config=config) + + if override is not None: + for id, v in override.items(): + if isinstance(v, str) and v.startswith(""): + v = load_config_file_content(v[6:]) + parser[id] = v + return parser + + @abstractmethod + def __call__(self, *args: Any, **kwargs: Any): + """ + Execute task specific script. + + """ + raise NotImplementedError(f"Subclass {self.__class__.__name__} must implement this method.") + + +class Run(Script): + """ + Specify metadata file and config file to run a regular training or evaluation program. + It's used to execute most of the supervised training, evaluation or inference cases. + + Args: + meta_file: filepath of the metadata file, if None, must provide it in `arg_file`. + if providing a list of files, wil merge the content of them. + config_file: filepath of the config file, if None, must provide it in `arg_file`. + if providing a list of files, wil merge the content of them. + override: override above config content with specified `id` and `value` pairs. + it can also be used to provide default value for placeholders. for example: + put a placeholder `"data": "@runtime_value"` in the config, then define + `runtime_value` in `override`. + it also supports a string representing a dict, like: "{'AA#BB': 123}", usually from command line. + target: ID name of the target workflow, it must have the `run` method, follow MONAI `BaseWorkflow`. + if None, must provide it in `arg_file`. + args_file: to avoid providing same args every time running the program, it supports + to put the args as a dictionary in a JSON or YAML file. + kwargs: other args that can set in `args_file` and override at runtime. + + """ + def __init__( + self, + meta_file: Optional[Union[str, Sequence[str]]] = None, + config_file: Optional[Union[str, Sequence[str]]] = None, + override: Optional[Union[Dict, str]] = None, + target: Optional[str] = None, + args_file: Optional[str] = None, + ): + super().__init__( + meta_file=meta_file, config_file=config_file, override=override, args_file=args_file, target=target, + ) + + def __call__(self): + # get expected workflow to run + workflow = self.parser.get_parsed_content(id=self.args["target"]) + workflow.run() + + +if __name__ == "__main__": + fire.Fire() diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py deleted file mode 100644 index e8afa613ac..0000000000 --- a/monai/bundle/scripts.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) MONAI Consortium -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict, Optional, Sequence, Union - -from monai.bundle.utils import id_value_str_to_dict, parse_config_file, update_default_args -from monai.utils import optional_import - -fire, _ = optional_import("fire") - - -def run( - meta_file: Optional[Union[str, Sequence[str]]] = None, - config_file: Optional[Union[str, Sequence[str]]] = None, - override: Optional[Union[Dict, str]] = None, - target: Optional[str] = None, - args_file: Optional[str] = None, -): - """ - Specify metadata file and config file to run a regular training or evaluation program. - It's used to execute most of the supervised training, evaluation or inference cases. - - Typical usage examples: - - 1. Execute the `run` API with other CLI tools, take `fire` for example: - `python -m fire monai.bundle run --meta_file= --config_file= --target=trainer` - - 2. Execute this module as CLI entry based on `fire`: - `python -m monai.bundle.scripts run --meta_file= --config_file= --target=trainer` - - 3. Override some config values at runtime, set `override` as a dict: - `python -m monai.bundle.scripts run --override={"'net##ndims'": 2} ...` - - 4. Override some config values at runtime, set `override` as a string: - `python -m monai.bundle.scripts run --override="{net##ndims: 2}" ...` - - 5. Override some config values with another config file: - `python -m monai.bundle.scripts run --override={"'net#'": "'/data/other.json'"} ...` - - 6. Override some config values with part content of another config file: - `python -m monai.bundle.scripts run --override={"'net#'": "'/data/other.json#net_arg'"} ...` - - 7. Set default args of `run` in a JSON / YAML file, help to record and simplify the command line. - Other args still can override the default args at runtime: - `python -m monai.bundle.scripts run --args_file="'/data/args.json'" --config_file=` - - Args: - meta_file: filepath of the metadata file, if None, must provide it in `arg_file`. - if providing a list of files, wil merge the content of them. - config_file: filepath of the config file, if None, must provide it in `arg_file`. - if providing a list of files, wil merge the content of them. - override: override above config content with specified `id` and `value` pairs. - it can also be used to provide default value for placeholders. for example: - put a placeholder `"data": "@runtime_value"` in the config, then define - `runtime_value` in `override`. - it also supports a string representing a dict, like: "{'AA#BB': 123}", usually from command line. - target: ID name of the target workflow, it must have the `run` method, follow MONAI `BaseWorkflow`. - if None, must provide it in `arg_file`. - args_file: to avoid providing same args every time running the program, it supports - to put the args as a dictionary in a JSON or YAML file. - - """ - - if isinstance(override, str): - # if override is a string representing a dict (usually from command line), convert it to dict - override = id_value_str_to_dict(override) - args = update_default_args( - args=args_file, meta_file=meta_file, config_file=config_file, override=override, target=target - ) - - config_parser = parse_config_file( - config_file=args["config_file"], meta_file=args["meta_file"], override=args["override"] - ) - # get expected workflow to run - workflow = config_parser.get_parsed_content(id=args["target"]) - workflow.run() - - -if __name__ == "__main__": - fire.Fire() diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 1e8691e0c9..d51074647c 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -12,21 +12,21 @@ import json import re from distutils.util import strtobool -from typing import Any, Dict, Optional, Sequence, Tuple, Union +from typing import Any, Dict, Tuple, Union from monai.bundle.config_parser import ConfigParser -from monai.utils import ensure_tuple, optional_import +from monai.utils import optional_import yaml, _ = optional_import("yaml") def load_config_file(filepath: str, **kwargs): """ - Load config file with specified file path. + Load structured config file with the specified file path. Suppprt JSON and YAML formats. Args: - filepath: path of target file to load, supported postfixes: `.json`, `yml`, `yaml`. + filepath: path of target file to load, supported postfixes: `.json`, `.yml`, `.yaml`. kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. for more details, please check: https://docs.python.org/3/library/json.html#json.load. @@ -44,6 +44,7 @@ def load_config_file(filepath: str, **kwargs): def load_config_file_content(path: str, **kwargs): """ Load part of the content from a config file with specified `id` in the path. + If no `id` provided, load the whole content of the file. Suppprt JSON and YAML formats file. Args: @@ -66,33 +67,13 @@ def load_config_file_content(path: str, **kwargs): return parser[paths[1][1:] if paths[1] != "" else ""] -def update_default_args(args: Optional[Union[str, Dict]] = None, **kwargs) -> Dict: - """ - Update the `args` with the input `kwargs`. - For dict data, recursively update the content based on the keys. - - Args: - args: source args to update. - kwargs: destination args to update. - - """ - args_: Dict = args if isinstance(args, dict) is None else {} # type: ignore - if isinstance(args, str): - args_ = load_config_file_content(args) - - # recursively update the default args with new args - for k, v in kwargs.items(): - args_[k] = update_default_args(args_[k], **v) if isinstance(v, dict) and isinstance(args_.get(k), dict) else v - return args_ - - def parse_id_value(pair: str) -> Tuple[str, Any]: """ Parse the "id:value" pair string to `id` and `value`. Will try to convert the correct data type of `value` from string. Args: - pair (str): input "id:value" pair to parse. + pair: input "id:value" pair to parse. """ items = pair.split(":") @@ -121,49 +102,7 @@ def id_value_str_to_dict(pairs: str) -> Dict[str, Any]: """ Utility to convert a string which represents a dict of `id:value` pairs to a python dict. For example: `"{postprocessing##postfix: output, network: other.json#net_args}"` - Will try to convert the correct data type of `value` from string. - - """ - return dict(map(parse_id_value, pairs[1:-1].split(","))) - - -def parse_config_file( - config_file: Union[str, Sequence[str]], meta_file: Union[str, Sequence[str]], override: Optional[Dict] = None -) -> ConfigParser: - """ - Read the config file, metadata file and override with specified `id=value` pairs. - Put metadata in the config content with key "". - The `id` in `override` identifies target position to override with the `value`. - If `value` starts with "", it will automatically read the `file` - and use the content as `value`. - - Args: - config_file: filepath of the config file, the config content must be a dictionary, - if providing a list of files, wil merge the content of them. - meta_file: filepath of the metadata file, the config content must be a dictionary, - if providing a list of files, wil merge the content of them. - override: dict of `{id: value}` pairs to override or add the config content. + Will try to convert the data type of `value` from string to real type. """ - config: Dict = {"": {}} - for f in ensure_tuple(config_file): - content = load_config_file(f) - if not isinstance(content, dict): - raise ValueError("input config content must be a dictionary.") - config.update(content) - - for f in ensure_tuple(meta_file): - content = load_config_file(f) - if not isinstance(content, dict): - raise ValueError("meta data content must be a dictionary.") - config[""].update(content) - - parser = ConfigParser(config=config) - - if override is not None: - for id, v in override.items(): - if isinstance(v, str) and v.startswith(""): - v = load_config_file_content(v[6:]) - parser[id] = v - - return parser + return dict(map(parse_id_value, pairs[1: -1].split(","))) diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 1a7868168f..89daf6b876 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -135,8 +135,8 @@ def test_shape(self, config, expected_shape): [ f"{sys.executable}", "-m", - "monai.bundle.scripts", - "run", + "monai.bundle.script", + "Run", "--meta_file", f"{meta_file}", "--config_file", @@ -160,7 +160,7 @@ def test_shape(self, config, expected_shape): "-m", "fire", "monai.bundle", - "run", + "Run", "--meta_file", f"{meta_file}", "--config_file", From 9bbd57e1b35e296d3fe8514189c0eabe5a0ea4e0 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 7 Mar 2022 12:14:03 +0800 Subject: [PATCH 42/76] [DLMED] restore design Signed-off-by: Nic Ma --- monai/bundle/__init__.py | 4 +- monai/bundle/script.py | 197 --------------------------------------- monai/bundle/scripts.py | 88 +++++++++++++++++ monai/bundle/utils.py | 75 +++++++++++++-- tests/test_bundle_run.py | 6 +- 5 files changed, 162 insertions(+), 208 deletions(-) delete mode 100644 monai/bundle/script.py create mode 100644 monai/bundle/scripts.py diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py index c9e4ff43f1..adc7fac20b 100644 --- a/monai/bundle/__init__.py +++ b/monai/bundle/__init__.py @@ -12,10 +12,12 @@ from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, Instantiable from .config_parser import ConfigParser from .reference_resolver import ReferenceResolver -from .script import Script, Run +from .scripts import run from .utils import ( id_value_str_to_dict, load_config_file, load_config_file_content, + parse_config_file, parse_id_value, + update_default_args, ) diff --git a/monai/bundle/script.py b/monai/bundle/script.py deleted file mode 100644 index ad7a0f456d..0000000000 --- a/monai/bundle/script.py +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright (c) MONAI Consortium -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from abc import ABC, abstractmethod -from typing import Any, Dict, Optional, Sequence, Union - -from monai.bundle.config_parser import ConfigParser -from monai.bundle.utils import id_value_str_to_dict, load_config_file, load_config_file_content -from monai.utils import ensure_tuple, optional_import - -fire, _ = optional_import("fire") - - -class Script(ABC): - """ - Base class for typical config based scripts in the bundle. - Typical usage examples: - - 1. Execute the `run` API with other CLI tools, take `fire` for example: - `python -m fire monai.bundle run --meta_file= --config_file= --target=trainer` - - 2. Execute this module as CLI entry based on `fire`: - `python -m monai.bundle.scripts run --meta_file= --config_file= --target=trainer` - - 3. Override some config values at runtime, set `override` as a dict: - `python -m monai.bundle.scripts run --override={"'net##ndims'": 2} ...` - - 4. Override some config values at runtime, set `override` as a string: - `python -m monai.bundle.scripts run --override="{net##ndims: 2}" ...` - - 5. Override some config values with another config file: - `python -m monai.bundle.scripts run --override={"'net#'": "'/data/other.json'"} ...` - - 6. Override some config values with part content of another config file: - `python -m monai.bundle.scripts run --override={"'net#'": "'/data/other.json#net_arg'"} ...` - - 7. Set default args of `run` in a JSON / YAML file, help to record and simplify the command line. - Other args still can override the default args at runtime: - `python -m monai.bundle.scripts run --args_file="'/data/args.json'" --config_file=` - - Args: - meta_file: filepath of the metadata file, if None, must provide it in `arg_file`. - if providing a list of files, wil merge the content of them. - config_file: filepath of the config file, if None, must provide it in `arg_file`. - if providing a list of files, wil merge the content of them. - override: override above config content with specified `id` and `value` pairs. - it can also be used to provide default value for placeholders. for example: - put a placeholder `"data": "@runtime_value"` in the config, then define - `runtime_value` in `override`. - it also supports a string representing a dict, like: "{'AA#BB': 123}", usually from command line. - args_file: to avoid providing same args every time running the program, it supports - to put the args as a dictionary in a JSON or YAML file. - kwargs: other args that can set in `args_file` and override at runtime. - - """ - def __init__( - self, - meta_file: Optional[Union[str, Sequence[str]]] = None, - config_file: Optional[Union[str, Sequence[str]]] = None, - override: Optional[Union[Dict, str]] = None, - args_file: Optional[str] = None, - **kwargs, - ): - if isinstance(override, str): - # if override is a string representing a dict (usually from command line), convert it to dict - override = id_value_str_to_dict(override) - self.args = self._update_default_args( - args=args_file, meta_file=meta_file, config_file=config_file, override=override, **kwargs, - ) - self.parser = self.parse_config_file( - config_file=self.args["config_file"], meta_file=self.args["meta_file"], override=self.args["override"] - ) - - def _update_default_args(self, args: Optional[Union[str, Dict]] = None, **kwargs) -> Dict: - """ - Update the `args` with the input `kwargs`. - For dict data, recursively update the content based on the keys. - - Args: - args: source args to update. - kwargs: destination args to update. - - """ - args_: Dict = args if isinstance(args, dict) is None else {} # type: ignore - if isinstance(args, str): - args_ = load_config_file_content(args) - - # recursively update the default args with new args - for k, v in kwargs.items(): - if isinstance(v, dict) and isinstance(args_.get(k), dict): - args_[k] = self._update_default_args(args_[k], **v) - else: - args_[k] = v - return args_ - - def parse_config_file( - self, - config_file: Union[str, Sequence[str]], - meta_file: Union[str, Sequence[str]], - override: Optional[Dict] = None, - ) -> ConfigParser: - """ - Read the config file, metadata file and override with specified `id=value` pairs. - Put metadata in the config content with key "". - The `id` in `override` identifies target position to override with the `value`. - If `value` starts with "", it will automatically read the `file` - and use the content as `value`. - - Args: - config_file: filepath of the config file, the config content must be a dictionary, - if providing a list of files, wil merge the content of them. - meta_file: filepath of the metadata file, the config content must be a dictionary, - if providing a list of files, wil merge the content of them. - override: dict of `{id: value}` pairs to override or add the config content. - - """ - config: Dict = {"": {}} - for f in ensure_tuple(config_file): - content = load_config_file(f) - if not isinstance(content, dict): - raise ValueError("input config content must be a dictionary.") - config.update(content) - - for f in ensure_tuple(meta_file): - content = load_config_file(f) - if not isinstance(content, dict): - raise ValueError("meta data content must be a dictionary.") - config[""].update(content) - - parser = ConfigParser(config=config) - - if override is not None: - for id, v in override.items(): - if isinstance(v, str) and v.startswith(""): - v = load_config_file_content(v[6:]) - parser[id] = v - return parser - - @abstractmethod - def __call__(self, *args: Any, **kwargs: Any): - """ - Execute task specific script. - - """ - raise NotImplementedError(f"Subclass {self.__class__.__name__} must implement this method.") - - -class Run(Script): - """ - Specify metadata file and config file to run a regular training or evaluation program. - It's used to execute most of the supervised training, evaluation or inference cases. - - Args: - meta_file: filepath of the metadata file, if None, must provide it in `arg_file`. - if providing a list of files, wil merge the content of them. - config_file: filepath of the config file, if None, must provide it in `arg_file`. - if providing a list of files, wil merge the content of them. - override: override above config content with specified `id` and `value` pairs. - it can also be used to provide default value for placeholders. for example: - put a placeholder `"data": "@runtime_value"` in the config, then define - `runtime_value` in `override`. - it also supports a string representing a dict, like: "{'AA#BB': 123}", usually from command line. - target: ID name of the target workflow, it must have the `run` method, follow MONAI `BaseWorkflow`. - if None, must provide it in `arg_file`. - args_file: to avoid providing same args every time running the program, it supports - to put the args as a dictionary in a JSON or YAML file. - kwargs: other args that can set in `args_file` and override at runtime. - - """ - def __init__( - self, - meta_file: Optional[Union[str, Sequence[str]]] = None, - config_file: Optional[Union[str, Sequence[str]]] = None, - override: Optional[Union[Dict, str]] = None, - target: Optional[str] = None, - args_file: Optional[str] = None, - ): - super().__init__( - meta_file=meta_file, config_file=config_file, override=override, args_file=args_file, target=target, - ) - - def __call__(self): - # get expected workflow to run - workflow = self.parser.get_parsed_content(id=self.args["target"]) - workflow.run() - - -if __name__ == "__main__": - fire.Fire() diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py new file mode 100644 index 0000000000..e8afa613ac --- /dev/null +++ b/monai/bundle/scripts.py @@ -0,0 +1,88 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional, Sequence, Union + +from monai.bundle.utils import id_value_str_to_dict, parse_config_file, update_default_args +from monai.utils import optional_import + +fire, _ = optional_import("fire") + + +def run( + meta_file: Optional[Union[str, Sequence[str]]] = None, + config_file: Optional[Union[str, Sequence[str]]] = None, + override: Optional[Union[Dict, str]] = None, + target: Optional[str] = None, + args_file: Optional[str] = None, +): + """ + Specify metadata file and config file to run a regular training or evaluation program. + It's used to execute most of the supervised training, evaluation or inference cases. + + Typical usage examples: + + 1. Execute the `run` API with other CLI tools, take `fire` for example: + `python -m fire monai.bundle run --meta_file= --config_file= --target=trainer` + + 2. Execute this module as CLI entry based on `fire`: + `python -m monai.bundle.scripts run --meta_file= --config_file= --target=trainer` + + 3. Override some config values at runtime, set `override` as a dict: + `python -m monai.bundle.scripts run --override={"'net##ndims'": 2} ...` + + 4. Override some config values at runtime, set `override` as a string: + `python -m monai.bundle.scripts run --override="{net##ndims: 2}" ...` + + 5. Override some config values with another config file: + `python -m monai.bundle.scripts run --override={"'net#'": "'/data/other.json'"} ...` + + 6. Override some config values with part content of another config file: + `python -m monai.bundle.scripts run --override={"'net#'": "'/data/other.json#net_arg'"} ...` + + 7. Set default args of `run` in a JSON / YAML file, help to record and simplify the command line. + Other args still can override the default args at runtime: + `python -m monai.bundle.scripts run --args_file="'/data/args.json'" --config_file=` + + Args: + meta_file: filepath of the metadata file, if None, must provide it in `arg_file`. + if providing a list of files, wil merge the content of them. + config_file: filepath of the config file, if None, must provide it in `arg_file`. + if providing a list of files, wil merge the content of them. + override: override above config content with specified `id` and `value` pairs. + it can also be used to provide default value for placeholders. for example: + put a placeholder `"data": "@runtime_value"` in the config, then define + `runtime_value` in `override`. + it also supports a string representing a dict, like: "{'AA#BB': 123}", usually from command line. + target: ID name of the target workflow, it must have the `run` method, follow MONAI `BaseWorkflow`. + if None, must provide it in `arg_file`. + args_file: to avoid providing same args every time running the program, it supports + to put the args as a dictionary in a JSON or YAML file. + + """ + + if isinstance(override, str): + # if override is a string representing a dict (usually from command line), convert it to dict + override = id_value_str_to_dict(override) + args = update_default_args( + args=args_file, meta_file=meta_file, config_file=config_file, override=override, target=target + ) + + config_parser = parse_config_file( + config_file=args["config_file"], meta_file=args["meta_file"], override=args["override"] + ) + # get expected workflow to run + workflow = config_parser.get_parsed_content(id=args["target"]) + workflow.run() + + +if __name__ == "__main__": + fire.Fire() diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index d51074647c..6a75d215b6 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -12,17 +12,17 @@ import json import re from distutils.util import strtobool -from typing import Any, Dict, Tuple, Union +from typing import Any, Dict, Optional, Sequence, Tuple, Union from monai.bundle.config_parser import ConfigParser -from monai.utils import optional_import +from monai.utils import ensure_tuple, optional_import yaml, _ = optional_import("yaml") def load_config_file(filepath: str, **kwargs): """ - Load structured config file with the specified file path. + Load config file with specified file path. Suppprt JSON and YAML formats. Args: @@ -44,7 +44,6 @@ def load_config_file(filepath: str, **kwargs): def load_config_file_content(path: str, **kwargs): """ Load part of the content from a config file with specified `id` in the path. - If no `id` provided, load the whole content of the file. Suppprt JSON and YAML formats file. Args: @@ -67,10 +66,30 @@ def load_config_file_content(path: str, **kwargs): return parser[paths[1][1:] if paths[1] != "" else ""] +def update_default_args(args: Optional[Union[str, Dict]] = None, **kwargs) -> Dict: + """ + Update the `args` with the input `kwargs`. + For dict data, recursively update the content based on the keys. + + Args: + args: source args to update. + kwargs: destination args to update. + + """ + args_: Dict = args if isinstance(args, dict) is None else {} # type: ignore + if isinstance(args, str): + args_ = load_config_file_content(args) + + # recursively update the default args with new args + for k, v in kwargs.items(): + args_[k] = update_default_args(args_[k], **v) if isinstance(v, dict) and isinstance(args_.get(k), dict) else v + return args_ + + def parse_id_value(pair: str) -> Tuple[str, Any]: """ Parse the "id:value" pair string to `id` and `value`. - Will try to convert the correct data type of `value` from string. + Will try to convert the data type of `value` from string to the correct type. Args: pair: input "id:value" pair to parse. @@ -102,7 +121,49 @@ def id_value_str_to_dict(pairs: str) -> Dict[str, Any]: """ Utility to convert a string which represents a dict of `id:value` pairs to a python dict. For example: `"{postprocessing##postfix: output, network: other.json#net_args}"` - Will try to convert the data type of `value` from string to real type. + Will try to convert the correct data type of `value` from string. + + """ + return dict(map(parse_id_value, pairs[1:-1].split(","))) + + +def parse_config_file( + config_file: Union[str, Sequence[str]], meta_file: Union[str, Sequence[str]], override: Optional[Dict] = None +) -> ConfigParser: + """ + Read the config file, metadata file and override with specified `id=value` pairs. + Put metadata in the config content with key "". + The `id` in `override` identifies target position to override with the `value`. + If `value` starts with "", it will automatically read the `file` + and use the content as `value`. + + Args: + config_file: filepath of the config file, the config content must be a dictionary, + if providing a list of files, wil merge the content of them. + meta_file: filepath of the metadata file, the config content must be a dictionary, + if providing a list of files, wil merge the content of them. + override: dict of `{id: value}` pairs to override or add the config content. """ - return dict(map(parse_id_value, pairs[1: -1].split(","))) + config: Dict = {"": {}} + for f in ensure_tuple(config_file): + content = load_config_file(f) + if not isinstance(content, dict): + raise ValueError("input config content must be a dictionary.") + config.update(content) + + for f in ensure_tuple(meta_file): + content = load_config_file(f) + if not isinstance(content, dict): + raise ValueError("meta data content must be a dictionary.") + config[""].update(content) + + parser = ConfigParser(config=config) + + if override is not None: + for id, v in override.items(): + if isinstance(v, str) and v.startswith(""): + v = load_config_file_content(v[6:]) + parser[id] = v + + return parser diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 89daf6b876..1a7868168f 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -135,8 +135,8 @@ def test_shape(self, config, expected_shape): [ f"{sys.executable}", "-m", - "monai.bundle.script", - "Run", + "monai.bundle.scripts", + "run", "--meta_file", f"{meta_file}", "--config_file", @@ -160,7 +160,7 @@ def test_shape(self, config, expected_shape): "-m", "fire", "monai.bundle", - "Run", + "run", "--meta_file", f"{meta_file}", "--config_file", From 1810d0dcf5b468b779bc5782df8cfc93cb57cd24 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 7 Mar 2022 22:30:15 +0800 Subject: [PATCH 43/76] [DLMED] fix args override Signed-off-by: Nic Ma --- monai/bundle/scripts.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index e8afa613ac..405b8d5cc2 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -72,12 +72,16 @@ def run( if isinstance(override, str): # if override is a string representing a dict (usually from command line), convert it to dict override = id_value_str_to_dict(override) - args = update_default_args( - args=args_file, meta_file=meta_file, config_file=config_file, override=override, target=target - ) + + kwargs = {} + for k, v in {"meta_file": meta_file, "config_file": config_file, "override": override, "target": target}.items(): + if v is not None: + # skip None args + kwargs[k] = v + args = update_default_args(args=args_file, **kwargs) config_parser = parse_config_file( - config_file=args["config_file"], meta_file=args["meta_file"], override=args["override"] + config_file=args["config_file"], meta_file=args["meta_file"], override=args.get("override") ) # get expected workflow to run workflow = config_parser.get_parsed_content(id=args["target"]) From a1f86994c6c2953d5d7121762c7d66fc0b4f3014 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Mon, 7 Mar 2022 23:18:25 +0800 Subject: [PATCH 44/76] [DLMED] fix typo Signed-off-by: Nic Ma --- monai/bundle/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 6a75d215b6..3534bc9cd5 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -76,7 +76,7 @@ def update_default_args(args: Optional[Union[str, Dict]] = None, **kwargs) -> Di kwargs: destination args to update. """ - args_: Dict = args if isinstance(args, dict) is None else {} # type: ignore + args_: Dict = args if isinstance(args, dict) else {} # type: ignore if isinstance(args, str): args_ = load_config_file_content(args) From 7adf4a945d4ccafcefb11d9397a9fb3c4aec500c Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 8 Mar 2022 00:00:07 +0800 Subject: [PATCH 45/76] [DLMED] test windows Signed-off-by: Nic Ma --- monai/bundle/scripts.py | 6 +---- tests/test_bundle_run.py | 49 ++++++++++++++++++++-------------------- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index 405b8d5cc2..e7d55b55df 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -11,7 +11,7 @@ from typing import Dict, Optional, Sequence, Union -from monai.bundle.utils import id_value_str_to_dict, parse_config_file, update_default_args +from monai.bundle.utils import parse_config_file, update_default_args from monai.utils import optional_import fire, _ = optional_import("fire") @@ -69,10 +69,6 @@ def run( """ - if isinstance(override, str): - # if override is a string representing a dict (usually from command line), convert it to dict - override = id_value_str_to_dict(override) - kwargs = {} for k, v in {"meta_file": meta_file, "config_file": config_file, "override": override, "target": target}.items(): if v is not None: diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 1a7868168f..83a04b758c 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -143,8 +143,8 @@ def test_shape(self, config, expected_shape): f"{config_file}", # `fire` can not parse below string, will pass to `run` as a string representing a dict "--override", - f"{{postprocessing##transforms#2##output_postfix: seg, \ - network: {overridefile1}#move_net, dataset#: {overridefile2}}}", + f"{{'postprocessing##transforms#2##output_postfix':'seg',\ + 'network':'{overridefile1}#move_net','dataset#':'{overridefile2}'}}", "--target", "evaluator", ] @@ -152,29 +152,28 @@ def test_shape(self, config, expected_shape): self.assertEqual(ret, 0) self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_seg.nii.gz")).shape, expected_shape) - if sys.platform != "win32": - # here test the script with `google fire` tool as CLI - ret = subprocess.check_call( - [ - f"{sys.executable}", - "-m", - "fire", - "monai.bundle", - "run", - "--meta_file", - f"{meta_file}", - "--config_file", - f"{config_file}", - "--override", - # `fire` can parse below string as a dictionary - f"{{'evaluator##amp':False,'network':'{overridefile1}#move_net', \ - 'dataset#':'{overridefile2}'}}", - "--target", - "evaluator", - ] - ) - self.assertEqual(ret, 0) - self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_trans.nii.gz")).shape, expected_shape) + # here test the script with `google fire` tool as CLI + ret = subprocess.check_call( + [ + f"{sys.executable}", + "-m", + "fire", + "monai.bundle", + "run", + "--meta_file", + f"{meta_file}", + "--config_file", + f"{config_file}", + "--override", + # `fire` can parse below string as a dictionary + f"{{'evaluator##amp':False,'network':'{overridefile1}#move_net',\ + 'dataset#':'{overridefile2}'}}", + "--target", + "evaluator", + ] + ) + self.assertEqual(ret, 0) + self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_trans.nii.gz")).shape, expected_shape) if __name__ == "__main__": From 4469f3858aba11ac3461d843c415cc61856a102f Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 8 Mar 2022 09:06:58 +0800 Subject: [PATCH 46/76] [DLMED] test windows Signed-off-by: Nic Ma --- tests/test_bundle_run.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 83a04b758c..f09c2abbb1 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -130,6 +130,10 @@ def test_shape(self, config, expected_shape): saver = LoadImage(image_only=True) + if sys.platform == "win32": + override = "'network':'$@network_def.to(@device)','dataset#':'Dataset'" + else: + override = f"'network':'{overridefile1}#move_net','dataset#':'{overridefile2}'" # test with `monai.bundle.scripts` as CLI entry directly ret = subprocess.check_call( [ @@ -143,8 +147,7 @@ def test_shape(self, config, expected_shape): f"{config_file}", # `fire` can not parse below string, will pass to `run` as a string representing a dict "--override", - f"{{'postprocessing##transforms#2##output_postfix':'seg',\ - 'network':'{overridefile1}#move_net','dataset#':'{overridefile2}'}}", + f"{{'postprocessing##transforms#2##output_postfix':'seg',{override}}}", "--target", "evaluator", ] @@ -166,8 +169,7 @@ def test_shape(self, config, expected_shape): f"{config_file}", "--override", # `fire` can parse below string as a dictionary - f"{{'evaluator##amp':False,'network':'{overridefile1}#move_net',\ - 'dataset#':'{overridefile2}'}}", + f"{{'evaluator##amp':False,{override}}}", "--target", "evaluator", ] From c1aeaf23af83796c2804879b679e8c97790fa779 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 8 Mar 2022 14:24:35 +0800 Subject: [PATCH 47/76] [DLMED] update according to comments Signed-off-by: Nic Ma --- docs/source/bundle.rst | 4 - monai/bundle/__init__.py | 9 +-- monai/bundle/utils.py | 44 +---------- tests/test_bundle_run.py | 83 ++------------------ tests/testing_data/inference.json | 126 ++++++++++++++++++++++++++++++ tests/testing_data/inference.yaml | 85 ++++++++++++++++++++ 6 files changed, 219 insertions(+), 132 deletions(-) create mode 100644 tests/testing_data/inference.json create mode 100644 tests/testing_data/inference.yaml diff --git a/docs/source/bundle.rst b/docs/source/bundle.rst index 1f34c9ea21..2c4372b3b4 100644 --- a/docs/source/bundle.rst +++ b/docs/source/bundle.rst @@ -45,8 +45,4 @@ Model Bundle .. autofunction:: update_default_args -.. autofunction:: parse_id_value - -.. autofunction:: id_value_str_to_dict - .. autofunction:: parse_config_file diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py index adc7fac20b..45c07a3b82 100644 --- a/monai/bundle/__init__.py +++ b/monai/bundle/__init__.py @@ -13,11 +13,4 @@ from .config_parser import ConfigParser from .reference_resolver import ReferenceResolver from .scripts import run -from .utils import ( - id_value_str_to_dict, - load_config_file, - load_config_file_content, - parse_config_file, - parse_id_value, - update_default_args, -) +from .utils import load_config_file, load_config_file_content, parse_config_file, update_default_args diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 3534bc9cd5..0a61e4e4a0 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -11,8 +11,7 @@ import json import re -from distutils.util import strtobool -from typing import Any, Dict, Optional, Sequence, Tuple, Union +from typing import Dict, Optional, Sequence, Union from monai.bundle.config_parser import ConfigParser from monai.utils import ensure_tuple, optional_import @@ -86,47 +85,6 @@ def update_default_args(args: Optional[Union[str, Dict]] = None, **kwargs) -> Di return args_ -def parse_id_value(pair: str) -> Tuple[str, Any]: - """ - Parse the "id:value" pair string to `id` and `value`. - Will try to convert the data type of `value` from string to the correct type. - - Args: - pair: input "id:value" pair to parse. - - """ - items = pair.split(":") - # remove blanks around id - id = items[0].strip() - value: Union[str, int, float, bool] = "" - if len(items) > 1: - # rejoin the rest, and remove blanks around value - value = ":".join(items[1:]).strip() - - # try to convert the correct data type - try: - value = int(value) - except ValueError: - try: - value = float(value) - except ValueError: - try: - value = bool(strtobool(str(value))) - except ValueError: - pass - return id, value - - -def id_value_str_to_dict(pairs: str) -> Dict[str, Any]: - """ - Utility to convert a string which represents a dict of `id:value` pairs to a python dict. For example: - `"{postprocessing##postfix: output, network: other.json#net_args}"` - Will try to convert the correct data type of `value` from string. - - """ - return dict(map(parse_id_value, pairs[1:-1].split(","))) - - def parse_config_file( config_file: Union[str, Sequence[str]], meta_file: Union[str, Sequence[str]], override: Optional[Dict] = None ) -> ConfigParser: diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index f09c2abbb1..2b0e796a45 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -23,81 +23,14 @@ from monai.transforms import LoadImage -TEST_CASE_1 = [ - { - "device": "$torch.device('cuda' if torch.cuda.is_available() else 'cpu')", - "network_def": { - "": "UNet", - "": { - "spatial_dims": 3, - "in_channels": 1, - "out_channels": 2, - "channels": [16, 32, 64, 128, 256], - "strides": [2, 2, 2, 2], - "num_res_units": 2, - "norm": "batch", - }, - }, - "network": "will be overrided", - "preprocessing": { - "": "Compose", - "": { - "transforms": [ - {"": "LoadImaged", "": {"keys": "image"}}, - {"": "EnsureChannelFirstd", "": {"keys": "image"}}, - {"": "ScaleIntensityd", "": {"keys": "image"}}, - {"": "EnsureTyped", "": {"keys": "image"}}, - ] - }, - }, - "dataset": { - "": "will be overrided", - "": {"data": "@#datalist", "transform": "@preprocessing"}, # test placeholger with `datalist` - }, - "dataloader": { - "": "DataLoader", - "": {"dataset": "@dataset", "batch_size": 1, "shuffle": False, "num_workers": 4}, - }, - "inferer": { - "": "SlidingWindowInferer", - "": {"roi_size": [96, 96, 96], "sw_batch_size": 4, "overlap": 0.5}, - }, - "postprocessing": { - "": "Compose", - "": { - "transforms": [ - {"": "Activationsd", "": {"keys": "pred", "softmax": True}}, - {"": "AsDiscreted", "": {"keys": "pred", "argmax": True}}, - { - "": "SaveImaged", - "": { - "keys": "pred", - "meta_keys": "image_meta_dict", - "output_dir": "@#output_dir", # test placeholger with `output_dir` - }, - }, - ] - }, - }, - "evaluator": { - "": "SupervisedEvaluator", - "": { - "device": "@device", - "val_data_loader": "@dataloader", - "network": "@network", - "inferer": "@inferer", - "postprocessing": "@postprocessing", - "amp": False, - }, - }, - }, - (128, 128, 128), -] +TEST_CASE_1 = [os.path.join(os.path.dirname(__file__), "testing_data", "inference.json"), (128, 128, 128)] + +TEST_CASE_2 = [os.path.join(os.path.dirname(__file__), "testing_data", "inference.yaml"), (128, 128, 128)] class TestBundleRun(unittest.TestCase): - @parameterized.expand([TEST_CASE_1]) - def test_shape(self, config, expected_shape): + @parameterized.expand([TEST_CASE_1, TEST_CASE_2]) + def test_shape(self, config_file, expected_shape): test_image = np.random.rand(128, 128, 128) with tempfile.TemporaryDirectory() as tempdir: filename = os.path.join(tempdir, "image.nii") @@ -113,10 +46,6 @@ def test_shape(self, config, expected_shape): meta_file = os.path.join(tempdir, "meta.yaml") with open(meta_file, "w") as f: yaml.safe_dump(meta, f) - # test JSON file - config_file = os.path.join(tempdir, "config.json") - with open(config_file, "w") as f: - json.dump(config, f) # test override with file, up case postfix overridefile1 = os.path.join(tempdir, "override1.JSON") @@ -134,7 +63,7 @@ def test_shape(self, config, expected_shape): override = "'network':'$@network_def.to(@device)','dataset#':'Dataset'" else: override = f"'network':'{overridefile1}#move_net','dataset#':'{overridefile2}'" - # test with `monai.bundle.scripts` as CLI entry directly + # test with `monai.bundle` as CLI entry directly ret = subprocess.check_call( [ f"{sys.executable}", diff --git a/tests/testing_data/inference.json b/tests/testing_data/inference.json new file mode 100644 index 0000000000..0a1cc0277e --- /dev/null +++ b/tests/testing_data/inference.json @@ -0,0 +1,126 @@ +{ + "device": "$torch.device('cuda' if torch.cuda.is_available() else 'cpu')", + "network_def": { + "": "UNet", + "": { + "spatial_dims": 3, + "in_channels": 1, + "out_channels": 2, + "channels": [ + 16, + 32, + 64, + 128, + 256 + ], + "strides": [ + 2, + 2, + 2, + 2 + ], + "num_res_units": 2, + "norm": "batch" + } + }, + "network": "need override", + "preprocessing": { + "": "Compose", + "": { + "transforms": [ + { + "": "LoadImaged", + "": { + "keys": "image" + } + }, + { + "": "EnsureChannelFirstd", + "": { + "keys": "image" + } + }, + { + "": "ScaleIntensityd", + "": { + "keys": "image" + } + }, + { + "": "EnsureTyped", + "": { + "keys": "image" + } + } + ] + } + }, + "dataset": { + "": "need override", + "": { + "data": "@#datalist", + "transform": "@preprocessing" + } + }, + "dataloader": { + "": "DataLoader", + "": { + "dataset": "@dataset", + "batch_size": 1, + "shuffle": false, + "num_workers": 4 + } + }, + "inferer": { + "": "SlidingWindowInferer", + "": { + "roi_size": [ + 96, + 96, + 96 + ], + "sw_batch_size": 4, + "overlap": 0.5 + } + }, + "postprocessing": { + "": "Compose", + "": { + "transforms": [ + { + "": "Activationsd", + "": { + "keys": "pred", + "softmax": true + } + }, + { + "": "AsDiscreted", + "": { + "keys": "pred", + "argmax": true + } + }, + { + "": "SaveImaged", + "": { + "keys": "pred", + "meta_keys": "image_meta_dict", + "output_dir": "@#output_dir" + } + } + ] + } + }, + "evaluator": { + "": "SupervisedEvaluator", + "": { + "device": "@device", + "val_data_loader": "@dataloader", + "network": "@network", + "inferer": "@inferer", + "postprocessing": "@postprocessing", + "amp": false + } + } +} diff --git a/tests/testing_data/inference.yaml b/tests/testing_data/inference.yaml new file mode 100644 index 0000000000..1a2f208a76 --- /dev/null +++ b/tests/testing_data/inference.yaml @@ -0,0 +1,85 @@ +--- +device: "$torch.device('cuda' if torch.cuda.is_available() else 'cpu')" +network_def: + "": UNet + "": + spatial_dims: 3 + in_channels: 1 + out_channels: 2 + channels: + - 16 + - 32 + - 64 + - 128 + - 256 + strides: + - 2 + - 2 + - 2 + - 2 + num_res_units: 2 + norm: batch +network: need override +preprocessing: + "": Compose + "": + transforms: + - "": LoadImaged + "": + keys: image + - "": EnsureChannelFirstd + "": + keys: image + - "": ScaleIntensityd + "": + keys: image + - "": EnsureTyped + "": + keys: image +dataset: + "": need override + "": + data: "@#datalist" + transform: "@preprocessing" +dataloader: + "": DataLoader + "": + dataset: "@dataset" + batch_size: 1 + shuffle: false + num_workers: 4 +inferer: + "": SlidingWindowInferer + "": + roi_size: + - 96 + - 96 + - 96 + sw_batch_size: 4 + overlap: 0.5 +postprocessing: + "": Compose + "": + transforms: + - "": Activationsd + "": + keys: pred + softmax: true + - "": AsDiscreted + "": + keys: pred + argmax: true + - "": SaveImaged + "": + keys: pred + meta_keys: image_meta_dict + output_dir: "@#output_dir" +evaluator: + "": SupervisedEvaluator + "": + device: "@device" + val_data_loader: "@dataloader" + network: "@network" + inferer: "@inferer" + postprocessing: "@postprocessing" + amp: false From bed62ba9c40313e482c952fd187c0ac59ec4e573 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 8 Mar 2022 19:44:39 +0800 Subject: [PATCH 48/76] [DLMED] refactor config reading logic Signed-off-by: Nic Ma --- docs/source/bundle.rst | 11 ++-- monai/bundle/__init__.py | 3 +- monai/bundle/config_reader.py | 116 ++++++++++++++++++++++++++++++++++ monai/bundle/scripts.py | 20 ++++-- monai/bundle/utils.py | 101 ++--------------------------- tests/test_bundle_run.py | 2 +- 6 files changed, 143 insertions(+), 110 deletions(-) create mode 100644 monai/bundle/config_reader.py diff --git a/docs/source/bundle.rst b/docs/source/bundle.rst index 2c4372b3b4..ed6321ff37 100644 --- a/docs/source/bundle.rst +++ b/docs/source/bundle.rst @@ -33,16 +33,15 @@ Model Bundle .. autoclass:: ConfigParser :members: +`Config Reader` +--------------- +.. autoclass:: ConfigReader + :members: + `Scripts` --------- .. autofunction:: run `Utilities` ----------- -.. autofunction:: load_config_file - -.. autofunction:: load_config_file_content - .. autofunction:: update_default_args - -.. autofunction:: parse_config_file diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py index 45c07a3b82..5236e6b788 100644 --- a/monai/bundle/__init__.py +++ b/monai/bundle/__init__.py @@ -11,6 +11,7 @@ from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, Instantiable from .config_parser import ConfigParser +from .config_reader import ConfigReader from .reference_resolver import ReferenceResolver from .scripts import run -from .utils import load_config_file, load_config_file_content, parse_config_file, update_default_args +from .utils import update_default_args diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py new file mode 100644 index 0000000000..b6b7a1af2d --- /dev/null +++ b/monai/bundle/config_reader.py @@ -0,0 +1,116 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from pathlib import Path +import re +from copy import deepcopy +from typing import Any, Dict, Optional, Sequence, Tuple, Union + +from monai.config import PathLike +from monai.bundle.config_parser import ConfigParser +from monai.utils import ensure_tuple, optional_import + +yaml, _ = optional_import("yaml") + + +class ConfigReader: + suffixes = ["json", "yaml", "yml"] + macro = "%" # macro prefix + meta_key = "" # field key to save meta data + + def __init__(self): + self.config: Dict = {self.meta_key: {}} + + @classmethod + def load_config_file(cls, filepath: PathLike, **kwargs): + """ + Load config file with specified file path. + Suppprt JSON and YAML formats. + + Args: + filepath: path of target file to load, supported postfixes: `.json`, `.yml`, `.yaml`. + kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. + for more details, please check: + https://docs.python.org/3/library/json.html#json.load. + https://pyyaml.org/wiki/PyYAMLDocumentation. + + """ + _filepath: str = str(Path(filepath)) + with open(_filepath) as f: + if _filepath.lower().endswith(cls.suffixes[0]): + return json.load(f, **kwargs) + if _filepath.lower().endswith(tuple(cls.suffixes[1:])): + return yaml.safe_load(f, **kwargs) + raise ValueError("only support JSON or YAML config file so far.") + + def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): + content = {} + if isinstance(f, dict): + # already loaded in dict + content = f + else: + for i in ensure_tuple(f): + content.update(self.load_config_file(i, **kwargs)) + self.config[self.meta_key] = content + + def read_config(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): + content = {self.meta_key: self.config[self.meta_key]} + if isinstance(f, dict): + # already loaded in dict + content.update(f) + else: + for i in ensure_tuple(f): + content.update(self.load_config_file(i, **kwargs)) + self.config = content + + def get(self): + return self.config + + def override(self, data: Dict[str, Any]): + parser = ConfigParser(config=self.config) + for id, v in data.items(): + parser[id] = v + + @classmethod + def split_file_path_id(cls, path: str) -> Optional[Tuple[str, str]]: + pattern = "|".join(cls.suffixes) + result = re.findall(pattern, path, re.IGNORECASE) + if len(result) != 1: + # path should only contain 1 file + return None + paths = path.split(result[0]) + # return file path and target id + return paths[0] + result[0], paths[1][1:] if paths[1] != "" else "" + + def _do_resolve(self, config, **kwargs): + if isinstance(config, (dict, list)): + subs = enumerate(config) if isinstance(config, list) else config.items() + for k, v in subs: + config[k] = self._do_resolve(v, **kwargs) + if isinstance(config, str) and config.startswith(self.macro): + # only support macro mark at the beginning of a string + id = config[len(self.macro):] + paths = self.split_file_path_id(id) + if paths is None: + # id is in the current config file + parser = ConfigParser(config=self.config) + data = deepcopy(parser[id]) + else: + # id is in another config file + parser = ConfigParser(config=self.load_config_file(paths[0], **kwargs)) + data = parser[paths[1]] + # recursively check the resolved content + return self._do_resolve(data, **kwargs) + return config + + def resolve_macro(self, **kwargs): + self.config = self._do_resolve(config=deepcopy(self.config)) diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index e7d55b55df..f41cf85865 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -11,7 +11,9 @@ from typing import Dict, Optional, Sequence, Union -from monai.bundle.utils import parse_config_file, update_default_args +from monai.bundle.config_parser import ConfigParser +from monai.bundle.config_reader import ConfigReader +from monai.bundle.utils import update_default_args from monai.utils import optional_import fire, _ = optional_import("fire") @@ -76,11 +78,19 @@ def run( kwargs[k] = v args = update_default_args(args=args_file, **kwargs) - config_parser = parse_config_file( - config_file=args["config_file"], meta_file=args["meta_file"], override=args.get("override") - ) + reader = ConfigReader() + reader.read_config(f=args["config_file"]) + reader.read_meta(f=args["meta_file"]) + + override = args.get("override") + if override is not None: + reader.override(data=override) + + reader.resolve_macro() + # get expected workflow to run - workflow = config_parser.get_parsed_content(id=args["target"]) + parser = ConfigParser(reader.get()) + workflow = parser.get_parsed_content(id=args["target"]) workflow.run() diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 0a61e4e4a0..4ba3249f6b 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -9,60 +9,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json -import re -from typing import Dict, Optional, Sequence, Union - -from monai.bundle.config_parser import ConfigParser -from monai.utils import ensure_tuple, optional_import - -yaml, _ = optional_import("yaml") - - -def load_config_file(filepath: str, **kwargs): - """ - Load config file with specified file path. - Suppprt JSON and YAML formats. - - Args: - filepath: path of target file to load, supported postfixes: `.json`, `.yml`, `.yaml`. - kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. - for more details, please check: - https://docs.python.org/3/library/json.html#json.load. - https://pyyaml.org/wiki/PyYAMLDocumentation. - - """ - with open(filepath) as f: - if filepath.lower().endswith(".json"): - return json.load(f, **kwargs) - if filepath.lower().endswith((".yml", ".yaml")): - return yaml.safe_load(f, **kwargs) - raise ValueError("only support JSON or YAML config file so far.") - - -def load_config_file_content(path: str, **kwargs): - """ - Load part of the content from a config file with specified `id` in the path. - Suppprt JSON and YAML formats file. - - Args: - path: path of target file to load, it can only load part of it appending target `id` - in the path with "#" mark. for example: `/data/config.json`, `/data/config.json#net#`. - kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. - for more details, please check: - https://docs.python.org/3/library/json.html#json.load. - https://pyyaml.org/wiki/PyYAMLDocumentation. - - """ - pattern = r"(json|yaml|yml)" - result = re.findall(pattern, path, re.IGNORECASE) - if len(result) != 1: - raise ValueError(f"path should only contain 1 file, but got: {path}.") - - # split the path into filepath and target id of the content - paths = path.split(result[0]) - parser = ConfigParser(config=load_config_file(paths[0] + result[0], **kwargs)) - return parser[paths[1][1:] if paths[1] != "" else ""] +from typing import Dict, Optional, Union +from monai.bundle.config_reader import ConfigReader def update_default_args(args: Optional[Union[str, Dict]] = None, **kwargs) -> Dict: @@ -77,51 +25,10 @@ def update_default_args(args: Optional[Union[str, Dict]] = None, **kwargs) -> Di """ args_: Dict = args if isinstance(args, dict) else {} # type: ignore if isinstance(args, str): - args_ = load_config_file_content(args) + # args are defined in a structured file + args_ = ConfigReader.load_config_file(args) # recursively update the default args with new args for k, v in kwargs.items(): args_[k] = update_default_args(args_[k], **v) if isinstance(v, dict) and isinstance(args_.get(k), dict) else v return args_ - - -def parse_config_file( - config_file: Union[str, Sequence[str]], meta_file: Union[str, Sequence[str]], override: Optional[Dict] = None -) -> ConfigParser: - """ - Read the config file, metadata file and override with specified `id=value` pairs. - Put metadata in the config content with key "". - The `id` in `override` identifies target position to override with the `value`. - If `value` starts with "", it will automatically read the `file` - and use the content as `value`. - - Args: - config_file: filepath of the config file, the config content must be a dictionary, - if providing a list of files, wil merge the content of them. - meta_file: filepath of the metadata file, the config content must be a dictionary, - if providing a list of files, wil merge the content of them. - override: dict of `{id: value}` pairs to override or add the config content. - - """ - config: Dict = {"": {}} - for f in ensure_tuple(config_file): - content = load_config_file(f) - if not isinstance(content, dict): - raise ValueError("input config content must be a dictionary.") - config.update(content) - - for f in ensure_tuple(meta_file): - content = load_config_file(f) - if not isinstance(content, dict): - raise ValueError("meta data content must be a dictionary.") - config[""].update(content) - - parser = ConfigParser(config=config) - - if override is not None: - for id, v in override.items(): - if isinstance(v, str) and v.startswith(""): - v = load_config_file_content(v[6:]) - parser[id] = v - - return parser diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 2b0e796a45..aa46666172 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -62,7 +62,7 @@ def test_shape(self, config_file, expected_shape): if sys.platform == "win32": override = "'network':'$@network_def.to(@device)','dataset#':'Dataset'" else: - override = f"'network':'{overridefile1}#move_net','dataset#':'{overridefile2}'" + override = f"'network':'%{overridefile1}#move_net','dataset#':'%{overridefile2}'" # test with `monai.bundle` as CLI entry directly ret = subprocess.check_call( [ From 894ca88e4a8cd812f2ab38977f7f26e8fb23ac2d Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 8 Mar 2022 22:19:15 +0800 Subject: [PATCH 49/76] [DLMED] add doc-string Signed-off-by: Nic Ma --- monai/bundle/config_reader.py | 83 ++++++++++++++++++++++++++++++++--- monai/bundle/utils.py | 1 + 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index b6b7a1af2d..80968f6cdd 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -10,19 +10,26 @@ # limitations under the License. import json -from pathlib import Path import re from copy import deepcopy +from pathlib import Path from typing import Any, Dict, Optional, Sequence, Tuple, Union -from monai.config import PathLike from monai.bundle.config_parser import ConfigParser +from monai.config import PathLike from monai.utils import ensure_tuple, optional_import yaml, _ = optional_import("yaml") class ConfigReader: + """ + Read metadata, config from structured JSON or YAML files. + Support to override the config content with specified `id` and value. + Support to resolve the macro tokens in the config content. + + """ + suffixes = ["json", "yaml", "yml"] macro = "%" # macro prefix meta_key = "" # field key to save meta data @@ -53,6 +60,20 @@ def load_config_file(cls, filepath: PathLike, **kwargs): raise ValueError("only support JSON or YAML config file so far.") def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): + """ + Read the metadata from specified JSON or YAML file. + Will put metadata in the config content with key "". + + Args: + f: filepath of the meta data file, the content must be a dictionary, + if providing a list of files, wil merge the content of them. + if providing a dictionary directly, use it as meta data. + kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. + for more details, please check: + https://docs.python.org/3/library/json.html#json.load. + https://pyyaml.org/wiki/PyYAMLDocumentation. + + """ content = {} if isinstance(f, dict): # already loaded in dict @@ -63,6 +84,20 @@ def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): self.config[self.meta_key] = content def read_config(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): + """ + Read the config from specified JSON or YAML file. + Will store the config content in the `self.config` property. + + Args: + f: filepath of the config file, the content must be a dictionary, + if providing a list of files, wil merge the content of them. + if providing a dictionary directly, use it as config. + kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. + for more details, please check: + https://docs.python.org/3/library/json.html#json.load. + https://pyyaml.org/wiki/PyYAMLDocumentation. + + """ content = {self.meta_key: self.config[self.meta_key]} if isinstance(f, dict): # already loaded in dict @@ -73,6 +108,10 @@ def read_config(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): self.config = content def get(self): + """ + Get the loaded config content. + + """ return self.config def override(self, data: Dict[str, Any]): @@ -82,6 +121,15 @@ def override(self, data: Dict[str, Any]): @classmethod def split_file_path_id(cls, path: str) -> Optional[Tuple[str, str]]: + """ + In order to load part of the content from a config file with specified `id`, split the full path + to `filepath` and target `id`. + + Args: + path: path of target file to load, it can specify part of it with appending target `id` + in the path with "#" mark. for example: `/data/config.json`, `/data/config.json#net#`. + + """ pattern = "|".join(cls.suffixes) result = re.findall(pattern, path, re.IGNORECASE) if len(result) != 1: @@ -91,14 +139,27 @@ def split_file_path_id(cls, path: str) -> Optional[Tuple[str, str]]: # return file path and target id return paths[0] + result[0], paths[1][1:] if paths[1] != "" else "" - def _do_resolve(self, config, **kwargs): + def _do_resolve(self, config: Any, **kwargs): + """ + Recursively resolve the config content to replace the macro tokens with target content. + The macro tokens are marked as starting with "%", can be from another structured file, like: + `"net": "%default_net"`, `"net": "%/data/config.json#net#"`. + + Args: + config: input config file to resolve. + kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. + for more details, please check: + https://docs.python.org/3/library/json.html#json.load. + https://pyyaml.org/wiki/PyYAMLDocumentation. + + """ if isinstance(config, (dict, list)): subs = enumerate(config) if isinstance(config, list) else config.items() for k, v in subs: config[k] = self._do_resolve(v, **kwargs) if isinstance(config, str) and config.startswith(self.macro): # only support macro mark at the beginning of a string - id = config[len(self.macro):] + id = config[len(self.macro) :] paths = self.split_file_path_id(id) if paths is None: # id is in the current config file @@ -113,4 +174,16 @@ def _do_resolve(self, config, **kwargs): return config def resolve_macro(self, **kwargs): - self.config = self._do_resolve(config=deepcopy(self.config)) + """ + Recursively resolve `self.config` to replace the macro tokens with target content. + The macro tokens are marked as starting with "%", can be from another structured file, like: + `"net": "%default_net"`, `"net": "%/data/config.json#net#"`. + + Args: + kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. + for more details, please check: + https://docs.python.org/3/library/json.html#json.load. + https://pyyaml.org/wiki/PyYAMLDocumentation. + + """ + self.config = self._do_resolve(config=deepcopy(self.config), **kwargs) diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 4ba3249f6b..35fd9e46a4 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -10,6 +10,7 @@ # limitations under the License. from typing import Dict, Optional, Union + from monai.bundle.config_reader import ConfigReader From 682c99178a2933087cb7c6bd7c27aa7fcf5d1232 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 8 Mar 2022 22:47:50 +0800 Subject: [PATCH 50/76] [DLMED] add export and test Signed-off-by: Nic Ma --- monai/bundle/config_reader.py | 33 +++++++++++++++++++++++++++------ tests/test_bundle_run.py | 13 +++++++------ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index 80968f6cdd..99d1d97bac 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -45,20 +45,41 @@ def load_config_file(cls, filepath: PathLike, **kwargs): Args: filepath: path of target file to load, supported postfixes: `.json`, `.yml`, `.yaml`. - kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. + kwargs: other arguments for `json.load` or `yaml.safe_load`, depends on file format. for more details, please check: https://docs.python.org/3/library/json.html#json.load. https://pyyaml.org/wiki/PyYAMLDocumentation. """ _filepath: str = str(Path(filepath)) - with open(_filepath) as f: + with open(_filepath, "r") as f: if _filepath.lower().endswith(cls.suffixes[0]): return json.load(f, **kwargs) if _filepath.lower().endswith(tuple(cls.suffixes[1:])): return yaml.safe_load(f, **kwargs) raise ValueError("only support JSON or YAML config file so far.") + def export_config_file(self, filepath: PathLike, **kwargs): + """ + Export the config content to the specified file path. + Suppprt JSON and YAML formats. + + Args: + filepath: target file path to save, supported postfixes: `.json`, `.yml`, `.yaml`. + kwargs: other arguments for `json.dump` or `yaml.safe_dump`, depends on file format. + for more details, please check: + https://docs.python.org/3/library/json.html#json.dump. + https://pyyaml.org/wiki/PyYAMLDocumentation. + + """ + _filepath: str = str(Path(filepath)) + with open(_filepath, "w") as f: + if _filepath.lower().endswith(self.suffixes[0]): + return json.dump(self.config, f, **kwargs) + if _filepath.lower().endswith(tuple(self.suffixes[1:])): + return yaml.safe_dump(self.config, f, **kwargs) + raise ValueError("only support JSON or YAML config file so far.") + def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): """ Read the metadata from specified JSON or YAML file. @@ -68,7 +89,7 @@ def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): f: filepath of the meta data file, the content must be a dictionary, if providing a list of files, wil merge the content of them. if providing a dictionary directly, use it as meta data. - kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. + kwargs: other arguments for `json.load` or `yaml.safe_load`, depends on file format. for more details, please check: https://docs.python.org/3/library/json.html#json.load. https://pyyaml.org/wiki/PyYAMLDocumentation. @@ -92,7 +113,7 @@ def read_config(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): f: filepath of the config file, the content must be a dictionary, if providing a list of files, wil merge the content of them. if providing a dictionary directly, use it as config. - kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. + kwargs: other arguments for `json.load` or `yaml.safe_load`, depends on file format. for more details, please check: https://docs.python.org/3/library/json.html#json.load. https://pyyaml.org/wiki/PyYAMLDocumentation. @@ -147,7 +168,7 @@ def _do_resolve(self, config: Any, **kwargs): Args: config: input config file to resolve. - kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. + kwargs: other arguments for `json.load` or `yaml.safe_load`, depends on file format. for more details, please check: https://docs.python.org/3/library/json.html#json.load. https://pyyaml.org/wiki/PyYAMLDocumentation. @@ -180,7 +201,7 @@ def resolve_macro(self, **kwargs): `"net": "%default_net"`, `"net": "%/data/config.json#net#"`. Args: - kwargs: other arguments for `json.load` or `yaml.load`, depends on file format. + kwargs: other arguments for `json.load` or `yaml.safe_load`, depends on file format. for more details, please check: https://docs.python.org/3/library/json.html#json.load. https://pyyaml.org/wiki/PyYAMLDocumentation. diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index aa46666172..552af3a9a3 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -18,9 +18,9 @@ import nibabel as nib import numpy as np -import yaml from parameterized import parameterized +from monai.bundle import ConfigReader from monai.transforms import LoadImage TEST_CASE_1 = [os.path.join(os.path.dirname(__file__), "testing_data", "inference.json"), (128, 128, 128)] @@ -36,16 +36,17 @@ def test_shape(self, config_file, expected_shape): filename = os.path.join(tempdir, "image.nii") nib.save(nib.Nifti1Image(test_image, np.eye(4)), filename) - # generate default args in a file + reader = ConfigReader() + # generate default args in a JSON file + reader.read_config({"config_file": "will be overrided by `config_file` arg"}) def_args = os.path.join(tempdir, "def_args.json") - with open(def_args, "w") as f: - json.dump({"config_file": "will be overrided by `config_file` arg"}, f) + reader.export_config_file(filepath=def_args) meta = {"datalist": [{"image": filename}], "output_dir": tempdir, "window": (96, 96, 96)} # test YAML file + reader.read_config(meta) meta_file = os.path.join(tempdir, "meta.yaml") - with open(meta_file, "w") as f: - yaml.safe_dump(meta, f) + reader.export_config_file(filepath=meta_file) # test override with file, up case postfix overridefile1 = os.path.join(tempdir, "override1.JSON") From 737ccc345414777673f367b64ff343b7ea701a20 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 8 Mar 2022 14:48:31 +0000 Subject: [PATCH 51/76] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- monai/bundle/config_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index 99d1d97bac..6ec88bfaad 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -52,7 +52,7 @@ def load_config_file(cls, filepath: PathLike, **kwargs): """ _filepath: str = str(Path(filepath)) - with open(_filepath, "r") as f: + with open(_filepath) as f: if _filepath.lower().endswith(cls.suffixes[0]): return json.load(f, **kwargs) if _filepath.lower().endswith(tuple(cls.suffixes[1:])): From 5292db61a0c5b9ea2d23dd9843a197922b89df95 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Tue, 8 Mar 2022 15:25:11 +0000 Subject: [PATCH 52/76] python -m monai.bundle Signed-off-by: Wenqi Li --- monai/__init__.py | 2 +- monai/bundle/__init__.py | 11 ++++---- monai/bundle/__main__.py | 19 +++++++++++++ monai/bundle/config_parser.py | 2 ++ monai/bundle/config_reader.py | 2 ++ monai/bundle/reference_resolver.py | 2 ++ monai/bundle/scripts.py | 8 ------ tests/test_bundle_run.py | 43 ++++++------------------------ 8 files changed, 39 insertions(+), 50 deletions(-) create mode 100644 monai/bundle/__main__.py diff --git a/monai/__init__.py b/monai/__init__.py index a823a3e1e2..e56a2f3444 100644 --- a/monai/__init__.py +++ b/monai/__init__.py @@ -39,7 +39,7 @@ # handlers_* have some external decorators the users may not have installed # *.so files and folder "_C" may not exist when the cpp extensions are not compiled -excludes = "(^(monai.handlers))|((\\.so)$)|(^(monai._C))" +excludes = "(^(monai.handlers))|(^(monai.bundle))|((\\.so)$)|(^(monai._C))" # load directory modules only, skip loading individual files load_submodules(sys.modules[__name__], False, exclude_pattern=excludes) diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py index 5236e6b788..78f7ff4a1e 100644 --- a/monai/bundle/__init__.py +++ b/monai/bundle/__init__.py @@ -9,9 +9,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, Instantiable -from .config_parser import ConfigParser -from .config_reader import ConfigReader -from .reference_resolver import ReferenceResolver -from .scripts import run -from .utils import update_default_args +from monai.bundle.config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, Instantiable +from monai.bundle.config_parser import ConfigParser +from monai.bundle.config_reader import ConfigReader +from monai.bundle.reference_resolver import ReferenceResolver +from monai.bundle.utils import update_default_args diff --git a/monai/bundle/__main__.py b/monai/bundle/__main__.py new file mode 100644 index 0000000000..148947b863 --- /dev/null +++ b/monai/bundle/__main__.py @@ -0,0 +1,19 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from monai.bundle.scripts import run # noqa: F401 + +if __name__ == "__main__": + from monai.utils import optional_import + + fire, _ = optional_import("fire") + fire.Fire() diff --git a/monai/bundle/config_parser.py b/monai/bundle/config_parser.py index 5ebcfd03b4..7cb8e3bb3a 100644 --- a/monai/bundle/config_parser.py +++ b/monai/bundle/config_parser.py @@ -16,6 +16,8 @@ from monai.bundle.config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem from monai.bundle.reference_resolver import ReferenceResolver +__all__ = ["ConfigParser"] + class ConfigParser: """ diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index 6ec88bfaad..c01e687487 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -21,6 +21,8 @@ yaml, _ = optional_import("yaml") +__all__ = ["ConfigReader"] + class ConfigReader: """ diff --git a/monai/bundle/reference_resolver.py b/monai/bundle/reference_resolver.py index 45d897af05..b3834bf76c 100644 --- a/monai/bundle/reference_resolver.py +++ b/monai/bundle/reference_resolver.py @@ -15,6 +15,8 @@ from monai.bundle.config_item import ConfigComponent, ConfigExpression, ConfigItem from monai.utils import look_up_option +__all__ = ["ReferenceResolver"] + class ReferenceResolver: """ diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index f41cf85865..c7d86b14d2 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -14,9 +14,6 @@ from monai.bundle.config_parser import ConfigParser from monai.bundle.config_reader import ConfigReader from monai.bundle.utils import update_default_args -from monai.utils import optional_import - -fire, _ = optional_import("fire") def run( @@ -77,7 +74,6 @@ def run( # skip None args kwargs[k] = v args = update_default_args(args=args_file, **kwargs) - reader = ConfigReader() reader.read_config(f=args["config_file"]) reader.read_meta(f=args["meta_file"]) @@ -92,7 +88,3 @@ def run( parser = ConfigParser(reader.get()) workflow = parser.get_parsed_content(id=args["target"]) workflow.run() - - -if __name__ == "__main__": - fire.Fire() diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 552af3a9a3..37c7c6f05b 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -65,45 +65,18 @@ def test_shape(self, config_file, expected_shape): else: override = f"'network':'%{overridefile1}#move_net','dataset#':'%{overridefile2}'" # test with `monai.bundle` as CLI entry directly - ret = subprocess.check_call( - [ - f"{sys.executable}", - "-m", - "monai.bundle.scripts", - "run", - "--meta_file", - f"{meta_file}", - "--config_file", - f"{config_file}", - # `fire` can not parse below string, will pass to `run` as a string representing a dict - "--override", - f"{{'postprocessing##transforms#2##output_postfix':'seg',{override}}}", - "--target", - "evaluator", - ] - ) + cmd = f"{sys.executable} -m monai.bundle run --meta_file {meta_file} --config_file {config_file}" + cmd += f" --override {{'postprocessing##transforms#2##output_postfix':'seg',{override}}}" + cmd += " --target evaluator" + ret = subprocess.check_call(cmd.split(" ")) self.assertEqual(ret, 0) self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_seg.nii.gz")).shape, expected_shape) # here test the script with `google fire` tool as CLI - ret = subprocess.check_call( - [ - f"{sys.executable}", - "-m", - "fire", - "monai.bundle", - "run", - "--meta_file", - f"{meta_file}", - "--config_file", - f"{config_file}", - "--override", - # `fire` can parse below string as a dictionary - f"{{'evaluator##amp':False,{override}}}", - "--target", - "evaluator", - ] - ) + cmd = f"{sys.executable} -m monai.bundle run --meta_file {meta_file} --config_file {config_file}" + cmd += f" --override {{'evaluator##amp':False,{override}}}" + cmd += " --target evaluator" + ret = subprocess.check_call(cmd.split(" ")) self.assertEqual(ret, 0) self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_trans.nii.gz")).shape, expected_shape) From 45b240d0953d511665fd404348c55c63f2a5b87d Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Tue, 8 Mar 2022 15:46:19 +0000 Subject: [PATCH 53/76] fixes doc tests Signed-off-by: Wenqi Li --- monai/bundle/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py index 78f7ff4a1e..e12696814f 100644 --- a/monai/bundle/__init__.py +++ b/monai/bundle/__init__.py @@ -13,4 +13,5 @@ from monai.bundle.config_parser import ConfigParser from monai.bundle.config_reader import ConfigReader from monai.bundle.reference_resolver import ReferenceResolver +from monai.bundle.scripts import run from monai.bundle.utils import update_default_args From 7db72644df01ddf8b00175c95bb20185b329264a Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Wed, 9 Mar 2022 07:15:04 +0800 Subject: [PATCH 54/76] [DLMED] enhance test Signed-off-by: Nic Ma --- tests/test_bundle_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 37c7c6f05b..4b60555fce 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -73,8 +73,8 @@ def test_shape(self, config_file, expected_shape): self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_seg.nii.gz")).shape, expected_shape) # here test the script with `google fire` tool as CLI - cmd = f"{sys.executable} -m monai.bundle run --meta_file {meta_file} --config_file {config_file}" - cmd += f" --override {{'evaluator##amp':False,{override}}}" + cmd = f"{sys.executable} -m fire monai.bundle.scripts run --meta_file {meta_file}" + cmd += f" --config_file {config_file} --override {{'evaluator##amp':False,{override}}}" cmd += " --target evaluator" ret = subprocess.check_call(cmd.split(" ")) self.assertEqual(ret, 0) From 469874a90f2497bba160617c2ceee8aa35c8c486 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 9 Mar 2022 00:03:50 +0000 Subject: [PATCH 55/76] rel import Signed-off-by: Wenqi Li --- monai/bundle/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py index e12696814f..5236e6b788 100644 --- a/monai/bundle/__init__.py +++ b/monai/bundle/__init__.py @@ -9,9 +9,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from monai.bundle.config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, Instantiable -from monai.bundle.config_parser import ConfigParser -from monai.bundle.config_reader import ConfigReader -from monai.bundle.reference_resolver import ReferenceResolver -from monai.bundle.scripts import run -from monai.bundle.utils import update_default_args +from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, Instantiable +from .config_parser import ConfigParser +from .config_reader import ConfigReader +from .reference_resolver import ReferenceResolver +from .scripts import run +from .utils import update_default_args From e769e55b07f015ad26a996cec5cc65f35c006115 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Wed, 9 Mar 2022 22:47:01 +0800 Subject: [PATCH 56/76] [DLMED] refine macro Signed-off-by: Nic Ma --- monai/bundle/config_parser.py | 48 ++++++++++++++- monai/bundle/config_reader.py | 111 +++++++++------------------------- monai/bundle/scripts.py | 9 +-- tests/test_bundle_run.py | 9 +-- 4 files changed, 82 insertions(+), 95 deletions(-) diff --git a/monai/bundle/config_parser.py b/monai/bundle/config_parser.py index 7cb8e3bb3a..d47187373e 100644 --- a/monai/bundle/config_parser.py +++ b/monai/bundle/config_parser.py @@ -14,6 +14,7 @@ from typing import Any, Dict, Optional, Sequence, Union from monai.bundle.config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem +from monai.bundle.config_reader import ConfigReader from monai.bundle.reference_resolver import ReferenceResolver __all__ = ["ConfigParser"] @@ -76,6 +77,8 @@ class ConfigParser: """ + macro = "%" # macro prefix + def __init__( self, config: Any, @@ -164,6 +167,45 @@ def set(self, config: Any, id: str = ""): """ self[id] = config + def _do_resolve(self, config: Any): + """ + Recursively resolve the config content to replace the macro tokens with target content. + The macro tokens are marked as starting with "%", can be from another structured file, like: + `"net": "%default_net"`, `"net": "%/data/config.json#net#"`. + + Args: + config: input config file to resolve. + + """ + if isinstance(config, (dict, list)): + subs = enumerate(config) if isinstance(config, list) else config.items() + for k, v in subs: + config[k] = self._do_resolve(v) + if isinstance(config, str) and config.startswith(self.macro): + # only support macro mark at the beginning of a string + id = config[len(self.macro) :] + paths = ConfigReader.extract_file_path(id) + if paths is None: + # id is in the current config file + parser = ConfigParser(config=self.get()) + data = deepcopy(parser[id]) + else: + # id is in another config file + parser = ConfigParser(config=ConfigReader.load_config_file(paths[0])) + data = parser[paths[1][len(self.ref_resolver.sep) :] if paths[1] != "" else ""] + # recursively check the resolved content + return self._do_resolve(data) + return config + + def resolve_macro(self): + """ + Recursively resolve `self.config` to replace the macro tokens with target content. + The macro tokens are marked as starting with "%", can be from another structured file, like: + `"net": "%default_net"`, `"net": "%/data/config.json#net#"`. + + """ + self.set(self._do_resolve(config=deepcopy(self.get()))) + def _do_parse(self, config, id: str = ""): """ Recursively parse the nested data in config source, add every item as `ConfigItem` to the resolver. @@ -193,7 +235,8 @@ def _do_parse(self, config, id: str = ""): def parse(self, reset: bool = True): """ - Recursively parse the config source, add every item as ``ConfigItem`` to the resolver. + Recursively resolve `self.config` to replace the macro tokens with target content. + Then recursively parse the config source, add every item as ``ConfigItem`` to the reference resolver. Args: reset: whether to reset the ``reference_resolver`` before parsing. Defaults to `True`. @@ -201,7 +244,8 @@ def parse(self, reset: bool = True): """ if reset: self.ref_resolver.reset() - self._do_parse(config=self.config) + self.resolve_macro() + self._do_parse(config=self.get()) def get_parsed_content(self, id: str = "", **kwargs): """ diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index c01e687487..3189f65d1a 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -11,11 +11,9 @@ import json import re -from copy import deepcopy from pathlib import Path -from typing import Any, Dict, Optional, Sequence, Tuple, Union +from typing import Dict, Optional, Sequence, Tuple, Union -from monai.bundle.config_parser import ConfigParser from monai.config import PathLike from monai.utils import ensure_tuple, optional_import @@ -33,7 +31,6 @@ class ConfigReader: """ suffixes = ["json", "yaml", "yml"] - macro = "%" # macro prefix meta_key = "" # field key to save meta data def __init__(self): @@ -61,12 +58,14 @@ def load_config_file(cls, filepath: PathLike, **kwargs): return yaml.safe_load(f, **kwargs) raise ValueError("only support JSON or YAML config file so far.") - def export_config_file(self, filepath: PathLike, **kwargs): + @classmethod + def export_config_file(cls, config: Dict, filepath: PathLike, **kwargs): """ Export the config content to the specified file path. Suppprt JSON and YAML formats. Args: + config: source config content to export. filepath: target file path to save, supported postfixes: `.json`, `.yml`, `.yaml`. kwargs: other arguments for `json.dump` or `yaml.safe_dump`, depends on file format. for more details, please check: @@ -76,12 +75,32 @@ def export_config_file(self, filepath: PathLike, **kwargs): """ _filepath: str = str(Path(filepath)) with open(_filepath, "w") as f: - if _filepath.lower().endswith(self.suffixes[0]): - return json.dump(self.config, f, **kwargs) - if _filepath.lower().endswith(tuple(self.suffixes[1:])): - return yaml.safe_dump(self.config, f, **kwargs) + if _filepath.lower().endswith(cls.suffixes[0]): + return json.dump(config, f, **kwargs) + if _filepath.lower().endswith(tuple(cls.suffixes[1:])): + return yaml.safe_dump(config, f, **kwargs) raise ValueError("only support JSON or YAML config file so far.") + @classmethod + def extract_file_path(cls, src: str) -> Optional[Tuple[str, str]]: + """ + extract a config file path from the source string, return path and the rest string. + return `None` if can't find any config file path. + + Args: + src: source string to extract, it can be a config file path with / without additional information. + for example: "/data/config.json", "/data/config.json#net#". + + """ + pattern = "|".join(cls.suffixes) + result = re.findall(pattern, src, re.IGNORECASE) + if len(result) != 1: + # src should only contain 1 file + return None + items = src.split(result[0]) + # return file path and the rest + return items[0] + result[0], items[1] + def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): """ Read the metadata from specified JSON or YAML file. @@ -136,77 +155,3 @@ def get(self): """ return self.config - - def override(self, data: Dict[str, Any]): - parser = ConfigParser(config=self.config) - for id, v in data.items(): - parser[id] = v - - @classmethod - def split_file_path_id(cls, path: str) -> Optional[Tuple[str, str]]: - """ - In order to load part of the content from a config file with specified `id`, split the full path - to `filepath` and target `id`. - - Args: - path: path of target file to load, it can specify part of it with appending target `id` - in the path with "#" mark. for example: `/data/config.json`, `/data/config.json#net#`. - - """ - pattern = "|".join(cls.suffixes) - result = re.findall(pattern, path, re.IGNORECASE) - if len(result) != 1: - # path should only contain 1 file - return None - paths = path.split(result[0]) - # return file path and target id - return paths[0] + result[0], paths[1][1:] if paths[1] != "" else "" - - def _do_resolve(self, config: Any, **kwargs): - """ - Recursively resolve the config content to replace the macro tokens with target content. - The macro tokens are marked as starting with "%", can be from another structured file, like: - `"net": "%default_net"`, `"net": "%/data/config.json#net#"`. - - Args: - config: input config file to resolve. - kwargs: other arguments for `json.load` or `yaml.safe_load`, depends on file format. - for more details, please check: - https://docs.python.org/3/library/json.html#json.load. - https://pyyaml.org/wiki/PyYAMLDocumentation. - - """ - if isinstance(config, (dict, list)): - subs = enumerate(config) if isinstance(config, list) else config.items() - for k, v in subs: - config[k] = self._do_resolve(v, **kwargs) - if isinstance(config, str) and config.startswith(self.macro): - # only support macro mark at the beginning of a string - id = config[len(self.macro) :] - paths = self.split_file_path_id(id) - if paths is None: - # id is in the current config file - parser = ConfigParser(config=self.config) - data = deepcopy(parser[id]) - else: - # id is in another config file - parser = ConfigParser(config=self.load_config_file(paths[0], **kwargs)) - data = parser[paths[1]] - # recursively check the resolved content - return self._do_resolve(data, **kwargs) - return config - - def resolve_macro(self, **kwargs): - """ - Recursively resolve `self.config` to replace the macro tokens with target content. - The macro tokens are marked as starting with "%", can be from another structured file, like: - `"net": "%default_net"`, `"net": "%/data/config.json#net#"`. - - Args: - kwargs: other arguments for `json.load` or `yaml.safe_load`, depends on file format. - for more details, please check: - https://docs.python.org/3/library/json.html#json.load. - https://pyyaml.org/wiki/PyYAMLDocumentation. - - """ - self.config = self._do_resolve(config=deepcopy(self.config), **kwargs) diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index c7d86b14d2..c432f146f8 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -74,17 +74,18 @@ def run( # skip None args kwargs[k] = v args = update_default_args(args=args_file, **kwargs) + reader = ConfigReader() reader.read_config(f=args["config_file"]) reader.read_meta(f=args["meta_file"]) + parser = ConfigParser(reader.get()) + override = args.get("override") if override is not None: - reader.override(data=override) - - reader.resolve_macro() + for k, v in override.items(): + parser[k] = v # get expected workflow to run - parser = ConfigParser(reader.get()) workflow = parser.get_parsed_content(id=args["target"]) workflow.run() diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 4b60555fce..00b4bb14c9 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -36,17 +36,14 @@ def test_shape(self, config_file, expected_shape): filename = os.path.join(tempdir, "image.nii") nib.save(nib.Nifti1Image(test_image, np.eye(4)), filename) - reader = ConfigReader() # generate default args in a JSON file - reader.read_config({"config_file": "will be overrided by `config_file` arg"}) - def_args = os.path.join(tempdir, "def_args.json") - reader.export_config_file(filepath=def_args) + def_args = {"config_file": "will be overrided by `config_file` arg"} + ConfigReader.export_config_file(config=def_args, filepath=os.path.join(tempdir, "def_args.json")) meta = {"datalist": [{"image": filename}], "output_dir": tempdir, "window": (96, 96, 96)} # test YAML file - reader.read_config(meta) meta_file = os.path.join(tempdir, "meta.yaml") - reader.export_config_file(filepath=meta_file) + ConfigReader.export_config_file(config=meta, filepath=meta_file) # test override with file, up case postfix overridefile1 = os.path.join(tempdir, "override1.JSON") From efbdb21f711236e2b68fa951910f88e0190047c3 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 9 Mar 2022 15:17:12 +0000 Subject: [PATCH 57/76] fixes flake8 f401 Signed-off-by: Wenqi Li --- monai/bundle/__main__.py | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/bundle/__main__.py b/monai/bundle/__main__.py index 148947b863..7a87030bec 100644 --- a/monai/bundle/__main__.py +++ b/monai/bundle/__main__.py @@ -10,7 +10,7 @@ # limitations under the License. -from monai.bundle.scripts import run # noqa: F401 +from monai.bundle.scripts import run if __name__ == "__main__": from monai.utils import optional_import diff --git a/setup.cfg b/setup.cfg index 8ee71553fd..aa5eae07a9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -112,7 +112,7 @@ ignore = W504 C408 N812 # lowercase 'torch.nn.functional' imported as non lowercase 'F' -per_file_ignores = __init__.py: F401 +per_file_ignores = __init__.py: F401, __main__.py: F401 exclude = *.pyi,.git,.eggs,monai/_version.py,versioneer.py,venv,.venv,_version.py [isort] From d3fd2c4a34dc974b2fc1cd1a275b90d7c90cfece Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 9 Mar 2022 15:55:23 +0000 Subject: [PATCH 58/76] mv to monai.bundle Signed-off-by: Wenqi Li --- monai/bundle/config_item.py | 4 ++-- monai/bundle/config_parser.py | 4 ++-- tests/test_bundle_run.py | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/monai/bundle/config_item.py b/monai/bundle/config_item.py index 44cdd3c634..b6334ee9d5 100644 --- a/monai/bundle/config_item.py +++ b/monai/bundle/config_item.py @@ -160,7 +160,7 @@ def __repr__(self) -> str: class ConfigComponent(ConfigItem, Instantiable): """ - Subclass of :py:class:`monai.apps.ConfigItem`, this class uses a dictionary with string keys to + Subclass of :py:class:`monai.bundle.ConfigItem`, this class uses a dictionary with string keys to represent a component of `class` or `function` and supports instantiation. Currently, four special keys (strings surrounded by ``<>``) are defined and interpreted beyond the regular literals: @@ -283,7 +283,7 @@ def instantiate(self, **kwargs) -> object: # type: ignore class ConfigExpression(ConfigItem): """ - Subclass of :py:class:`monai.apps.ConfigItem`, the `ConfigItem` represents an executable expression + Subclass of :py:class:`monai.bundle.ConfigItem`, the `ConfigItem` represents an executable expression (execute based on ``eval()``). See also: diff --git a/monai/bundle/config_parser.py b/monai/bundle/config_parser.py index d47187373e..7ec50cf9a6 100644 --- a/monai/bundle/config_parser.py +++ b/monai/bundle/config_parser.py @@ -33,7 +33,7 @@ class ConfigParser: .. code-block:: python - from monai.apps import ConfigParser + from monai.bundle import ConfigParser config = { "my_dims": 2, @@ -73,7 +73,7 @@ class ConfigParser: See also: - - :py:class:`monai.apps.ConfigItem` + - :py:class:`monai.bundle.ConfigItem` """ diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 00b4bb14c9..2514df19da 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -50,7 +50,8 @@ def test_shape(self, config_file, expected_shape): with open(overridefile1, "w") as f: # test override with part of the overriding file json.dump({"move_net": "$@network_def.to(@device)"}, f) - overridefile2 = os.path.join(tempdir, "override2.JSON") + overridefile2 = os.path.join(tempdir, "jsons/override2.JSON") + os.mkdir(os.path.join(tempdir, "jsons")) with open(overridefile2, "w") as f: # test override with the whole overriding file json.dump("Dataset", f) From 74faa8e5941095ec9fab8994a0d1ab62f799ab8c Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 9 Mar 2022 19:31:46 +0000 Subject: [PATCH 59/76] fixes split_path_id, update docstrings Signed-off-by: Wenqi Li --- monai/bundle/config_parser.py | 25 +++------ monai/bundle/config_reader.py | 98 +++++++++++++++++------------------ monai/bundle/scripts.py | 67 ++++++++++++------------ tests/test_bundle_run.py | 4 +- 4 files changed, 91 insertions(+), 103 deletions(-) diff --git a/monai/bundle/config_parser.py b/monai/bundle/config_parser.py index 7ec50cf9a6..86a830990e 100644 --- a/monai/bundle/config_parser.py +++ b/monai/bundle/config_parser.py @@ -170,38 +170,27 @@ def set(self, config: Any, id: str = ""): def _do_resolve(self, config: Any): """ Recursively resolve the config content to replace the macro tokens with target content. - The macro tokens are marked as starting with "%", can be from another structured file, like: - `"net": "%default_net"`, `"net": "%/data/config.json#net#"`. + The macro tokens start with "%", can be from another structured file, like: + ``{"net": "%default_net"}``, ``{"net": "%/data/config.json#net#"}}``. Args: config: input config file to resolve. """ if isinstance(config, (dict, list)): - subs = enumerate(config) if isinstance(config, list) else config.items() - for k, v in subs: + for k, v in enumerate(config) if isinstance(config, list) else config.items(): config[k] = self._do_resolve(v) if isinstance(config, str) and config.startswith(self.macro): - # only support macro mark at the beginning of a string - id = config[len(self.macro) :] - paths = ConfigReader.extract_file_path(id) - if paths is None: - # id is in the current config file - parser = ConfigParser(config=self.get()) - data = deepcopy(parser[id]) - else: - # id is in another config file - parser = ConfigParser(config=ConfigReader.load_config_file(paths[0])) - data = parser[paths[1][len(self.ref_resolver.sep) :] if paths[1] != "" else ""] - # recursively check the resolved content - return self._do_resolve(data) + path, ids = ConfigReader.split_path_id(config[len(self.macro) :]) + parser = ConfigParser(config=self.get() if not path else ConfigReader.load_config_file(path)) + return self._do_resolve(config=deepcopy(parser[ids])) return config def resolve_macro(self): """ Recursively resolve `self.config` to replace the macro tokens with target content. The macro tokens are marked as starting with "%", can be from another structured file, like: - `"net": "%default_net"`, `"net": "%/data/config.json#net#"`. + ``"%default_net"``, ``"%/data/config.json#net#"``. """ self.set(self._do_resolve(config=deepcopy(self.get()))) diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index 3189f65d1a..c6d172cd70 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -12,10 +12,11 @@ import json import re from pathlib import Path -from typing import Dict, Optional, Sequence, Tuple, Union +from typing import Dict, Sequence, Tuple, Union +from monai.bundle.reference_resolver import ReferenceResolver from monai.config import PathLike -from monai.utils import ensure_tuple, optional_import +from monai.utils import ensure_tuple, look_up_option, optional_import yaml, _ = optional_import("yaml") @@ -24,14 +25,20 @@ class ConfigReader: """ - Read metadata, config from structured JSON or YAML files. + Read config and metadata from JSON or YAML files. Support to override the config content with specified `id` and value. Support to resolve the macro tokens in the config content. + See also: + + - https://docs.python.org/3/library/json.html#json.load + - https://pyyaml.org/wiki/PyYAMLDocumentation + """ - suffixes = ["json", "yaml", "yml"] - meta_key = "" # field key to save meta data + suffixes = ("json", "yaml", "yml") + path_match = re.compile(rf"(.*\.({'|'.join(suffixes)})$)", re.IGNORECASE) + meta_key = "" # field key to save metadata def __init__(self): self.config: Dict = {self.meta_key: {}} @@ -39,88 +46,80 @@ def __init__(self): @classmethod def load_config_file(cls, filepath: PathLike, **kwargs): """ - Load config file with specified file path. - Suppprt JSON and YAML formats. + Load config file with specified file path (currently support JSON and YAML files). Args: filepath: path of target file to load, supported postfixes: `.json`, `.yml`, `.yaml`. - kwargs: other arguments for `json.load` or `yaml.safe_load`, depends on file format. - for more details, please check: - https://docs.python.org/3/library/json.html#json.load. - https://pyyaml.org/wiki/PyYAMLDocumentation. + kwargs: other arguments for ``json.load`` or ```yaml.safe_load``, depends on the file format. """ _filepath: str = str(Path(filepath)) + if not cls.path_match.findall(_filepath): + raise ValueError(f'unknown file input: "{filepath}"') with open(_filepath) as f: if _filepath.lower().endswith(cls.suffixes[0]): return json.load(f, **kwargs) - if _filepath.lower().endswith(tuple(cls.suffixes[1:])): + if _filepath.lower().endswith(cls.suffixes[1:]): return yaml.safe_load(f, **kwargs) - raise ValueError("only support JSON or YAML config file so far.") + raise ValueError(f"only support JSON or YAML config file so far, got name {_filepath}.") @classmethod - def export_config_file(cls, config: Dict, filepath: PathLike, **kwargs): + def export_config_file(cls, config: Dict, filepath: PathLike, fmt="json", **kwargs): """ - Export the config content to the specified file path. - Suppprt JSON and YAML formats. + Export the config content to the specified file path (currently support JSON and YAML files). Args: config: source config content to export. - filepath: target file path to save, supported postfixes: `.json`, `.yml`, `.yaml`. - kwargs: other arguments for `json.dump` or `yaml.safe_dump`, depends on file format. - for more details, please check: - https://docs.python.org/3/library/json.html#json.dump. - https://pyyaml.org/wiki/PyYAMLDocumentation. + filepath: target file path to save. + fmt: format of config content, currently support ``"json"`` and ``"yaml"``. + kwargs: other arguments for ``json.dump`` or ``yaml.safe_dump``, depends on the file format. """ _filepath: str = str(Path(filepath)) + writer = look_up_option(fmt.lower(), {"json", "yaml"}) with open(_filepath, "w") as f: - if _filepath.lower().endswith(cls.suffixes[0]): + if writer == "json": return json.dump(config, f, **kwargs) - if _filepath.lower().endswith(tuple(cls.suffixes[1:])): + if writer == "yaml": return yaml.safe_dump(config, f, **kwargs) - raise ValueError("only support JSON or YAML config file so far.") + raise ValueError(f"only support JSON or YAML config file so far, got {writer}.") @classmethod - def extract_file_path(cls, src: str) -> Optional[Tuple[str, str]]: + def split_path_id(cls, src: str) -> Tuple[str, str]: """ - extract a config file path from the source string, return path and the rest string. - return `None` if can't find any config file path. + Split `src` string into two parts: a config file path and component id. + The file path should end with `(json|yaml|yml)`. The component id should be separated by `#` if it exists. Args: - src: source string to extract, it can be a config file path with / without additional information. - for example: "/data/config.json", "/data/config.json#net#". + src: source string to split. """ - pattern = "|".join(cls.suffixes) - result = re.findall(pattern, src, re.IGNORECASE) - if len(result) != 1: - # src should only contain 1 file - return None - items = src.split(result[0]) - # return file path and the rest - return items[0] + result[0], items[1] + path, *ids = f"{src}".rsplit(ReferenceResolver.sep, 1) + ids = ids[0] if ids else "" + path_name = cls.path_match.findall(path) + if not path_name: + return "", src # the src is a pure id + if len(path_name) < 1 and len(path_name[0]) < 1: + raise ValueError(f"invalid config file path: {path}") + return path_name[0][0], ids def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): """ Read the metadata from specified JSON or YAML file. - Will put metadata in the config content with key "". + The metadata as a dictionary will be stored at ``self.config[""]``. Args: - f: filepath of the meta data file, the content must be a dictionary, + f: filepath of the metadata file, the content must be a dictionary, if providing a list of files, wil merge the content of them. - if providing a dictionary directly, use it as meta data. - kwargs: other arguments for `json.load` or `yaml.safe_load`, depends on file format. - for more details, please check: - https://docs.python.org/3/library/json.html#json.load. - https://pyyaml.org/wiki/PyYAMLDocumentation. + if providing a dictionary directly, use it as metadata. + kwargs: other arguments for ``json.load`` or ``yaml.safe_load``, depends on the file format. """ - content = {} if isinstance(f, dict): # already loaded in dict content = f else: + content = {} for i in ensure_tuple(f): content.update(self.load_config_file(i, **kwargs)) self.config[self.meta_key] = content @@ -128,19 +127,16 @@ def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): def read_config(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): """ Read the config from specified JSON or YAML file. - Will store the config content in the `self.config` property. + The config content in the `self.config` dictionary. Args: f: filepath of the config file, the content must be a dictionary, if providing a list of files, wil merge the content of them. if providing a dictionary directly, use it as config. - kwargs: other arguments for `json.load` or `yaml.safe_load`, depends on file format. - for more details, please check: - https://docs.python.org/3/library/json.html#json.load. - https://pyyaml.org/wiki/PyYAMLDocumentation. + kwargs: other arguments for ``json.load`` or ``yaml.safe_load``, depends on the file format. """ - content = {self.meta_key: self.config[self.meta_key]} + content = {self.meta_key: self.config.get(self.meta_key)} if isinstance(f, dict): # already loaded in dict content.update(f) diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index c432f146f8..fe86a91a25 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -22,6 +22,7 @@ def run( override: Optional[Union[Dict, str]] = None, target: Optional[str] = None, args_file: Optional[str] = None, + **kwargs, ): """ Specify metadata file and config file to run a regular training or evaluation program. @@ -29,63 +30,65 @@ def run( Typical usage examples: - 1. Execute the `run` API with other CLI tools, take `fire` for example: - `python -m fire monai.bundle run --meta_file= --config_file= --target=trainer` + .. code-block:: bash - 2. Execute this module as CLI entry based on `fire`: - `python -m monai.bundle.scripts run --meta_file= --config_file= --target=trainer` + # Execute this module as CLI entry: + python -m monai.bundle run --meta_file= --config_file= --target=trainer - 3. Override some config values at runtime, set `override` as a dict: - `python -m monai.bundle.scripts run --override={"'net##ndims'": 2} ...` + # Override some config values at runtime, set `override` as a dict: + python -m monai.bundle run --override='{"net##ndims": 2}' ... - 4. Override some config values at runtime, set `override` as a string: - `python -m monai.bundle.scripts run --override="{net##ndims: 2}" ...` + # Override some config values at runtime: + python -m monai.bundle run --"net##input_chns" 1 ... - 5. Override some config values with another config file: - `python -m monai.bundle.scripts run --override={"'net#'": "'/data/other.json'"} ...` + # Override some config values with another config file: + python -m monai.bundle run --override='{"net#": "%/data/other.json"}' ... - 6. Override some config values with part content of another config file: - `python -m monai.bundle.scripts run --override={"'net#'": "'/data/other.json#net_arg'"} ...` + # Override some config values with part content of another config file: + python -m monai.bundle run --override='{"net#": "%/data/other.json#net_arg"}' ... - 7. Set default args of `run` in a JSON / YAML file, help to record and simplify the command line. - Other args still can override the default args at runtime: - `python -m monai.bundle.scripts run --args_file="'/data/args.json'" --config_file=` + # Set default args of `run` in a JSON / YAML file, help to record and simplify the command line. + # Other args still can override the default args at runtime: + python -m monai.bundle run --args_file="'/workspace/data/args.json'" --config_file= Args: - meta_file: filepath of the metadata file, if None, must provide it in `arg_file`. + meta_file: filepath of the metadata file, if `None`, must provide it in `args_file`. if providing a list of files, wil merge the content of them. - config_file: filepath of the config file, if None, must provide it in `arg_file`. + config_file: filepath of the config file, if `None`, must provide it in `args_file`. if providing a list of files, wil merge the content of them. - override: override above config content with specified `id` and `value` pairs. + override: override config content with specified `id` and `value` pairs. it can also be used to provide default value for placeholders. for example: - put a placeholder `"data": "@runtime_value"` in the config, then define - `runtime_value` in `override`. + put a placeholder `"data": "@runtime_value"` in the config, then define `runtime_value` in `override`. it also supports a string representing a dict, like: "{'AA#BB': 123}", usually from command line. target: ID name of the target workflow, it must have the `run` method, follow MONAI `BaseWorkflow`. if None, must provide it in `arg_file`. args_file: to avoid providing same args every time running the program, it supports to put the args as a dictionary in a JSON or YAML file. + kwargs: additional id-value pairs to override the config content. """ - - kwargs = {} - for k, v in {"meta_file": meta_file, "config_file": config_file, "override": override, "target": target}.items(): - if v is not None: - # skip None args - kwargs[k] = v - args = update_default_args(args=args_file, **kwargs) + k_v = zip( + ["meta_file", "config_file", "override", "target", "args_file"], + [meta_file, config_file, override, target, args_file], + ) + input_args = {k: v for k, v in k_v if v is not None} + _args = update_default_args(args=args_file, **input_args) + for k in ("meta_file", "config_file", "target"): + if k not in _args: + raise ValueError(f"{k} is required.") reader = ConfigReader() - reader.read_config(f=args["config_file"]) - reader.read_meta(f=args["meta_file"]) + reader.read_config(f=_args["config_file"]) + reader.read_meta(f=_args["meta_file"]) parser = ConfigParser(reader.get()) - override = args.get("override") - if override is not None: + override = _args.get("override", {}) + override.update(kwargs) + if override: for k, v in override.items(): parser[k] = v # get expected workflow to run - workflow = parser.get_parsed_content(id=args["target"]) + workflow = parser.get_parsed_content(id=_args["target"]) workflow.run() diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 2514df19da..67f207a00e 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -43,15 +43,15 @@ def test_shape(self, config_file, expected_shape): meta = {"datalist": [{"image": filename}], "output_dir": tempdir, "window": (96, 96, 96)} # test YAML file meta_file = os.path.join(tempdir, "meta.yaml") - ConfigReader.export_config_file(config=meta, filepath=meta_file) + ConfigReader.export_config_file(config=meta, filepath=meta_file, fmt="yaml") # test override with file, up case postfix overridefile1 = os.path.join(tempdir, "override1.JSON") with open(overridefile1, "w") as f: # test override with part of the overriding file json.dump({"move_net": "$@network_def.to(@device)"}, f) + os.makedirs(os.path.join(tempdir, "jsons"), exist_ok=True) overridefile2 = os.path.join(tempdir, "jsons/override2.JSON") - os.mkdir(os.path.join(tempdir, "jsons")) with open(overridefile2, "w") as f: # test override with the whole overriding file json.dump("Dataset", f) From 7958161560285ad8453a2baf9c1c37283f44c40f Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 9 Mar 2022 20:22:40 +0000 Subject: [PATCH 60/76] fixes mypy Signed-off-by: Wenqi Li --- monai/bundle/config_reader.py | 4 ++-- monai/bundle/scripts.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index c6d172cd70..5420fd0925 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -95,13 +95,13 @@ def split_path_id(cls, src: str) -> Tuple[str, str]: """ path, *ids = f"{src}".rsplit(ReferenceResolver.sep, 1) - ids = ids[0] if ids else "" path_name = cls.path_match.findall(path) if not path_name: return "", src # the src is a pure id if len(path_name) < 1 and len(path_name[0]) < 1: raise ValueError(f"invalid config file path: {path}") - return path_name[0][0], ids + ids_string: str = ids[0] if ids else "" + return path_name[0][0], ids_string def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): """ diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index fe86a91a25..03452b3966 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -84,8 +84,9 @@ def run( parser = ConfigParser(reader.get()) override = _args.get("override", {}) - override.update(kwargs) - if override: + if isinstance(override, dict): + override.update(kwargs) + if override and isinstance(override, dict): for k, v in override.items(): parser[k] = v From ffb8087d0e27a511fcef9d195f43cfd0896ad698 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 9 Mar 2022 21:58:24 +0000 Subject: [PATCH 61/76] update cli, update according to the comments Signed-off-by: Wenqi Li --- monai/bundle/config_reader.py | 33 ++++++++++++----------- monai/bundle/scripts.py | 49 ++++++++++++++++------------------- tests/test_bundle_run.py | 18 ++++++------- 3 files changed, 50 insertions(+), 50 deletions(-) diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index 5420fd0925..c57ccc5443 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -31,7 +31,7 @@ class ConfigReader: See also: - - https://docs.python.org/3/library/json.html#json.load + - https://docs.python.org/3/library/json.html - https://pyyaml.org/wiki/PyYAMLDocumentation """ @@ -63,6 +63,21 @@ def load_config_file(cls, filepath: PathLike, **kwargs): return yaml.safe_load(f, **kwargs) raise ValueError(f"only support JSON or YAML config file so far, got name {_filepath}.") + @classmethod + def load_config_files(cls, files: Union[PathLike, Sequence[PathLike], dict], **kwargs) -> dict: + """ + Load config files into a single config dict. + + Args: + files: path of target files to load, supported postfixes: `.json`, `.yml`, `.yaml`. + """ + if isinstance(files, dict): # already a config dict + return files + content = {} + for i in ensure_tuple(files): + content.update(cls.load_config_file(i, **kwargs)) + return content + @classmethod def export_config_file(cls, config: Dict, filepath: PathLike, fmt="json", **kwargs): """ @@ -115,14 +130,7 @@ def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): kwargs: other arguments for ``json.load`` or ``yaml.safe_load``, depends on the file format. """ - if isinstance(f, dict): - # already loaded in dict - content = f - else: - content = {} - for i in ensure_tuple(f): - content.update(self.load_config_file(i, **kwargs)) - self.config[self.meta_key] = content + self.config[self.meta_key] = self.load_config_files(f, **kwargs) def read_config(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): """ @@ -137,12 +145,7 @@ def read_config(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): """ content = {self.meta_key: self.config.get(self.meta_key)} - if isinstance(f, dict): - # already loaded in dict - content.update(f) - else: - for i in ensure_tuple(f): - content.update(self.load_config_file(i, **kwargs)) + content.update(self.load_config_files(f, **kwargs)) self.config = content def get(self): diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index 03452b3966..7e473f6d4b 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -25,26 +25,25 @@ def run( **kwargs, ): """ - Specify metadata file and config file to run a regular training or evaluation program. - It's used to execute most of the supervised training, evaluation or inference cases. + Specify `meta_file` and `config_file` to run monai bundle components and workflows. Typical usage examples: .. code-block:: bash - # Execute this module as CLI entry: + # Execute this module as a CLI entry: python -m monai.bundle run --meta_file= --config_file= --target=trainer - # Override some config values at runtime, set `override` as a dict: + # Override config values at runtime, set `override` as a dictionary: python -m monai.bundle run --override='{"net##ndims": 2}' ... - # Override some config values at runtime: + # Override config values at runtime by specifying the component id and its new value: python -m monai.bundle run --"net##input_chns" 1 ... - # Override some config values with another config file: - python -m monai.bundle run --override='{"net#": "%/data/other.json"}' ... + # Override config values with another config file `/path/to/another.json`: + python -m monai.bundle run --override='{"net#": "%/path/to/another.json"}' ... - # Override some config values with part content of another config file: + # Override config values with part content of another config file: python -m monai.bundle run --override='{"net#": "%/data/other.json#net_arg"}' ... # Set default args of `run` in a JSON / YAML file, help to record and simplify the command line. @@ -52,19 +51,16 @@ def run( python -m monai.bundle run --args_file="'/workspace/data/args.json'" --config_file= Args: - meta_file: filepath of the metadata file, if `None`, must provide it in `args_file`. - if providing a list of files, wil merge the content of them. - config_file: filepath of the config file, if `None`, must provide it in `args_file`. - if providing a list of files, wil merge the content of them. - override: override config content with specified `id` and `value` pairs. - it can also be used to provide default value for placeholders. for example: - put a placeholder `"data": "@runtime_value"` in the config, then define `runtime_value` in `override`. - it also supports a string representing a dict, like: "{'AA#BB': 123}", usually from command line. - target: ID name of the target workflow, it must have the `run` method, follow MONAI `BaseWorkflow`. - if None, must provide it in `arg_file`. - args_file: to avoid providing same args every time running the program, it supports - to put the args as a dictionary in a JSON or YAML file. - kwargs: additional id-value pairs to override the config content. + meta_file: filepath of the metadata file, if `None`, must be provided in `args_file`. + if it is a list of file paths, the content of them will be merged. + config_file: filepath of the config file, if `None`, must be provided in `args_file`. + if it is a list of file paths, the content of them will be merged. + override: values to override in the config content provided in `meta_file`. + target: ID name of the target component or workflow, it must have a `run` method. + args_file: a JSON or YAML file to provide default values for `meta_file`, `config_file`, `target`. + so that the command line inputs can be simplified. + kwargs: additional id-value pairs to override the corresponding config content by id. + e.g. ``--"net##input_chns" 42``. """ k_v = zip( @@ -73,15 +69,15 @@ def run( ) input_args = {k: v for k, v in k_v if v is not None} _args = update_default_args(args=args_file, **input_args) - for k in ("meta_file", "config_file", "target"): + for k in ("meta_file", "config_file"): if k not in _args: - raise ValueError(f"{k} is required.") + raise ValueError(f"{k} is required for 'monai.bundle run'.\n{run.__doc__}") reader = ConfigReader() reader.read_config(f=_args["config_file"]) reader.read_meta(f=_args["meta_file"]) - parser = ConfigParser(reader.get()) + parser = ConfigParser(config=reader.get()) override = _args.get("override", {}) if isinstance(override, dict): @@ -90,6 +86,7 @@ def run( for k, v in override.items(): parser[k] = v - # get expected workflow to run - workflow = parser.get_parsed_content(id=_args["target"]) + workflow = parser.get_parsed_content(id=_args.get("target", "")) + if not hasattr(workflow, "run"): + raise ValueError(f"The parsed workflow {type(workflow)} does not have a `run` method.\n{run.__doc__}") workflow.run() diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 67f207a00e..37f6f78814 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -31,13 +31,13 @@ class TestBundleRun(unittest.TestCase): @parameterized.expand([TEST_CASE_1, TEST_CASE_2]) def test_shape(self, config_file, expected_shape): - test_image = np.random.rand(128, 128, 128) + test_image = np.random.rand(*expected_shape) with tempfile.TemporaryDirectory() as tempdir: filename = os.path.join(tempdir, "image.nii") nib.save(nib.Nifti1Image(test_image, np.eye(4)), filename) # generate default args in a JSON file - def_args = {"config_file": "will be overrided by `config_file` arg"} + def_args = {"config_file": "will be replaced by `config_file` arg"} ConfigReader.export_config_file(config=def_args, filepath=os.path.join(tempdir, "def_args.json")) meta = {"datalist": [{"image": filename}], "output_dir": tempdir, "window": (96, 96, 96)} @@ -63,18 +63,18 @@ def test_shape(self, config_file, expected_shape): else: override = f"'network':'%{overridefile1}#move_net','dataset#':'%{overridefile2}'" # test with `monai.bundle` as CLI entry directly - cmd = f"{sys.executable} -m monai.bundle run --meta_file {meta_file} --config_file {config_file}" + cmd = "-m monai.bundle run --target evaluator" cmd += f" --override {{'postprocessing##transforms#2##output_postfix':'seg',{override}}}" - cmd += " --target evaluator" - ret = subprocess.check_call(cmd.split(" ")) + la = [f"{sys.executable}"] + cmd.split(" ") + ["--meta_file", meta_file] + ["--config_file", config_file] + ret = subprocess.check_call(la) self.assertEqual(ret, 0) self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_seg.nii.gz")).shape, expected_shape) # here test the script with `google fire` tool as CLI - cmd = f"{sys.executable} -m fire monai.bundle.scripts run --meta_file {meta_file}" - cmd += f" --config_file {config_file} --override {{'evaluator##amp':False,{override}}}" - cmd += " --target evaluator" - ret = subprocess.check_call(cmd.split(" ")) + cmd = "-m fire monai.bundle.scripts run --target evaluator" + cmd += f" --override {{'evaluator##amp':False,{override}}}" + la = [f"{sys.executable}"] + cmd.split(" ") + ["--meta_file", meta_file] + ["--config_file", config_file] + ret = subprocess.check_call(la) self.assertEqual(ret, 0) self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_trans.nii.gz")).shape, expected_shape) From 5cd54259f531df436fd1ab797aa731488ddfbbc3 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 10 Mar 2022 11:38:19 +0800 Subject: [PATCH 62/76] [DLMED] fix typo in doc-string Signed-off-by: Nic Ma --- monai/bundle/config_parser.py | 2 +- monai/bundle/config_reader.py | 1 + monai/bundle/scripts.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/monai/bundle/config_parser.py b/monai/bundle/config_parser.py index 86a830990e..9a9d274adf 100644 --- a/monai/bundle/config_parser.py +++ b/monai/bundle/config_parser.py @@ -171,7 +171,7 @@ def _do_resolve(self, config: Any): """ Recursively resolve the config content to replace the macro tokens with target content. The macro tokens start with "%", can be from another structured file, like: - ``{"net": "%default_net"}``, ``{"net": "%/data/config.json#net#"}}``. + ``{"net": "%default_net"}``, ``{"net": "%/data/config.json#net#"}``. Args: config: input config file to resolve. diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index c57ccc5443..d4112b41dc 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -70,6 +70,7 @@ def load_config_files(cls, files: Union[PathLike, Sequence[PathLike], dict], **k Args: files: path of target files to load, supported postfixes: `.json`, `.yml`, `.yaml`. + kwargs: other arguments for ``json.load`` or ```yaml.safe_load``, depends on the file format. """ if isinstance(files, dict): # already a config dict return files diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index 7e473f6d4b..567f55b842 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -19,7 +19,7 @@ def run( meta_file: Optional[Union[str, Sequence[str]]] = None, config_file: Optional[Union[str, Sequence[str]]] = None, - override: Optional[Union[Dict, str]] = None, + override: Optional[Dict] = None, target: Optional[str] = None, args_file: Optional[str] = None, **kwargs, From d8beecac114ec80e5add33b63b5449acb158278a Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 10 Mar 2022 17:55:53 +0800 Subject: [PATCH 63/76] [DLMED] update according to discussion Signed-off-by: Nic Ma --- monai/bundle/config_reader.py | 4 +-- monai/bundle/scripts.py | 53 +++++++++++++++-------------------- tests/test_bundle_run.py | 17 +++++------ 3 files changed, 32 insertions(+), 42 deletions(-) diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index d4112b41dc..75e45c9a13 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -110,12 +110,10 @@ def split_path_id(cls, src: str) -> Tuple[str, str]: src: source string to split. """ - path, *ids = f"{src}".rsplit(ReferenceResolver.sep, 1) + path, *ids = src.rsplit(ReferenceResolver.sep, 1) path_name = cls.path_match.findall(path) if not path_name: return "", src # the src is a pure id - if len(path_name) < 1 and len(path_name[0]) < 1: - raise ValueError(f"invalid config file path: {path}") ids_string: str = ids[0] if ids else "" return path_name[0][0], ids_string diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index 567f55b842..5b25de982d 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -19,10 +19,9 @@ def run( meta_file: Optional[Union[str, Sequence[str]]] = None, config_file: Optional[Union[str, Sequence[str]]] = None, - override: Optional[Dict] = None, - target: Optional[str] = None, + target_id: Optional[str] = None, args_file: Optional[str] = None, - **kwargs, + **override, ): """ Specify `meta_file` and `config_file` to run monai bundle components and workflows. @@ -32,19 +31,16 @@ def run( .. code-block:: bash # Execute this module as a CLI entry: - python -m monai.bundle run --meta_file= --config_file= --target=trainer - - # Override config values at runtime, set `override` as a dictionary: - python -m monai.bundle run --override='{"net##ndims": 2}' ... + python -m monai.bundle run --meta_file= --config_file= --target_id=trainer # Override config values at runtime by specifying the component id and its new value: - python -m monai.bundle run --"net##input_chns" 1 ... + python -m monai.bundle run --"'net##input_chns'"=1 ... # Override config values with another config file `/path/to/another.json`: - python -m monai.bundle run --override='{"net#": "%/path/to/another.json"}' ... + python -m monai.bundle run --"'net#'"="'%/path/to/another.json'" ... # Override config values with part content of another config file: - python -m monai.bundle run --override='{"net#": "%/data/other.json#net_arg"}' ... + python -m monai.bundle run --"'net#'"="'%/data/other.json#net_arg'" ... # Set default args of `run` in a JSON / YAML file, help to record and simplify the command line. # Other args still can override the default args at runtime: @@ -55,38 +51,33 @@ def run( if it is a list of file paths, the content of them will be merged. config_file: filepath of the config file, if `None`, must be provided in `args_file`. if it is a list of file paths, the content of them will be merged. - override: values to override in the config content provided in `meta_file`. - target: ID name of the target component or workflow, it must have a `run` method. - args_file: a JSON or YAML file to provide default values for `meta_file`, `config_file`, `target`. - so that the command line inputs can be simplified. - kwargs: additional id-value pairs to override the corresponding config content by id. + target_id: ID name of the target component or workflow, it must have a `run` method. + args_file: a JSON or YAML file to provide default values for `meta_file`, `config_file`, + `target_id` and override pairs. so that the command line inputs can be simplified. + override: id-value pairs to override or add the corresponding config content. e.g. ``--"net##input_chns" 42``. """ - k_v = zip( - ["meta_file", "config_file", "override", "target", "args_file"], - [meta_file, config_file, override, target, args_file], - ) - input_args = {k: v for k, v in k_v if v is not None} - _args = update_default_args(args=args_file, **input_args) + k_v = zip(["meta_file", "config_file", "target_id"], [meta_file, config_file, target_id]) + for k, v in k_v: + if v is not None: + override[k] = v + _args = update_default_args(args=args_file, **override) for k in ("meta_file", "config_file"): if k not in _args: raise ValueError(f"{k} is required for 'monai.bundle run'.\n{run.__doc__}") reader = ConfigReader() - reader.read_config(f=_args["config_file"]) - reader.read_meta(f=_args["meta_file"]) + reader.read_config(f=_args.pop("config_file")) + reader.read_meta(f=_args.pop("meta_file")) + id = _args.pop("target_id", "") parser = ConfigParser(config=reader.get()) + # the rest key-values in the args are to override config content + for k, v in _args.items(): + parser[k] = v - override = _args.get("override", {}) - if isinstance(override, dict): - override.update(kwargs) - if override and isinstance(override, dict): - for k, v in override.items(): - parser[k] = v - - workflow = parser.get_parsed_content(id=_args.get("target", "")) + workflow = parser.get_parsed_content(id=id) if not hasattr(workflow, "run"): raise ValueError(f"The parsed workflow {type(workflow)} does not have a `run` method.\n{run.__doc__}") workflow.run() diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 37f6f78814..f25c4e256e 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -38,7 +38,8 @@ def test_shape(self, config_file, expected_shape): # generate default args in a JSON file def_args = {"config_file": "will be replaced by `config_file` arg"} - ConfigReader.export_config_file(config=def_args, filepath=os.path.join(tempdir, "def_args.json")) + def_args_file = os.path.join(tempdir, "def_args.json") + ConfigReader.export_config_file(config=def_args, filepath=def_args_file) meta = {"datalist": [{"image": filename}], "output_dir": tempdir, "window": (96, 96, 96)} # test YAML file @@ -59,20 +60,20 @@ def test_shape(self, config_file, expected_shape): saver = LoadImage(image_only=True) if sys.platform == "win32": - override = "'network':'$@network_def.to(@device)','dataset#':'Dataset'" + override = "--network $@network_def.to(@device) --dataset# Dataset" else: - override = f"'network':'%{overridefile1}#move_net','dataset#':'%{overridefile2}'" + override = f"--network %{overridefile1}#move_net --dataset# %{overridefile2}" # test with `monai.bundle` as CLI entry directly - cmd = "-m monai.bundle run --target evaluator" - cmd += f" --override {{'postprocessing##transforms#2##output_postfix':'seg',{override}}}" + cmd = "-m monai.bundle run --target_id evaluator" + cmd += f" --postprocessing##transforms#2##output_postfix seg {override}" la = [f"{sys.executable}"] + cmd.split(" ") + ["--meta_file", meta_file] + ["--config_file", config_file] - ret = subprocess.check_call(la) + ret = subprocess.check_call(la + ["--args_file", def_args_file]) self.assertEqual(ret, 0) self.assertTupleEqual(saver(os.path.join(tempdir, "image", "image_seg.nii.gz")).shape, expected_shape) # here test the script with `google fire` tool as CLI - cmd = "-m fire monai.bundle.scripts run --target evaluator" - cmd += f" --override {{'evaluator##amp':False,{override}}}" + cmd = "-m fire monai.bundle.scripts run --target_id evaluator" + cmd += f" --evaluator##amp False {override}" la = [f"{sys.executable}"] + cmd.split(" ") + ["--meta_file", meta_file] + ["--config_file", config_file] ret = subprocess.check_call(la) self.assertEqual(ret, 0) From 81fabd33a025ca7529e44d881c2a216e27425eca Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 10 Mar 2022 19:23:07 +0800 Subject: [PATCH 64/76] [DLMED] fix path_id match Signed-off-by: Nic Ma --- monai/bundle/config_reader.py | 18 ++++++++++-------- monai/bundle/scripts.py | 12 ++++++------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index 75e45c9a13..98f422ea0b 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -14,7 +14,6 @@ from pathlib import Path from typing import Dict, Sequence, Tuple, Union -from monai.bundle.reference_resolver import ReferenceResolver from monai.config import PathLike from monai.utils import ensure_tuple, look_up_option, optional_import @@ -37,8 +36,10 @@ class ConfigReader: """ suffixes = ("json", "yaml", "yml") - path_match = re.compile(rf"(.*\.({'|'.join(suffixes)})$)", re.IGNORECASE) + suffix_match = rf"\.({'|'.join(suffixes)})" + path_match = rf"(.*{suffix_match}$)" meta_key = "" # field key to save metadata + sep = "#" # separator for file path and the id of content in the file def __init__(self): self.config: Dict = {self.meta_key: {}} @@ -54,7 +55,7 @@ def load_config_file(cls, filepath: PathLike, **kwargs): """ _filepath: str = str(Path(filepath)) - if not cls.path_match.findall(_filepath): + if not re.compile(cls.path_match, re.IGNORECASE).findall(_filepath): raise ValueError(f'unknown file input: "{filepath}"') with open(_filepath) as f: if _filepath.lower().endswith(cls.suffixes[0]): @@ -105,17 +106,18 @@ def split_path_id(cls, src: str) -> Tuple[str, str]: """ Split `src` string into two parts: a config file path and component id. The file path should end with `(json|yaml|yml)`. The component id should be separated by `#` if it exists. + If no path or no id, return "". Args: src: source string to split. """ - path, *ids = src.rsplit(ReferenceResolver.sep, 1) - path_name = cls.path_match.findall(path) - if not path_name: + result = re.compile(cls.suffix_match, re.IGNORECASE).findall(src) + if len(result) != 1: return "", src # the src is a pure id - ids_string: str = ids[0] if ids else "" - return path_name[0][0], ids_string + items = src.split(result[0]) + # return file path and the + return items[0] + result[0], items[1][len(cls.sep) :] if items[1] != "" else "" def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): """ diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index 5b25de982d..820726fb46 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -9,7 +9,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, Optional, Sequence, Union +from typing import Optional, Sequence, Union from monai.bundle.config_parser import ConfigParser from monai.bundle.config_reader import ConfigReader @@ -31,20 +31,20 @@ def run( .. code-block:: bash # Execute this module as a CLI entry: - python -m monai.bundle run --meta_file= --config_file= --target_id=trainer + python -m monai.bundle run --meta_file --config_file --target_id trainer # Override config values at runtime by specifying the component id and its new value: - python -m monai.bundle run --"'net##input_chns'"=1 ... + python -m monai.bundle run --"net##input_chns" 1 ... # Override config values with another config file `/path/to/another.json`: - python -m monai.bundle run --"'net#'"="'%/path/to/another.json'" ... + python -m monai.bundle run --"net#" "%/path/to/another.json" ... # Override config values with part content of another config file: - python -m monai.bundle run --"'net#'"="'%/data/other.json#net_arg'" ... + python -m monai.bundle run --"net#" "%/data/other.json#net_arg" ... # Set default args of `run` in a JSON / YAML file, help to record and simplify the command line. # Other args still can override the default args at runtime: - python -m monai.bundle run --args_file="'/workspace/data/args.json'" --config_file= + python -m monai.bundle run --args_file "/workspace/data/args.json" --config_file Args: meta_file: filepath of the metadata file, if `None`, must be provided in `args_file`. From c89e3a2926c826b559decd093498417eb7759ef3 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 10 Mar 2022 12:34:54 +0000 Subject: [PATCH 65/76] revise according to the comments Signed-off-by: Wenqi Li --- monai/bundle/config_reader.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index 98f422ea0b..4be7bf0fa8 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -112,12 +112,12 @@ def split_path_id(cls, src: str) -> Tuple[str, str]: src: source string to split. """ - result = re.compile(cls.suffix_match, re.IGNORECASE).findall(src) - if len(result) != 1: + result = re.compile(rf"(.*{cls.suffix_match}(?=(?:{cls.sep}.*)|$))", re.IGNORECASE).findall(src) + if not result: return "", src # the src is a pure id - items = src.split(result[0]) - # return file path and the - return items[0] + result[0], items[1][len(cls.sep) :] if items[1] != "" else "" + path_name = result[0][0] # at most one path_name + _, ids = src.rsplit(path_name, 1) + return path_name, ids[len(cls.sep) :] if ids.startswith(cls.sep) else "" def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): """ From ca166815f9b7ecd60852809aae69180ff2c337a8 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 10 Mar 2022 22:15:25 +0800 Subject: [PATCH 66/76] [DLMED] update config item Signed-off-by: Nic Ma --- monai/bundle/config_item.py | 43 +++++++++++++++++++------------------ tests/test_config_item.py | 28 ++++++++++++------------ 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/monai/bundle/config_item.py b/monai/bundle/config_item.py index b6334ee9d5..9392f294f5 100644 --- a/monai/bundle/config_item.py +++ b/monai/bundle/config_item.py @@ -163,22 +163,21 @@ class ConfigComponent(ConfigItem, Instantiable): Subclass of :py:class:`monai.bundle.ConfigItem`, this class uses a dictionary with string keys to represent a component of `class` or `function` and supports instantiation. - Currently, four special keys (strings surrounded by ``<>``) are defined and interpreted beyond the regular literals: + Currently, four special keys (strings surrounded by ``_``) are defined and interpreted beyond the regular literals: - class or function identifier of the python module, specified by one of the two keys. - - ``""``: indicates build-in python classes or functions such as "LoadImageDict". - - ``""``: full module name, such as "monai.transforms.LoadImageDict". - - ``""``: input arguments to the python module. - - ``""``: a flag to indicate whether to skip the instantiation. + - ``"_name_"``: indicates build-in python classes or functions such as "LoadImageDict". + - ``"_path_"``: full module name, such as "monai.transforms.LoadImageDict". + - ``"_disabled_"``: a flag to indicate whether to skip the instantiation. + + Other fields in the config content are input arguments to the python module. .. code-block:: python locator = ComponentLocator(excludes=["modules_to_exclude"]) config = { - "": "LoadImaged", - "": { - "keys": ["image", "label"] - } + "_name_": "LoadImaged", + "keys": ["image", "label"] } configer = ConfigComponent(config, id="test", locator=locator) @@ -195,6 +194,8 @@ class ConfigComponent(ConfigItem, Instantiable): """ + not_arg_keys = ["_name_", "_path_", "_disabled_"] + def __init__( self, config: Any, @@ -214,27 +215,27 @@ def is_instantiable(config: Any) -> bool: config: input config content to check. """ - return isinstance(config, Mapping) and ("" in config or "" in config) + return isinstance(config, Mapping) and ("_path_" in config or "_name_" in config) def resolve_module_name(self): """ Resolve the target module name from current config content. - The config content must have ``""`` or ``""``. - When both are specified, ``""`` will be used. + The config content must have ``"_path_"`` or ``"_name_"`` key. + When both are specified, ``"_path_"`` will be used. """ config = dict(self.get_config()) - path = config.get("") + path = config.get("_path_") if path is not None: if not isinstance(path, str): - raise ValueError(f"'' must be a string, but got: {path}.") - if "" in config: - warnings.warn(f"both '' and '', default to use '': {path}.") + raise ValueError(f"'_path_' must be a string, but got: {path}.") + if "_name_" in config: + warnings.warn(f"both '_path_' and '_name_', default to use '_path_': {path}.") return path - name = config.get("") + name = config.get("_name_") if not isinstance(name, str): - raise ValueError("must provide a string for `` or `` of target component to instantiate.") + raise ValueError("must provide a string for `_path_` or `_name_` of target component to instantiate.") module = self.locator.get_component_module_name(name) if module is None: @@ -242,7 +243,7 @@ def resolve_module_name(self): if isinstance(module, list): warnings.warn( f"there are more than 1 component have name `{name}`: {module}, use the first one `{module[0]}." - f" if want to use others, please set its module path in `` directly." + f" if want to use others, please set its module path in `_path_` directly." ) module = module[0] return f"{module}.{name}" @@ -252,14 +253,14 @@ def resolve_args(self): Utility function used in `instantiate()` to resolve the arguments from current config content. """ - return self.get_config().get("", {}) + return {k: v for k, v in self.get_config().items() if k not in self.not_arg_keys} def is_disabled(self) -> bool: # type: ignore """ Utility function used in `instantiate()` to check whether to skip the instantiation. """ - _is_disabled = self.get_config().get("", False) + _is_disabled = self.get_config().get("_disabled_", False) return _is_disabled.lower().strip() == "true" if isinstance(_is_disabled, str) else bool(_is_disabled) def instantiate(self, **kwargs) -> object: # type: ignore diff --git a/tests/test_config_item.py b/tests/test_config_item.py index 1284efab56..9ce08561f3 100644 --- a/tests/test_config_item.py +++ b/tests/test_config_item.py @@ -26,22 +26,22 @@ TEST_CASE_1 = [{"lr": 0.001}, 0.0001] -TEST_CASE_2 = [{"": "LoadImaged", "": {"keys": ["image"]}}, LoadImaged] -# test python `` -TEST_CASE_3 = [{"": "monai.transforms.LoadImaged", "": {"keys": ["image"]}}, LoadImaged] -# test `` -TEST_CASE_4 = [{"": "LoadImaged", "": True, "": {"keys": ["image"]}}, dict] -# test `` -TEST_CASE_5 = [{"": "LoadImaged", "": "true", "": {"keys": ["image"]}}, dict] +TEST_CASE_2 = [{"_name_": "LoadImaged", "keys": ["image"]}, LoadImaged] +# test python `_path_` +TEST_CASE_3 = [{"_path_": "monai.transforms.LoadImaged", "keys": ["image"]}, LoadImaged] +# test `_disabled_` +TEST_CASE_4 = [{"_name_": "LoadImaged", "_disabled_": True, "keys": ["image"]}, dict] +# test `_disabled_` with string +TEST_CASE_5 = [{"_name_": "LoadImaged", "_disabled_": "true", "keys": ["image"]}, dict] # test non-monai modules and excludes TEST_CASE_6 = [ - {"": "torch.optim.Adam", "": {"params": torch.nn.PReLU().parameters(), "lr": 1e-4}}, + {"_path_": "torch.optim.Adam", "params": torch.nn.PReLU().parameters(), "lr": 1e-4}, torch.optim.Adam, ] -TEST_CASE_7 = [{"": "decollate_batch", "": {"detach": True, "pad": True}}, partial] +TEST_CASE_7 = [{"_name_": "decollate_batch", "detach": True, "pad": True}, partial] # test args contains "name" field TEST_CASE_8 = [ - {"": "RandTorchVisiond", "": {"keys": "image", "name": "ColorJitter", "brightness": 0.25}}, + {"_name_": "RandTorchVisiond", "keys": "image", "name": "ColorJitter", "brightness": 0.25}, RandTorchVisiond, ] # test execute some function in args, test pre-imported global packages `monai` @@ -67,8 +67,8 @@ def test_component(self, test_input, output_type): locator = ComponentLocator(excludes=["metrics"]) configer = ConfigComponent(id="test", config=test_input, locator=locator) ret = configer.instantiate() - if test_input.get("", False): - # test `` works fine + if test_input.get("_disabled_", False): + # test `_disabled_` works fine self.assertEqual(ret, None) return self.assertTrue(isinstance(ret, output_type)) @@ -83,11 +83,11 @@ def test_expression(self, id, test_input): self.assertTrue(isinstance(ret, Callable)) def test_lazy_instantiation(self): - config = {"": "DataLoader", "": {"dataset": Dataset(data=[1, 2]), "batch_size": 2}} + config = {"_name_": "DataLoader", "dataset": Dataset(data=[1, 2]), "batch_size": 2} configer = ConfigComponent(config=config, locator=None) init_config = configer.get_config() # modify config content at runtime - init_config[""]["batch_size"] = 4 + init_config["batch_size"] = 4 configer.update_config(config=init_config) ret = configer.instantiate() From f9cb68aa9e0ecb0a3fb829d48bdb1a9d141a1165 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 10 Mar 2022 23:18:45 +0800 Subject: [PATCH 67/76] [DLMED] update patterns Signed-off-by: Nic Ma --- monai/bundle/config_parser.py | 26 ++-- monai/bundle/config_reader.py | 4 +- monai/bundle/reference_resolver.py | 4 +- monai/bundle/scripts.py | 8 +- tests/test_bundle_run.py | 8 +- tests/test_config_item.py | 5 +- tests/test_config_parser.py | 49 ++++---- tests/test_reference_resolver.py | 48 +++----- tests/testing_data/inference.json | 188 ++++++++++++----------------- tests/testing_data/inference.yaml | 136 ++++++++++----------- 10 files changed, 208 insertions(+), 268 deletions(-) diff --git a/monai/bundle/config_parser.py b/monai/bundle/config_parser.py index 9a9d274adf..70993a71a5 100644 --- a/monai/bundle/config_parser.py +++ b/monai/bundle/config_parser.py @@ -38,19 +38,17 @@ class ConfigParser: config = { "my_dims": 2, "dims_1": "$@my_dims + 1", - "my_xform": {"": "LoadImage"}, - "my_net": {"": "BasicUNet", - "": {"spatial_dims": "@dims_1", "in_channels": 1, "out_channels": 4}}, - "trainer": {"": "SupervisedTrainer", - "": {"network": "@my_net", "preprocessing": "@my_xform"}} + "my_xform": {"_name_": "LoadImage"}, + "my_net": {"_name_": "BasicUNet", "spatial_dims": "@dims_1", "in_channels": 1, "out_channels": 4}, + "trainer": {"_name_": "SupervisedTrainer", "network": "@my_net", "preprocessing": "@my_xform"} } # in the example $@my_dims + 1 is an expression, which adds 1 to the value of @my_dims parser = ConfigParser(config) # get/set configuration content, the set method should happen before calling parse() - print(parser["my_net"][""]["in_channels"]) # original input channels 1 - parser["my_net"][""]["in_channels"] = 4 # change input channels to 4 - print(parser["my_net"][""]["in_channels"]) + print(parser["my_net"]["in_channels"]) # original input channels 1 + parser["my_net"]["in_channels"] = 4 # change input channels to 4 + print(parser["my_net"]["in_channels"]) # instantiate the network component parser.parse(True) @@ -107,7 +105,7 @@ def __getitem__(self, id: Union[str, int]): id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to go one level further into the nested structures. Use digits indexing from "0" for list or other strings for dict. - For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. + For example: ``"xform#5"``, ``"net#channels"``. ``""`` indicates the entire ``self.config``. """ if id == "": @@ -129,7 +127,7 @@ def __setitem__(self, id: Union[str, int], config: Any): id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to go one level further into the nested structures. Use digits indexing from "0" for list or other strings for dict. - For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. + For example: ``"xform#5"``, ``"net#channels"``. ``""`` indicates the entire ``self.config``. config: config to set at location ``id``. """ @@ -171,7 +169,7 @@ def _do_resolve(self, config: Any): """ Recursively resolve the config content to replace the macro tokens with target content. The macro tokens start with "%", can be from another structured file, like: - ``{"net": "%default_net"}``, ``{"net": "%/data/config.json#net#"}``. + ``{"net": "%default_net"}``, ``{"net": "%/data/config.json#net"}``. Args: config: input config file to resolve. @@ -190,7 +188,7 @@ def resolve_macro(self): """ Recursively resolve `self.config` to replace the macro tokens with target content. The macro tokens are marked as starting with "%", can be from another structured file, like: - ``"%default_net"``, ``"%/data/config.json#net#"``. + ``"%default_net"``, ``"%/data/config.json#net"``. """ self.set(self._do_resolve(config=deepcopy(self.get()))) @@ -204,7 +202,7 @@ def _do_parse(self, config, id: str = ""): id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to go one level further into the nested structures. Use digits indexing from "0" for list or other strings for dict. - For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. + For example: ``"xform#5"``, ``"net#channels"``. ``""`` indicates the entire ``self.config``. """ if isinstance(config, (dict, list)): @@ -248,7 +246,7 @@ def get_parsed_content(self, id: str = "", **kwargs): id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to go one level further into the nested structures. Use digits indexing from "0" for list or other strings for dict. - For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. + For example: ``"xform#5"``, ``"net#channels"``. ``""`` indicates the entire ``self.config``. kwargs: additional keyword arguments to be passed to ``_resolve_one_item``. Currently support ``reset`` (for parse), ``instantiate`` and ``eval_expr``. All defaulting to True. diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index 4be7bf0fa8..170883c616 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -38,7 +38,7 @@ class ConfigReader: suffixes = ("json", "yaml", "yml") suffix_match = rf"\.({'|'.join(suffixes)})" path_match = rf"(.*{suffix_match}$)" - meta_key = "" # field key to save metadata + meta_key = "_meta_" # field key to save metadata sep = "#" # separator for file path and the id of content in the file def __init__(self): @@ -122,7 +122,7 @@ def split_path_id(cls, src: str) -> Tuple[str, str]: def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): """ Read the metadata from specified JSON or YAML file. - The metadata as a dictionary will be stored at ``self.config[""]``. + The metadata as a dictionary will be stored at ``self.config["_meta_"]``. Args: f: filepath of the metadata file, the content must be a dictionary, diff --git a/monai/bundle/reference_resolver.py b/monai/bundle/reference_resolver.py index b3834bf76c..8a8b5ed89d 100644 --- a/monai/bundle/reference_resolver.py +++ b/monai/bundle/reference_resolver.py @@ -47,8 +47,8 @@ class ReferenceResolver: _vars = "__local_refs" sep = "#" # separator for key indexing ref = "@" # reference prefix - # match a reference string, e.g. "@id#key", "@id#key#0", "@##key" - id_matcher = re.compile(rf"{ref}(?:(?:<\w*>)|(?:\w*))(?:(?:{sep}<\w*>)|(?:{sep}\w*))*") + # match a reference string, e.g. "@id#key", "@id#key#0", "@_name_#key" + id_matcher = re.compile(rf"{ref}(?:\w*)(?:{sep}\w*)*") def __init__(self, items: Optional[Sequence[ConfigItem]] = None): # save the items in a dictionary with the `ConfigItem.id` as key diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index 820726fb46..a5749c1e1e 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -34,13 +34,13 @@ def run( python -m monai.bundle run --meta_file --config_file --target_id trainer # Override config values at runtime by specifying the component id and its new value: - python -m monai.bundle run --"net##input_chns" 1 ... + python -m monai.bundle run --net#input_chns 1 ... # Override config values with another config file `/path/to/another.json`: - python -m monai.bundle run --"net#" "%/path/to/another.json" ... + python -m monai.bundle run --net %/path/to/another.json ... # Override config values with part content of another config file: - python -m monai.bundle run --"net#" "%/data/other.json#net_arg" ... + python -m monai.bundle run --net %/data/other.json#net_arg ... # Set default args of `run` in a JSON / YAML file, help to record and simplify the command line. # Other args still can override the default args at runtime: @@ -55,7 +55,7 @@ def run( args_file: a JSON or YAML file to provide default values for `meta_file`, `config_file`, `target_id` and override pairs. so that the command line inputs can be simplified. override: id-value pairs to override or add the corresponding config content. - e.g. ``--"net##input_chns" 42``. + e.g. ``--net#input_chns 42``. """ k_v = zip(["meta_file", "config_file", "target_id"], [meta_file, config_file, target_id]) diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index f25c4e256e..d6c41271f5 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -60,12 +60,12 @@ def test_shape(self, config_file, expected_shape): saver = LoadImage(image_only=True) if sys.platform == "win32": - override = "--network $@network_def.to(@device) --dataset# Dataset" + override = "--network $@network_def.to(@device) --dataset#_name_ Dataset" else: - override = f"--network %{overridefile1}#move_net --dataset# %{overridefile2}" + override = f"--network %{overridefile1}#move_net --dataset#_name_ %{overridefile2}" # test with `monai.bundle` as CLI entry directly cmd = "-m monai.bundle run --target_id evaluator" - cmd += f" --postprocessing##transforms#2##output_postfix seg {override}" + cmd += f" --postprocessing#transforms#2#output_postfix seg {override}" la = [f"{sys.executable}"] + cmd.split(" ") + ["--meta_file", meta_file] + ["--config_file", config_file] ret = subprocess.check_call(la + ["--args_file", def_args_file]) self.assertEqual(ret, 0) @@ -73,7 +73,7 @@ def test_shape(self, config_file, expected_shape): # here test the script with `google fire` tool as CLI cmd = "-m fire monai.bundle.scripts run --target_id evaluator" - cmd += f" --evaluator##amp False {override}" + cmd += f" --evaluator#amp False {override}" la = [f"{sys.executable}"] + cmd.split(" ") + ["--meta_file", meta_file] + ["--config_file", config_file] ret = subprocess.check_call(la) self.assertEqual(ret, 0) diff --git a/tests/test_config_item.py b/tests/test_config_item.py index 9ce08561f3..507b5a7d92 100644 --- a/tests/test_config_item.py +++ b/tests/test_config_item.py @@ -34,10 +34,7 @@ # test `_disabled_` with string TEST_CASE_5 = [{"_name_": "LoadImaged", "_disabled_": "true", "keys": ["image"]}, dict] # test non-monai modules and excludes -TEST_CASE_6 = [ - {"_path_": "torch.optim.Adam", "params": torch.nn.PReLU().parameters(), "lr": 1e-4}, - torch.optim.Adam, -] +TEST_CASE_6 = [{"_path_": "torch.optim.Adam", "params": torch.nn.PReLU().parameters(), "lr": 1e-4}, torch.optim.Adam] TEST_CASE_7 = [{"_name_": "decollate_batch", "detach": True, "pad": True}, partial] # test args contains "name" field TEST_CASE_8 = [ diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 5b5aa2b816..c57ca48cbd 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -25,24 +25,21 @@ TEST_CASE_1 = [ { "transform": { - "": "Compose", - "": { - "transforms": [ - {"": "LoadImaged", "": {"keys": "image"}}, - { - "": "RandTorchVisiond", - "": {"keys": "image", "name": "ColorJitter", "brightness": 0.25}, - }, - ] - }, + "_name_": "Compose", + "transforms": [ + {"_name_": "LoadImaged", "keys": "image"}, + {"_name_": "RandTorchVisiond", "keys": "image", "name": "ColorJitter", "brightness": 0.25}, + ], }, - "dataset": {"": "Dataset", "": {"data": [1, 2], "transform": "@transform"}}, + "dataset": {"_name_": "Dataset", "data": [1, 2], "transform": "@transform"}, "dataloader": { - "": "DataLoader", - "": {"dataset": "@dataset", "batch_size": 2, "collate_fn": "monai.data.list_data_collate"}, + "_name_": "DataLoader", + "dataset": "@dataset", + "batch_size": 2, + "collate_fn": "monai.data.list_data_collate", }, }, - ["transform", "transform##transforms#0", "transform##transforms#1", "dataset", "dataloader"], + ["transform", "transform#transforms#0", "transform#transforms#1", "dataset", "dataloader"], [Compose, LoadImaged, RandTorchVisiond, Dataset, DataLoader], ] @@ -67,9 +64,9 @@ def __call__(self, a, b): "cls_func": "$TestClass.cls_compute", "lambda_static_func": "$lambda x, y: TestClass.compute(x, y)", "lambda_cls_func": "$lambda x, y: TestClass.cls_compute(x, y)", - "compute": {"": "tests.test_config_parser.TestClass.compute", "": {"func": "@basic_func"}}, - "cls_compute": {"": "tests.test_config_parser.TestClass.cls_compute", "": {"func": "@basic_func"}}, - "call_compute": {"": "tests.test_config_parser.TestClass"}, + "compute": {"_path_": "tests.test_config_parser.TestClass.compute", "func": "@basic_func"}, + "cls_compute": {"_path_": "tests.test_config_parser.TestClass.cls_compute", "func": "@basic_func"}, + "call_compute": {"_path_": "tests.test_config_parser.TestClass"}, "error_func": "$TestClass.__call__", "": "$lambda x, y: x + y", } @@ -78,17 +75,17 @@ def __call__(self, a, b): class TestConfigComponent(unittest.TestCase): def test_config_content(self): - test_config = {"preprocessing": [{"": "LoadImage"}], "dataset": {"": "Dataset"}} + test_config = {"preprocessing": [{"_name_": "LoadImage"}], "dataset": {"_name_": "Dataset"}} parser = ConfigParser(config=test_config) # test `get`, `set`, `__getitem__`, `__setitem__` self.assertEqual(str(parser.get()), str(test_config)) parser.set(config=test_config) self.assertListEqual(parser["preprocessing"], test_config["preprocessing"]) - parser["dataset"] = {"": "CacheDataset"} - self.assertEqual(parser["dataset"][""], "CacheDataset") + parser["dataset"] = {"_name_": "CacheDataset"} + self.assertEqual(parser["dataset"]["_name_"], "CacheDataset") # test nested ids - parser["dataset#"] = "Dataset" - self.assertEqual(parser["dataset#"], "Dataset") + parser["dataset#_name_"] = "Dataset" + self.assertEqual(parser["dataset#_name_"], "Dataset") # test int id parser.set(["test1", "test2", "test3"]) parser[1] = "test4" @@ -99,11 +96,11 @@ def test_config_content(self): def test_parse(self, config, expected_ids, output_types): parser = ConfigParser(config=config, globals={"monai": "monai"}) # test lazy instantiation with original config content - parser["transform"][""]["transforms"][0][""]["keys"] = "label1" - self.assertEqual(parser.get_parsed_content(id="transform##transforms#0").keys[0], "label1") + parser["transform"]["transforms"][0]["keys"] = "label1" + self.assertEqual(parser.get_parsed_content(id="transform#transforms#0").keys[0], "label1") # test nested id - parser["transform##transforms#0##keys"] = "label2" - self.assertEqual(parser.get_parsed_content(id="transform##transforms#0").keys[0], "label2") + parser["transform#transforms#0#keys"] = "label2" + self.assertEqual(parser.get_parsed_content(id="transform#transforms#0").keys[0], "label2") for id, cls in zip(expected_ids, output_types): self.assertTrue(isinstance(parser.get_parsed_content(id), cls)) # test root content diff --git a/tests/test_reference_resolver.py b/tests/test_reference_resolver.py index e16a795c40..66b8655402 100644 --- a/tests/test_reference_resolver.py +++ b/tests/test_reference_resolver.py @@ -27,11 +27,10 @@ TEST_CASE_1 = [ { # all the recursively parsed config items - "transform#1": {"": "LoadImaged", "": {"keys": ["image"]}}, - "transform#1#": "LoadImaged", - "transform#1#": {"keys": ["image"]}, - "transform#1##keys": ["image"], - "transform#1##keys#0": "image", + "transform#1": {"_name_": "LoadImaged", "keys": ["image"]}, + "transform#1#_name_": "LoadImaged", + "transform#1#keys": ["image"], + "transform#1#keys#0": "image", }, "transform#1", LoadImaged, @@ -40,20 +39,15 @@ TEST_CASE_2 = [ { # some the recursively parsed config items - "dataloader": { - "": "DataLoader", - "": {"dataset": "@dataset", "collate_fn": "$monai.data.list_data_collate"}, - }, - "dataset": {"": "Dataset", "": {"data": [1, 2]}}, - "dataloader#": "DataLoader", - "dataloader#": {"dataset": "@dataset", "collate_fn": "$monai.data.list_data_collate"}, - "dataloader##dataset": "@dataset", - "dataloader##collate_fn": "$monai.data.list_data_collate", - "dataset#": "Dataset", - "dataset#": {"data": [1, 2]}, - "dataset##data": [1, 2], - "dataset##data#0": 1, - "dataset##data#1": 2, + "dataloader": {"_name_": "DataLoader", "dataset": "@dataset", "collate_fn": "$monai.data.list_data_collate"}, + "dataset": {"_name_": "Dataset", "data": [1, 2]}, + "dataloader#_name_": "DataLoader", + "dataloader#dataset": "@dataset", + "dataloader#collate_fn": "$monai.data.list_data_collate", + "dataset#_name_": "Dataset", + "dataset#data": [1, 2], + "dataset#data#0": 1, + "dataset#data#1": 2, }, "dataloader", DataLoader, @@ -62,15 +56,11 @@ TEST_CASE_3 = [ { # all the recursively parsed config items - "transform#1": { - "": "RandTorchVisiond", - "": {"keys": "image", "name": "ColorJitter", "brightness": 0.25}, - }, - "transform#1#": "RandTorchVisiond", - "transform#1#": {"keys": "image", "name": "ColorJitter", "brightness": 0.25}, - "transform#1##keys": "image", - "transform#1##name": "ColorJitter", - "transform#1##brightness": 0.25, + "transform#1": {"_name_": "RandTorchVisiond", "keys": "image", "name": "ColorJitter", "brightness": 0.25}, + "transform#1#_name_": "RandTorchVisiond", + "transform#1#keys": "image", + "transform#1#name": "ColorJitter", + "transform#1#brightness": 0.25, }, "transform#1", RandTorchVisiond, @@ -97,7 +87,7 @@ def test_resolve(self, configs, expected_id, output_type): # test lazy instantiation item = resolver.get_item(expected_id, resolve=True) config = item.get_config() - config[""] = False + config["_disabled_"] = False item.update_config(config=config) if isinstance(item, ConfigComponent): result = item.instantiate() diff --git a/tests/testing_data/inference.json b/tests/testing_data/inference.json index 0a1cc0277e..dcc64817d6 100644 --- a/tests/testing_data/inference.json +++ b/tests/testing_data/inference.json @@ -1,126 +1,98 @@ { "device": "$torch.device('cuda' if torch.cuda.is_available() else 'cpu')", "network_def": { - "": "UNet", - "": { - "spatial_dims": 3, - "in_channels": 1, - "out_channels": 2, - "channels": [ - 16, - 32, - 64, - 128, - 256 - ], - "strides": [ - 2, - 2, - 2, - 2 - ], - "num_res_units": 2, - "norm": "batch" - } + "_name_": "UNet", + "spatial_dims": 3, + "in_channels": 1, + "out_channels": 2, + "channels": [ + 16, + 32, + 64, + 128, + 256 + ], + "strides": [ + 2, + 2, + 2, + 2 + ], + "num_res_units": 2, + "norm": "batch" }, "network": "need override", "preprocessing": { - "": "Compose", - "": { - "transforms": [ - { - "": "LoadImaged", - "": { - "keys": "image" - } - }, - { - "": "EnsureChannelFirstd", - "": { - "keys": "image" - } - }, - { - "": "ScaleIntensityd", - "": { - "keys": "image" - } - }, - { - "": "EnsureTyped", - "": { - "keys": "image" - } - } - ] - } + "_name_": "Compose", + "transforms": [ + { + "_name_": "LoadImaged", + "keys": "image" + }, + { + "_name_": "EnsureChannelFirstd", + "keys": "image" + }, + { + "_name_": "ScaleIntensityd", + "keys": "image" + }, + { + "_name_": "EnsureTyped", + "keys": "image" + } + ] }, "dataset": { - "": "need override", - "": { - "data": "@#datalist", - "transform": "@preprocessing" - } + "_name_": "need override", + "data": "@_meta_#datalist", + "transform": "@preprocessing" }, "dataloader": { - "": "DataLoader", - "": { - "dataset": "@dataset", - "batch_size": 1, - "shuffle": false, - "num_workers": 4 - } + "_name_": "DataLoader", + "dataset": "@dataset", + "batch_size": 1, + "shuffle": false, + "num_workers": 4 }, "inferer": { - "": "SlidingWindowInferer", - "": { - "roi_size": [ - 96, - 96, - 96 - ], - "sw_batch_size": 4, - "overlap": 0.5 - } + "_name_": "SlidingWindowInferer", + "roi_size": [ + 96, + 96, + 96 + ], + "sw_batch_size": 4, + "overlap": 0.5 }, "postprocessing": { - "": "Compose", - "": { - "transforms": [ - { - "": "Activationsd", - "": { - "keys": "pred", - "softmax": true - } - }, - { - "": "AsDiscreted", - "": { - "keys": "pred", - "argmax": true - } - }, - { - "": "SaveImaged", - "": { - "keys": "pred", - "meta_keys": "image_meta_dict", - "output_dir": "@#output_dir" - } - } - ] - } + "_name_": "Compose", + "transforms": [ + { + "_name_": "Activationsd", + "keys": "pred", + "softmax": true + }, + { + "_name_": "AsDiscreted", + "keys": "pred", + "argmax": true + }, + { + "_name_": "SaveImaged", + "keys": "pred", + "meta_keys": "image_meta_dict", + "output_dir": "@_meta_#output_dir" + } + ] }, "evaluator": { - "": "SupervisedEvaluator", - "": { - "device": "@device", - "val_data_loader": "@dataloader", - "network": "@network", - "inferer": "@inferer", - "postprocessing": "@postprocessing", - "amp": false - } + "_name_": "SupervisedEvaluator", + "device": "@device", + "val_data_loader": "@dataloader", + "network": "@network", + "inferer": "@inferer", + "postprocessing": "@postprocessing", + "amp": false } } diff --git a/tests/testing_data/inference.yaml b/tests/testing_data/inference.yaml index 1a2f208a76..35876f70ff 100644 --- a/tests/testing_data/inference.yaml +++ b/tests/testing_data/inference.yaml @@ -1,85 +1,71 @@ --- device: "$torch.device('cuda' if torch.cuda.is_available() else 'cpu')" network_def: - "": UNet - "": - spatial_dims: 3 - in_channels: 1 - out_channels: 2 - channels: - - 16 - - 32 - - 64 - - 128 - - 256 - strides: - - 2 - - 2 - - 2 - - 2 - num_res_units: 2 - norm: batch + _name_: UNet + spatial_dims: 3 + in_channels: 1 + out_channels: 2 + channels: + - 16 + - 32 + - 64 + - 128 + - 256 + strides: + - 2 + - 2 + - 2 + - 2 + num_res_units: 2 + norm: batch network: need override preprocessing: - "": Compose - "": - transforms: - - "": LoadImaged - "": - keys: image - - "": EnsureChannelFirstd - "": - keys: image - - "": ScaleIntensityd - "": - keys: image - - "": EnsureTyped - "": - keys: image + _name_: Compose + transforms: + - _name_: LoadImaged + keys: image + - _name_: EnsureChannelFirstd + keys: image + - _name_: ScaleIntensityd + keys: image + - _name_: EnsureTyped + keys: image dataset: - "": need override - "": - data: "@#datalist" - transform: "@preprocessing" + _name_: need override + data: "@_meta_#datalist" + transform: "@preprocessing" dataloader: - "": DataLoader - "": - dataset: "@dataset" - batch_size: 1 - shuffle: false - num_workers: 4 + _name_: DataLoader + dataset: "@dataset" + batch_size: 1 + shuffle: false + num_workers: 4 inferer: - "": SlidingWindowInferer - "": - roi_size: - - 96 - - 96 - - 96 - sw_batch_size: 4 - overlap: 0.5 + _name_: SlidingWindowInferer + roi_size: + - 96 + - 96 + - 96 + sw_batch_size: 4 + overlap: 0.5 postprocessing: - "": Compose - "": - transforms: - - "": Activationsd - "": - keys: pred - softmax: true - - "": AsDiscreted - "": - keys: pred - argmax: true - - "": SaveImaged - "": - keys: pred - meta_keys: image_meta_dict - output_dir: "@#output_dir" + _name_: Compose + transforms: + - _name_: Activationsd + keys: pred + softmax: true + - _name_: AsDiscreted + keys: pred + argmax: true + - _name_: SaveImaged + keys: pred + meta_keys: image_meta_dict + output_dir: "@_meta_#output_dir" evaluator: - "": SupervisedEvaluator - "": - device: "@device" - val_data_loader: "@dataloader" - network: "@network" - inferer: "@inferer" - postprocessing: "@postprocessing" - amp: false + _name_: SupervisedEvaluator + device: "@device" + val_data_loader: "@dataloader" + network: "@network" + inferer: "@inferer" + postprocessing: "@postprocessing" + amp: false From 2aaa23a1b44ab948d32688c70dab03d13cc1f07a Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 10 Mar 2022 23:57:04 +0800 Subject: [PATCH 68/76] [DLMED] enhance disabled Signed-off-by: Nic Ma --- monai/bundle/reference_resolver.py | 3 +++ tests/testing_data/inference.json | 5 +++++ tests/testing_data/inference.yaml | 3 +++ 3 files changed, 11 insertions(+) diff --git a/monai/bundle/reference_resolver.py b/monai/bundle/reference_resolver.py index 8a8b5ed89d..9b5f00f9ef 100644 --- a/monai/bundle/reference_resolver.py +++ b/monai/bundle/reference_resolver.py @@ -259,6 +259,9 @@ def update_config_with_refs(cls, config, id: str, refs: Optional[Dict] = None): sub_id = f"{id}{cls.sep}{idx}" if id != "" else f"{idx}" if ConfigComponent.is_instantiable(v) or ConfigExpression.is_expression(v): updated = refs_[sub_id] + if ConfigComponent.is_instantiable(v) and updated is None: + # the component is disabled + continue else: updated = cls.update_config_with_refs(v, sub_id, refs_) ret.update({idx: updated}) if isinstance(ret, dict) else ret.append(updated) diff --git a/tests/testing_data/inference.json b/tests/testing_data/inference.json index dcc64817d6..608a62a97d 100644 --- a/tests/testing_data/inference.json +++ b/tests/testing_data/inference.json @@ -37,6 +37,11 @@ "_name_": "ScaleIntensityd", "keys": "image" }, + { + "_name_": "RandRotated", + "_disabled_": true, + "keys": "image" + }, { "_name_": "EnsureTyped", "keys": "image" diff --git a/tests/testing_data/inference.yaml b/tests/testing_data/inference.yaml index 35876f70ff..325d69e749 100644 --- a/tests/testing_data/inference.yaml +++ b/tests/testing_data/inference.yaml @@ -28,6 +28,9 @@ preprocessing: keys: image - _name_: ScaleIntensityd keys: image + - _name_: RandRotated + _disabled_: true + keys: image - _name_: EnsureTyped keys: image dataset: From 91dd87f0b4dc8f0095c84c54e5c7e09eae2f1383 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 10 Mar 2022 19:44:24 +0000 Subject: [PATCH 69/76] special chars to utils Signed-off-by: Wenqi Li --- monai/bundle/__init__.py | 2 +- monai/bundle/config_item.py | 7 ++++--- monai/bundle/config_parser.py | 3 ++- monai/bundle/config_reader.py | 9 +++++---- monai/bundle/reference_resolver.py | 5 +++-- monai/bundle/utils.py | 8 ++++++++ 6 files changed, 23 insertions(+), 11 deletions(-) diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py index 5236e6b788..ce57e6cb89 100644 --- a/monai/bundle/__init__.py +++ b/monai/bundle/__init__.py @@ -14,4 +14,4 @@ from .config_reader import ConfigReader from .reference_resolver import ReferenceResolver from .scripts import run -from .utils import update_default_args +from .utils import EXPR_KEY, ID_REF_KEY, ID_SEP_KEY, MACRO_KEY, update_default_args diff --git a/monai/bundle/config_item.py b/monai/bundle/config_item.py index 9392f294f5..3311f5ae3a 100644 --- a/monai/bundle/config_item.py +++ b/monai/bundle/config_item.py @@ -17,6 +17,7 @@ from importlib import import_module from typing import Any, Dict, List, Mapping, Optional, Sequence, Union +from monai.bundle.utils import EXPR_KEY from monai.utils import ensure_tuple, instantiate __all__ = ["ComponentLocator", "ConfigItem", "ConfigExpression", "ConfigComponent"] @@ -194,7 +195,7 @@ class ConfigComponent(ConfigItem, Instantiable): """ - not_arg_keys = ["_name_", "_path_", "_disabled_"] + non_arg_keys = {"_name_", "_path_", "_disabled_"} def __init__( self, @@ -253,7 +254,7 @@ def resolve_args(self): Utility function used in `instantiate()` to resolve the arguments from current config content. """ - return {k: v for k, v in self.get_config().items() if k not in self.not_arg_keys} + return {k: v for k, v in self.get_config().items() if k not in self.non_arg_keys} def is_disabled(self) -> bool: # type: ignore """ @@ -309,7 +310,7 @@ class ConfigExpression(ConfigItem): """ - prefix = "$" + prefix = EXPR_KEY run_eval = False if os.environ.get("MONAI_EVAL_EXPR", "1") == "0" else True def __init__(self, config: Any, id: str = "", globals: Optional[Dict] = None) -> None: diff --git a/monai/bundle/config_parser.py b/monai/bundle/config_parser.py index 70993a71a5..b7aab78ffc 100644 --- a/monai/bundle/config_parser.py +++ b/monai/bundle/config_parser.py @@ -16,6 +16,7 @@ from monai.bundle.config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem from monai.bundle.config_reader import ConfigReader from monai.bundle.reference_resolver import ReferenceResolver +from monai.bundle.utils import MACRO_KEY __all__ = ["ConfigParser"] @@ -75,7 +76,7 @@ class ConfigParser: """ - macro = "%" # macro prefix + macro = MACRO_KEY # macro prefix def __init__( self, diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index 170883c616..740e8a9b12 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -14,6 +14,7 @@ from pathlib import Path from typing import Dict, Sequence, Tuple, Union +from monai.bundle.utils import ID_SEP_KEY from monai.config import PathLike from monai.utils import ensure_tuple, look_up_option, optional_import @@ -36,10 +37,10 @@ class ConfigReader: """ suffixes = ("json", "yaml", "yml") - suffix_match = rf"\.({'|'.join(suffixes)})" - path_match = rf"(.*{suffix_match}$)" + suffix_match = rf".*\.({'|'.join(suffixes)})" + path_match = rf"({suffix_match}$)" meta_key = "_meta_" # field key to save metadata - sep = "#" # separator for file path and the id of content in the file + sep = ID_SEP_KEY # separator for file path and the id of content in the file def __init__(self): self.config: Dict = {self.meta_key: {}} @@ -112,7 +113,7 @@ def split_path_id(cls, src: str) -> Tuple[str, str]: src: source string to split. """ - result = re.compile(rf"(.*{cls.suffix_match}(?=(?:{cls.sep}.*)|$))", re.IGNORECASE).findall(src) + result = re.compile(rf"({cls.suffix_match}(?=(?:{cls.sep}.*)|$))", re.IGNORECASE).findall(src) if not result: return "", src # the src is a pure id path_name = result[0][0] # at most one path_name diff --git a/monai/bundle/reference_resolver.py b/monai/bundle/reference_resolver.py index 9b5f00f9ef..eee456028b 100644 --- a/monai/bundle/reference_resolver.py +++ b/monai/bundle/reference_resolver.py @@ -13,6 +13,7 @@ from typing import Any, Dict, Optional, Sequence, Set from monai.bundle.config_item import ConfigComponent, ConfigExpression, ConfigItem +from monai.bundle.utils import ID_REF_KEY, ID_SEP_KEY from monai.utils import look_up_option __all__ = ["ReferenceResolver"] @@ -45,8 +46,8 @@ class ReferenceResolver: """ _vars = "__local_refs" - sep = "#" # separator for key indexing - ref = "@" # reference prefix + sep = ID_SEP_KEY # separator for key indexing + ref = ID_REF_KEY # reference prefix # match a reference string, e.g. "@id#key", "@id#key#0", "@_name_#key" id_matcher = re.compile(rf"{ref}(?:\w*)(?:{sep}\w*)*") diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 35fd9e46a4..b0882b8e32 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -13,6 +13,14 @@ from monai.bundle.config_reader import ConfigReader +__all__ = ["ID_REF_KEY", "ID_SEP_KEY", "EXPR_KEY", "MACRO_KEY", "update_default_args"] + + +ID_REF_KEY = "@" # start of a reference to a ConfigItem +ID_SEP_KEY = "#" # separator for the ID of a ConfigItem +EXPR_KEY = "$" # start of an ConfigExpression +MACRO_KEY = "%" # start of a macro of a config + def update_default_args(args: Optional[Union[str, Dict]] = None, **kwargs) -> Dict: """ From 194b113f28bdf04b31d6f893c5715015eedc83d4 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 10 Mar 2022 20:19:55 +0000 Subject: [PATCH 70/76] remove circular import Signed-off-by: Wenqi Li --- docs/source/bundle.rst | 4 ---- monai/bundle/__init__.py | 2 +- monai/bundle/scripts.py | 26 +++++++++++++++++++++++--- monai/bundle/utils.py | 27 +-------------------------- 4 files changed, 25 insertions(+), 34 deletions(-) diff --git a/docs/source/bundle.rst b/docs/source/bundle.rst index ed6321ff37..e07efb7ba3 100644 --- a/docs/source/bundle.rst +++ b/docs/source/bundle.rst @@ -41,7 +41,3 @@ Model Bundle `Scripts` --------- .. autofunction:: run - -`Utilities` ------------ -.. autofunction:: update_default_args diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py index ce57e6cb89..298d6938a7 100644 --- a/monai/bundle/__init__.py +++ b/monai/bundle/__init__.py @@ -14,4 +14,4 @@ from .config_reader import ConfigReader from .reference_resolver import ReferenceResolver from .scripts import run -from .utils import EXPR_KEY, ID_REF_KEY, ID_SEP_KEY, MACRO_KEY, update_default_args +from .utils import EXPR_KEY, ID_REF_KEY, ID_SEP_KEY, MACRO_KEY diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index a5749c1e1e..b8fe76f639 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -9,11 +9,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Sequence, Union +from typing import Dict, Optional, Sequence, Union from monai.bundle.config_parser import ConfigParser from monai.bundle.config_reader import ConfigReader -from monai.bundle.utils import update_default_args + + +def _update_default_args(args: Optional[Union[str, Dict]] = None, **kwargs) -> Dict: + """ + Update the `args` with the input `kwargs`. + For dict data, recursively update the content based on the keys. + + Args: + args: source args to update. + kwargs: destination args to update. + + """ + args_: Dict = args if isinstance(args, dict) else {} # type: ignore + if isinstance(args, str): + # args are defined in a structured file + args_ = ConfigReader.load_config_file(args) + + # recursively update the default args with new args + for k, v in kwargs.items(): + args_[k] = _update_default_args(args_[k], **v) if isinstance(v, dict) and isinstance(args_.get(k), dict) else v + return args_ def run( @@ -62,7 +82,7 @@ def run( for k, v in k_v: if v is not None: override[k] = v - _args = update_default_args(args=args_file, **override) + _args = _update_default_args(args=args_file, **override) for k in ("meta_file", "config_file"): if k not in _args: raise ValueError(f"{k} is required for 'monai.bundle run'.\n{run.__doc__}") diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index b0882b8e32..75a0ebc2a2 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -9,35 +9,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, Optional, Union - -from monai.bundle.config_reader import ConfigReader - -__all__ = ["ID_REF_KEY", "ID_SEP_KEY", "EXPR_KEY", "MACRO_KEY", "update_default_args"] +__all__ = ["ID_REF_KEY", "ID_SEP_KEY", "EXPR_KEY", "MACRO_KEY"] ID_REF_KEY = "@" # start of a reference to a ConfigItem ID_SEP_KEY = "#" # separator for the ID of a ConfigItem EXPR_KEY = "$" # start of an ConfigExpression MACRO_KEY = "%" # start of a macro of a config - - -def update_default_args(args: Optional[Union[str, Dict]] = None, **kwargs) -> Dict: - """ - Update the `args` with the input `kwargs`. - For dict data, recursively update the content based on the keys. - - Args: - args: source args to update. - kwargs: destination args to update. - - """ - args_: Dict = args if isinstance(args, dict) else {} # type: ignore - if isinstance(args, str): - # args are defined in a structured file - args_ = ConfigReader.load_config_file(args) - - # recursively update the default args with new args - for k, v in kwargs.items(): - args_[k] = update_default_args(args_[k], **v) if isinstance(v, dict) and isinstance(args_.get(k), dict) else v - return args_ From 056a1199c5860b75027c83ca24ff62ee422d6ee1 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 10 Mar 2022 20:58:17 +0000 Subject: [PATCH 71/76] configreader -> configparser following cpython/configparser Signed-off-by: Wenqi Li --- docs/source/bundle.rst | 5 - monai/bundle/__init__.py | 1 - monai/bundle/config_parser.py | 179 ++++++++++++++++++++++++++++------ monai/bundle/config_reader.py | 158 ------------------------------ monai/bundle/scripts.py | 10 +- tests/test_bundle_run.py | 6 +- 6 files changed, 155 insertions(+), 204 deletions(-) delete mode 100644 monai/bundle/config_reader.py diff --git a/docs/source/bundle.rst b/docs/source/bundle.rst index e07efb7ba3..22260d822f 100644 --- a/docs/source/bundle.rst +++ b/docs/source/bundle.rst @@ -33,11 +33,6 @@ Model Bundle .. autoclass:: ConfigParser :members: -`Config Reader` ---------------- -.. autoclass:: ConfigReader - :members: - `Scripts` --------- .. autofunction:: run diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py index 298d6938a7..b411406e84 100644 --- a/monai/bundle/__init__.py +++ b/monai/bundle/__init__.py @@ -11,7 +11,6 @@ from .config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem, Instantiable from .config_parser import ConfigParser -from .config_reader import ConfigReader from .reference_resolver import ReferenceResolver from .scripts import run from .utils import EXPR_KEY, ID_REF_KEY, ID_SEP_KEY, MACRO_KEY diff --git a/monai/bundle/config_parser.py b/monai/bundle/config_parser.py index b7aab78ffc..9a3d24c7af 100644 --- a/monai/bundle/config_parser.py +++ b/monai/bundle/config_parser.py @@ -10,13 +10,19 @@ # limitations under the License. import importlib +import json +import re from copy import deepcopy -from typing import Any, Dict, Optional, Sequence, Union +from pathlib import Path +from typing import Any, Dict, Optional, Sequence, Tuple, Union from monai.bundle.config_item import ComponentLocator, ConfigComponent, ConfigExpression, ConfigItem -from monai.bundle.config_reader import ConfigReader from monai.bundle.reference_resolver import ReferenceResolver -from monai.bundle.utils import MACRO_KEY +from monai.bundle.utils import ID_SEP_KEY, MACRO_KEY +from monai.config import PathLike +from monai.utils import ensure_tuple, look_up_option, optional_import + +yaml, _ = optional_import("yaml") __all__ = ["ConfigParser"] @@ -73,14 +79,18 @@ class ConfigParser: See also: - :py:class:`monai.bundle.ConfigItem` + - :py:class:`monai.bundle.scripts.run` """ - macro = MACRO_KEY # macro prefix + suffixes = ("json", "yaml", "yml") + suffix_match = rf".*\.({'|'.join(suffixes)})" + path_match = rf"({suffix_match}$)" + meta_key = "_meta_" # field key to save metadata def __init__( self, - config: Any, + config: Any = None, excludes: Optional[Union[Sequence[str], str]] = None, globals: Optional[Dict[str, Any]] = None, ): @@ -93,6 +103,8 @@ def __init__( self.locator = ComponentLocator(excludes=excludes) self.ref_resolver = ReferenceResolver() + if config is None: + config = {self.meta_key: {}} self.set(config=config) def __repr__(self): @@ -166,6 +178,72 @@ def set(self, config: Any, id: str = ""): """ self[id] = config + def parse(self, reset: bool = True): + """ + Recursively resolve `self.config` to replace the macro tokens with target content. + Then recursively parse the config source, add every item as ``ConfigItem`` to the reference resolver. + + Args: + reset: whether to reset the ``reference_resolver`` before parsing. Defaults to `True`. + + """ + if reset: + self.ref_resolver.reset() + self.resolve_macro() + self._do_parse(config=self.get()) + + def get_parsed_content(self, id: str = "", **kwargs): + """ + Get the parsed result of ``ConfigItem`` with the specified ``id``. + + - If the item is ``ConfigComponent`` and ``instantiate=True``, the result is the instance. + - If the item is ``ConfigExpression`` and ``eval_expr=True``, the result is the evaluated output. + - Else, the result is the configuration content of `ConfigItem`. + + Args: + id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to + go one level further into the nested structures. + Use digits indexing from "0" for list or other strings for dict. + For example: ``"xform#5"``, ``"net#channels"``. ``""`` indicates the entire ``self.config``. + kwargs: additional keyword arguments to be passed to ``_resolve_one_item``. + Currently support ``reset`` (for parse), ``instantiate`` and ``eval_expr``. All defaulting to True. + + """ + if not self.ref_resolver.is_resolved(): + # not parsed the config source yet, parse it + self.parse(kwargs.get("reset", True)) + return self.ref_resolver.get_resolved_content(id=id, **kwargs) + + def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): + """ + Read the metadata from specified JSON or YAML file. + The metadata as a dictionary will be stored at ``self.config["_meta_"]``. + + Args: + f: filepath of the metadata file, the content must be a dictionary, + if providing a list of files, wil merge the content of them. + if providing a dictionary directly, use it as metadata. + kwargs: other arguments for ``json.load`` or ``yaml.safe_load``, depends on the file format. + + """ + self.set(self.load_config_files(f, **kwargs), self.meta_key) + + def read_config(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): + """ + Read the config from specified JSON or YAML file. + The config content in the `self.config` dictionary. + + Args: + f: filepath of the config file, the content must be a dictionary, + if providing a list of files, wil merge the content of them. + if providing a dictionary directly, use it as config. + kwargs: other arguments for ``json.load`` or ``yaml.safe_load``, depends on the file format. + + """ + content = {self.meta_key: self.get(self.meta_key, {})} + content.update(self.load_config_files(f, **kwargs)) + self.set(config=content) + def _do_resolve(self, config: Any): """ Recursively resolve the config content to replace the macro tokens with target content. @@ -179,9 +257,9 @@ def _do_resolve(self, config: Any): if isinstance(config, (dict, list)): for k, v in enumerate(config) if isinstance(config, list) else config.items(): config[k] = self._do_resolve(v) - if isinstance(config, str) and config.startswith(self.macro): - path, ids = ConfigReader.split_path_id(config[len(self.macro) :]) - parser = ConfigParser(config=self.get() if not path else ConfigReader.load_config_file(path)) + if isinstance(config, str) and config.startswith(MACRO_KEY): + path, ids = ConfigParser.split_path_id(config[len(MACRO_KEY) :]) + parser = ConfigParser(config=self.get() if not path else ConfigParser.load_config_file(path)) return self._do_resolve(config=deepcopy(parser[ids])) return config @@ -221,38 +299,77 @@ def _do_parse(self, config, id: str = ""): else: self.ref_resolver.add_item(ConfigItem(config=item_conf, id=id)) - def parse(self, reset: bool = True): + @classmethod + def load_config_file(cls, filepath: PathLike, **kwargs): """ - Recursively resolve `self.config` to replace the macro tokens with target content. - Then recursively parse the config source, add every item as ``ConfigItem`` to the reference resolver. + Load config file with specified file path (currently support JSON and YAML files). Args: - reset: whether to reset the ``reference_resolver`` before parsing. Defaults to `True`. + filepath: path of target file to load, supported postfixes: `.json`, `.yml`, `.yaml`. + kwargs: other arguments for ``json.load`` or ```yaml.safe_load``, depends on the file format. """ - if reset: - self.ref_resolver.reset() - self.resolve_macro() - self._do_parse(config=self.get()) + _filepath: str = str(Path(filepath)) + if not re.compile(cls.path_match, re.IGNORECASE).findall(_filepath): + raise ValueError(f'unknown file input: "{filepath}"') + with open(_filepath) as f: + if _filepath.lower().endswith(cls.suffixes[0]): + return json.load(f, **kwargs) + if _filepath.lower().endswith(cls.suffixes[1:]): + return yaml.safe_load(f, **kwargs) + raise ValueError(f"only support JSON or YAML config file so far, got name {_filepath}.") + + @classmethod + def load_config_files(cls, files: Union[PathLike, Sequence[PathLike], dict], **kwargs) -> dict: + """ + Load config files into a single config dict. - def get_parsed_content(self, id: str = "", **kwargs): + Args: + files: path of target files to load, supported postfixes: `.json`, `.yml`, `.yaml`. + kwargs: other arguments for ``json.load`` or ```yaml.safe_load``, depends on the file format. """ - Get the parsed result of ``ConfigItem`` with the specified ``id``. + if isinstance(files, dict): # already a config dict + return files + content = {} + for i in ensure_tuple(files): + content.update(cls.load_config_file(i, **kwargs)) + return content + + @classmethod + def export_config_file(cls, config: Dict, filepath: PathLike, fmt="json", **kwargs): + """ + Export the config content to the specified file path (currently support JSON and YAML files). - - If the item is ``ConfigComponent`` and ``instantiate=True``, the result is the instance. - - If the item is ``ConfigExpression`` and ``eval_expr=True``, the result is the evaluated output. - - Else, the result is the configuration content of `ConfigItem`. + Args: + config: source config content to export. + filepath: target file path to save. + fmt: format of config content, currently support ``"json"`` and ``"yaml"``. + kwargs: other arguments for ``json.dump`` or ``yaml.safe_dump``, depends on the file format. + + """ + _filepath: str = str(Path(filepath)) + writer = look_up_option(fmt.lower(), {"json", "yaml"}) + with open(_filepath, "w") as f: + if writer == "json": + return json.dump(config, f, **kwargs) + if writer == "yaml": + return yaml.safe_dump(config, f, **kwargs) + raise ValueError(f"only support JSON or YAML config file so far, got {writer}.") + + @classmethod + def split_path_id(cls, src: str) -> Tuple[str, str]: + """ + Split `src` string into two parts: a config file path and component id. + The file path should end with `(json|yaml|yml)`. The component id should be separated by `#` if it exists. + If no path or no id, return "". Args: - id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to - go one level further into the nested structures. - Use digits indexing from "0" for list or other strings for dict. - For example: ``"xform#5"``, ``"net#channels"``. ``""`` indicates the entire ``self.config``. - kwargs: additional keyword arguments to be passed to ``_resolve_one_item``. - Currently support ``reset`` (for parse), ``instantiate`` and ``eval_expr``. All defaulting to True. + src: source string to split. """ - if not self.ref_resolver.is_resolved(): - # not parsed the config source yet, parse it - self.parse(kwargs.get("reset", True)) - return self.ref_resolver.get_resolved_content(id=id, **kwargs) + result = re.compile(rf"({cls.suffix_match}(?=(?:{ID_SEP_KEY}.*)|$))", re.IGNORECASE).findall(src) + if not result: + return "", src # the src is a pure id + path_name = result[0][0] # at most one path_name + _, ids = src.rsplit(path_name, 1) + return path_name, ids[len(ID_SEP_KEY) :] if ids.startswith(ID_SEP_KEY) else "" diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py deleted file mode 100644 index 740e8a9b12..0000000000 --- a/monai/bundle/config_reader.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright (c) MONAI Consortium -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import re -from pathlib import Path -from typing import Dict, Sequence, Tuple, Union - -from monai.bundle.utils import ID_SEP_KEY -from monai.config import PathLike -from monai.utils import ensure_tuple, look_up_option, optional_import - -yaml, _ = optional_import("yaml") - -__all__ = ["ConfigReader"] - - -class ConfigReader: - """ - Read config and metadata from JSON or YAML files. - Support to override the config content with specified `id` and value. - Support to resolve the macro tokens in the config content. - - See also: - - - https://docs.python.org/3/library/json.html - - https://pyyaml.org/wiki/PyYAMLDocumentation - - """ - - suffixes = ("json", "yaml", "yml") - suffix_match = rf".*\.({'|'.join(suffixes)})" - path_match = rf"({suffix_match}$)" - meta_key = "_meta_" # field key to save metadata - sep = ID_SEP_KEY # separator for file path and the id of content in the file - - def __init__(self): - self.config: Dict = {self.meta_key: {}} - - @classmethod - def load_config_file(cls, filepath: PathLike, **kwargs): - """ - Load config file with specified file path (currently support JSON and YAML files). - - Args: - filepath: path of target file to load, supported postfixes: `.json`, `.yml`, `.yaml`. - kwargs: other arguments for ``json.load`` or ```yaml.safe_load``, depends on the file format. - - """ - _filepath: str = str(Path(filepath)) - if not re.compile(cls.path_match, re.IGNORECASE).findall(_filepath): - raise ValueError(f'unknown file input: "{filepath}"') - with open(_filepath) as f: - if _filepath.lower().endswith(cls.suffixes[0]): - return json.load(f, **kwargs) - if _filepath.lower().endswith(cls.suffixes[1:]): - return yaml.safe_load(f, **kwargs) - raise ValueError(f"only support JSON or YAML config file so far, got name {_filepath}.") - - @classmethod - def load_config_files(cls, files: Union[PathLike, Sequence[PathLike], dict], **kwargs) -> dict: - """ - Load config files into a single config dict. - - Args: - files: path of target files to load, supported postfixes: `.json`, `.yml`, `.yaml`. - kwargs: other arguments for ``json.load`` or ```yaml.safe_load``, depends on the file format. - """ - if isinstance(files, dict): # already a config dict - return files - content = {} - for i in ensure_tuple(files): - content.update(cls.load_config_file(i, **kwargs)) - return content - - @classmethod - def export_config_file(cls, config: Dict, filepath: PathLike, fmt="json", **kwargs): - """ - Export the config content to the specified file path (currently support JSON and YAML files). - - Args: - config: source config content to export. - filepath: target file path to save. - fmt: format of config content, currently support ``"json"`` and ``"yaml"``. - kwargs: other arguments for ``json.dump`` or ``yaml.safe_dump``, depends on the file format. - - """ - _filepath: str = str(Path(filepath)) - writer = look_up_option(fmt.lower(), {"json", "yaml"}) - with open(_filepath, "w") as f: - if writer == "json": - return json.dump(config, f, **kwargs) - if writer == "yaml": - return yaml.safe_dump(config, f, **kwargs) - raise ValueError(f"only support JSON or YAML config file so far, got {writer}.") - - @classmethod - def split_path_id(cls, src: str) -> Tuple[str, str]: - """ - Split `src` string into two parts: a config file path and component id. - The file path should end with `(json|yaml|yml)`. The component id should be separated by `#` if it exists. - If no path or no id, return "". - - Args: - src: source string to split. - - """ - result = re.compile(rf"({cls.suffix_match}(?=(?:{cls.sep}.*)|$))", re.IGNORECASE).findall(src) - if not result: - return "", src # the src is a pure id - path_name = result[0][0] # at most one path_name - _, ids = src.rsplit(path_name, 1) - return path_name, ids[len(cls.sep) :] if ids.startswith(cls.sep) else "" - - def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): - """ - Read the metadata from specified JSON or YAML file. - The metadata as a dictionary will be stored at ``self.config["_meta_"]``. - - Args: - f: filepath of the metadata file, the content must be a dictionary, - if providing a list of files, wil merge the content of them. - if providing a dictionary directly, use it as metadata. - kwargs: other arguments for ``json.load`` or ``yaml.safe_load``, depends on the file format. - - """ - self.config[self.meta_key] = self.load_config_files(f, **kwargs) - - def read_config(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): - """ - Read the config from specified JSON or YAML file. - The config content in the `self.config` dictionary. - - Args: - f: filepath of the config file, the content must be a dictionary, - if providing a list of files, wil merge the content of them. - if providing a dictionary directly, use it as config. - kwargs: other arguments for ``json.load`` or ``yaml.safe_load``, depends on the file format. - - """ - content = {self.meta_key: self.config.get(self.meta_key)} - content.update(self.load_config_files(f, **kwargs)) - self.config = content - - def get(self): - """ - Get the loaded config content. - - """ - return self.config diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index b8fe76f639..74f9dd874c 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -12,7 +12,6 @@ from typing import Dict, Optional, Sequence, Union from monai.bundle.config_parser import ConfigParser -from monai.bundle.config_reader import ConfigReader def _update_default_args(args: Optional[Union[str, Dict]] = None, **kwargs) -> Dict: @@ -28,7 +27,7 @@ def _update_default_args(args: Optional[Union[str, Dict]] = None, **kwargs) -> D args_: Dict = args if isinstance(args, dict) else {} # type: ignore if isinstance(args, str): # args are defined in a structured file - args_ = ConfigReader.load_config_file(args) + args_ = ConfigParser.load_config_file(args) # recursively update the default args with new args for k, v in kwargs.items(): @@ -87,12 +86,11 @@ def run( if k not in _args: raise ValueError(f"{k} is required for 'monai.bundle run'.\n{run.__doc__}") - reader = ConfigReader() - reader.read_config(f=_args.pop("config_file")) - reader.read_meta(f=_args.pop("meta_file")) + parser = ConfigParser() + parser.read_config(f=_args.pop("config_file")) + parser.read_meta(f=_args.pop("meta_file")) id = _args.pop("target_id", "") - parser = ConfigParser(config=reader.get()) # the rest key-values in the args are to override config content for k, v in _args.items(): parser[k] = v diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index d6c41271f5..8d7812bd22 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -20,7 +20,7 @@ import numpy as np from parameterized import parameterized -from monai.bundle import ConfigReader +from monai.bundle import ConfigParser from monai.transforms import LoadImage TEST_CASE_1 = [os.path.join(os.path.dirname(__file__), "testing_data", "inference.json"), (128, 128, 128)] @@ -39,12 +39,12 @@ def test_shape(self, config_file, expected_shape): # generate default args in a JSON file def_args = {"config_file": "will be replaced by `config_file` arg"} def_args_file = os.path.join(tempdir, "def_args.json") - ConfigReader.export_config_file(config=def_args, filepath=def_args_file) + ConfigParser.export_config_file(config=def_args, filepath=def_args_file) meta = {"datalist": [{"image": filename}], "output_dir": tempdir, "window": (96, 96, 96)} # test YAML file meta_file = os.path.join(tempdir, "meta.yaml") - ConfigReader.export_config_file(config=meta, filepath=meta_file, fmt="yaml") + ConfigParser.export_config_file(config=meta, filepath=meta_file, fmt="yaml") # test override with file, up case postfix overridefile1 = os.path.join(tempdir, "override1.JSON") From fe83f0762a58b8081b06ecc3afafa8ca2c047654 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 10 Mar 2022 21:34:44 +0000 Subject: [PATCH 72/76] printing Signed-off-by: Wenqi Li --- monai/bundle/scripts.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index 74f9dd874c..ebfd3e54ac 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -9,6 +9,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pprint from typing import Dict, Optional, Sequence, Union from monai.bundle.config_parser import ConfigParser @@ -81,6 +82,16 @@ def run( for k, v in k_v: if v is not None: override[k] = v + + full_kv = zip( + ("meta_file", "config_file", "target_id", "args_file", "override"), + (meta_file, config_file, target_id, args_file, override), + ) + print("\n--- input summary of monai.bundle.scripts.run ---") + for name, val in full_kv: + print(f"> {name}: {pprint.pformat(val)}") + print("---\n\n") + _args = _update_default_args(args=args_file, **override) for k in ("meta_file", "config_file"): if k not in _args: From 140390939eb82820e9dc05c63fbc3b48f1da8bfe Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 10 Mar 2022 21:35:58 +0000 Subject: [PATCH 73/76] fixes typing Signed-off-by: Wenqi Li --- monai/data/samplers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/data/samplers.py b/monai/data/samplers.py index f5175266d8..40eed03187 100644 --- a/monai/data/samplers.py +++ b/monai/data/samplers.py @@ -50,7 +50,7 @@ def __init__( super().__init__(dataset=dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle, **kwargs) if not even_divisible: - data_len = len(dataset) + data_len = len(dataset) # type: ignore extra_size = self.total_size - data_len if self.rank + extra_size >= self.num_replicas: self.num_samples -= 1 From 9d4ea6b79d119287273727804dc3522ad47b4a45 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Fri, 11 Mar 2022 09:53:39 +0800 Subject: [PATCH 74/76] [DLMED] unify "_name_" and "_path_" Signed-off-by: Nic Ma --- monai/bundle/config_item.py | 37 +++++++++++++++---------------------- monai/utils/module.py | 2 +- tests/test_config_item.py | 6 +++--- tests/test_config_parser.py | 6 +++--- 4 files changed, 22 insertions(+), 29 deletions(-) diff --git a/monai/bundle/config_item.py b/monai/bundle/config_item.py index 3311f5ae3a..d959006fde 100644 --- a/monai/bundle/config_item.py +++ b/monai/bundle/config_item.py @@ -167,8 +167,8 @@ class ConfigComponent(ConfigItem, Instantiable): Currently, four special keys (strings surrounded by ``_``) are defined and interpreted beyond the regular literals: - class or function identifier of the python module, specified by one of the two keys. - - ``"_name_"``: indicates build-in python classes or functions such as "LoadImageDict". - - ``"_path_"``: full module name, such as "monai.transforms.LoadImageDict". + - ``"_name_"``: indicates build-in python classes or functions such as "LoadImageDict", + or full module name, such as "monai.transforms.LoadImageDict". - ``"_disabled_"``: a flag to indicate whether to skip the instantiation. Other fields in the config content are input arguments to the python module. @@ -195,7 +195,7 @@ class ConfigComponent(ConfigItem, Instantiable): """ - non_arg_keys = {"_name_", "_path_", "_disabled_"} + non_arg_keys = {"_name_", "_disabled_"} def __init__( self, @@ -216,38 +216,31 @@ def is_instantiable(config: Any) -> bool: config: input config content to check. """ - return isinstance(config, Mapping) and ("_path_" in config or "_name_" in config) + return isinstance(config, Mapping) and "_name_" in config def resolve_module_name(self): """ Resolve the target module name from current config content. - The config content must have ``"_path_"`` or ``"_name_"`` key. - When both are specified, ``"_path_"`` will be used. + The config content must have ``"_name_"`` key. """ config = dict(self.get_config()) - path = config.get("_path_") - if path is not None: - if not isinstance(path, str): - raise ValueError(f"'_path_' must be a string, but got: {path}.") - if "_name_" in config: - warnings.warn(f"both '_path_' and '_name_', default to use '_path_': {path}.") - return path - - name = config.get("_name_") - if not isinstance(name, str): - raise ValueError("must provide a string for `_path_` or `_name_` of target component to instantiate.") + target = config.get("_name_") + if not isinstance(target, str): + raise ValueError("must provide a string for the `_name_` of component to instantiate.") - module = self.locator.get_component_module_name(name) + module = self.locator.get_component_module_name(target) if module is None: - raise ModuleNotFoundError(f"can not find component '{name}' in {self.locator.MOD_START} modules.") + # target is the full module name, no need to parse + return target + if isinstance(module, list): warnings.warn( - f"there are more than 1 component have name `{name}`: {module}, use the first one `{module[0]}." - f" if want to use others, please set its module path in `_path_` directly." + f"there are more than 1 component have name `{target}`: {module}, use the first one `{module[0]}." + f" if want to use others, please set its full module path in `_name_` directly." ) module = module[0] - return f"{module}.{name}" + return f"{module}.{target}" def resolve_args(self): """ diff --git a/monai/utils/module.py b/monai/utils/module.py index de2152d182..065cc8f7c8 100644 --- a/monai/utils/module.py +++ b/monai/utils/module.py @@ -212,7 +212,7 @@ def instantiate(path: str, **kwargs): component = locate(path) if component is None: - raise ModuleNotFoundError(f"Cannot locate '{path}'.") + raise ModuleNotFoundError(f"Cannot locate class or function path: '{path}'.") if isclass(component): return component(**kwargs) # support regular function, static method and class method diff --git a/tests/test_config_item.py b/tests/test_config_item.py index 507b5a7d92..4f0d5f141c 100644 --- a/tests/test_config_item.py +++ b/tests/test_config_item.py @@ -27,14 +27,14 @@ TEST_CASE_1 = [{"lr": 0.001}, 0.0001] TEST_CASE_2 = [{"_name_": "LoadImaged", "keys": ["image"]}, LoadImaged] -# test python `_path_` -TEST_CASE_3 = [{"_path_": "monai.transforms.LoadImaged", "keys": ["image"]}, LoadImaged] +# test full module path +TEST_CASE_3 = [{"_name_": "monai.transforms.LoadImaged", "keys": ["image"]}, LoadImaged] # test `_disabled_` TEST_CASE_4 = [{"_name_": "LoadImaged", "_disabled_": True, "keys": ["image"]}, dict] # test `_disabled_` with string TEST_CASE_5 = [{"_name_": "LoadImaged", "_disabled_": "true", "keys": ["image"]}, dict] # test non-monai modules and excludes -TEST_CASE_6 = [{"_path_": "torch.optim.Adam", "params": torch.nn.PReLU().parameters(), "lr": 1e-4}, torch.optim.Adam] +TEST_CASE_6 = [{"_name_": "torch.optim.Adam", "params": torch.nn.PReLU().parameters(), "lr": 1e-4}, torch.optim.Adam] TEST_CASE_7 = [{"_name_": "decollate_batch", "detach": True, "pad": True}, partial] # test args contains "name" field TEST_CASE_8 = [ diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index c57ca48cbd..eab10172d9 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -64,9 +64,9 @@ def __call__(self, a, b): "cls_func": "$TestClass.cls_compute", "lambda_static_func": "$lambda x, y: TestClass.compute(x, y)", "lambda_cls_func": "$lambda x, y: TestClass.cls_compute(x, y)", - "compute": {"_path_": "tests.test_config_parser.TestClass.compute", "func": "@basic_func"}, - "cls_compute": {"_path_": "tests.test_config_parser.TestClass.cls_compute", "func": "@basic_func"}, - "call_compute": {"_path_": "tests.test_config_parser.TestClass"}, + "compute": {"_name_": "tests.test_config_parser.TestClass.compute", "func": "@basic_func"}, + "cls_compute": {"_name_": "tests.test_config_parser.TestClass.cls_compute", "func": "@basic_func"}, + "call_compute": {"_name_": "tests.test_config_parser.TestClass"}, "error_func": "$TestClass.__call__", "": "$lambda x, y: x + y", } From 2780530a3cade26edb9acc16877e7d01bb37b6d2 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Fri, 11 Mar 2022 10:48:57 +0800 Subject: [PATCH 75/76] [DLMED] update to "_target_" Signed-off-by: Nic Ma --- monai/bundle/config_item.py | 18 +++++++++--------- monai/bundle/config_parser.py | 6 +++--- monai/bundle/reference_resolver.py | 2 +- tests/test_bundle_run.py | 4 ++-- tests/test_config_item.py | 16 ++++++++-------- tests/test_config_parser.py | 26 +++++++++++++------------- tests/test_reference_resolver.py | 16 ++++++++-------- tests/testing_data/inference.json | 30 +++++++++++++++--------------- tests/testing_data/inference.yaml | 30 +++++++++++++++--------------- 9 files changed, 74 insertions(+), 74 deletions(-) diff --git a/monai/bundle/config_item.py b/monai/bundle/config_item.py index d959006fde..cc6f1ed01e 100644 --- a/monai/bundle/config_item.py +++ b/monai/bundle/config_item.py @@ -167,8 +167,8 @@ class ConfigComponent(ConfigItem, Instantiable): Currently, four special keys (strings surrounded by ``_``) are defined and interpreted beyond the regular literals: - class or function identifier of the python module, specified by one of the two keys. - - ``"_name_"``: indicates build-in python classes or functions such as "LoadImageDict", - or full module name, such as "monai.transforms.LoadImageDict". + - ``"_target_"``: indicates build-in python classes or functions such as "LoadImageDict", + or full module name, such as "monai.transforms.LoadImageDict". - ``"_disabled_"``: a flag to indicate whether to skip the instantiation. Other fields in the config content are input arguments to the python module. @@ -177,7 +177,7 @@ class ConfigComponent(ConfigItem, Instantiable): locator = ComponentLocator(excludes=["modules_to_exclude"]) config = { - "_name_": "LoadImaged", + "_target_": "LoadImaged", "keys": ["image", "label"] } @@ -195,7 +195,7 @@ class ConfigComponent(ConfigItem, Instantiable): """ - non_arg_keys = {"_name_", "_disabled_"} + non_arg_keys = {"_target_", "_disabled_"} def __init__( self, @@ -216,18 +216,18 @@ def is_instantiable(config: Any) -> bool: config: input config content to check. """ - return isinstance(config, Mapping) and "_name_" in config + return isinstance(config, Mapping) and "_target_" in config def resolve_module_name(self): """ Resolve the target module name from current config content. - The config content must have ``"_name_"`` key. + The config content must have ``"_target_"`` key. """ config = dict(self.get_config()) - target = config.get("_name_") + target = config.get("_target_") if not isinstance(target, str): - raise ValueError("must provide a string for the `_name_` of component to instantiate.") + raise ValueError("must provide a string for the `_target_` of component to instantiate.") module = self.locator.get_component_module_name(target) if module is None: @@ -237,7 +237,7 @@ def resolve_module_name(self): if isinstance(module, list): warnings.warn( f"there are more than 1 component have name `{target}`: {module}, use the first one `{module[0]}." - f" if want to use others, please set its full module path in `_name_` directly." + f" if want to use others, please set its full module path in `_target_` directly." ) module = module[0] return f"{module}.{target}" diff --git a/monai/bundle/config_parser.py b/monai/bundle/config_parser.py index 9a3d24c7af..6fa7b3a2a2 100644 --- a/monai/bundle/config_parser.py +++ b/monai/bundle/config_parser.py @@ -45,9 +45,9 @@ class ConfigParser: config = { "my_dims": 2, "dims_1": "$@my_dims + 1", - "my_xform": {"_name_": "LoadImage"}, - "my_net": {"_name_": "BasicUNet", "spatial_dims": "@dims_1", "in_channels": 1, "out_channels": 4}, - "trainer": {"_name_": "SupervisedTrainer", "network": "@my_net", "preprocessing": "@my_xform"} + "my_xform": {"_target_": "LoadImage"}, + "my_net": {"_target_": "BasicUNet", "spatial_dims": "@dims_1", "in_channels": 1, "out_channels": 4}, + "trainer": {"_target_": "SupervisedTrainer", "network": "@my_net", "preprocessing": "@my_xform"} } # in the example $@my_dims + 1 is an expression, which adds 1 to the value of @my_dims parser = ConfigParser(config) diff --git a/monai/bundle/reference_resolver.py b/monai/bundle/reference_resolver.py index eee456028b..c1599c2124 100644 --- a/monai/bundle/reference_resolver.py +++ b/monai/bundle/reference_resolver.py @@ -48,7 +48,7 @@ class ReferenceResolver: _vars = "__local_refs" sep = ID_SEP_KEY # separator for key indexing ref = ID_REF_KEY # reference prefix - # match a reference string, e.g. "@id#key", "@id#key#0", "@_name_#key" + # match a reference string, e.g. "@id#key", "@id#key#0", "@_target_#key" id_matcher = re.compile(rf"{ref}(?:\w*)(?:{sep}\w*)*") def __init__(self, items: Optional[Sequence[ConfigItem]] = None): diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index 8d7812bd22..75002d3631 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -60,9 +60,9 @@ def test_shape(self, config_file, expected_shape): saver = LoadImage(image_only=True) if sys.platform == "win32": - override = "--network $@network_def.to(@device) --dataset#_name_ Dataset" + override = "--network $@network_def.to(@device) --dataset#_target_ Dataset" else: - override = f"--network %{overridefile1}#move_net --dataset#_name_ %{overridefile2}" + override = f"--network %{overridefile1}#move_net --dataset#_target_ %{overridefile2}" # test with `monai.bundle` as CLI entry directly cmd = "-m monai.bundle run --target_id evaluator" cmd += f" --postprocessing#transforms#2#output_postfix seg {override}" diff --git a/tests/test_config_item.py b/tests/test_config_item.py index 4f0d5f141c..7b43cd30ea 100644 --- a/tests/test_config_item.py +++ b/tests/test_config_item.py @@ -26,19 +26,19 @@ TEST_CASE_1 = [{"lr": 0.001}, 0.0001] -TEST_CASE_2 = [{"_name_": "LoadImaged", "keys": ["image"]}, LoadImaged] +TEST_CASE_2 = [{"_target_": "LoadImaged", "keys": ["image"]}, LoadImaged] # test full module path -TEST_CASE_3 = [{"_name_": "monai.transforms.LoadImaged", "keys": ["image"]}, LoadImaged] +TEST_CASE_3 = [{"_target_": "monai.transforms.LoadImaged", "keys": ["image"]}, LoadImaged] # test `_disabled_` -TEST_CASE_4 = [{"_name_": "LoadImaged", "_disabled_": True, "keys": ["image"]}, dict] +TEST_CASE_4 = [{"_target_": "LoadImaged", "_disabled_": True, "keys": ["image"]}, dict] # test `_disabled_` with string -TEST_CASE_5 = [{"_name_": "LoadImaged", "_disabled_": "true", "keys": ["image"]}, dict] +TEST_CASE_5 = [{"_target_": "LoadImaged", "_disabled_": "true", "keys": ["image"]}, dict] # test non-monai modules and excludes -TEST_CASE_6 = [{"_name_": "torch.optim.Adam", "params": torch.nn.PReLU().parameters(), "lr": 1e-4}, torch.optim.Adam] -TEST_CASE_7 = [{"_name_": "decollate_batch", "detach": True, "pad": True}, partial] +TEST_CASE_6 = [{"_target_": "torch.optim.Adam", "params": torch.nn.PReLU().parameters(), "lr": 1e-4}, torch.optim.Adam] +TEST_CASE_7 = [{"_target_": "decollate_batch", "detach": True, "pad": True}, partial] # test args contains "name" field TEST_CASE_8 = [ - {"_name_": "RandTorchVisiond", "keys": "image", "name": "ColorJitter", "brightness": 0.25}, + {"_target_": "RandTorchVisiond", "keys": "image", "name": "ColorJitter", "brightness": 0.25}, RandTorchVisiond, ] # test execute some function in args, test pre-imported global packages `monai` @@ -80,7 +80,7 @@ def test_expression(self, id, test_input): self.assertTrue(isinstance(ret, Callable)) def test_lazy_instantiation(self): - config = {"_name_": "DataLoader", "dataset": Dataset(data=[1, 2]), "batch_size": 2} + config = {"_target_": "DataLoader", "dataset": Dataset(data=[1, 2]), "batch_size": 2} configer = ConfigComponent(config=config, locator=None) init_config = configer.get_config() # modify config content at runtime diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index eab10172d9..ce98be1214 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -25,15 +25,15 @@ TEST_CASE_1 = [ { "transform": { - "_name_": "Compose", + "_target_": "Compose", "transforms": [ - {"_name_": "LoadImaged", "keys": "image"}, - {"_name_": "RandTorchVisiond", "keys": "image", "name": "ColorJitter", "brightness": 0.25}, + {"_target_": "LoadImaged", "keys": "image"}, + {"_target_": "RandTorchVisiond", "keys": "image", "name": "ColorJitter", "brightness": 0.25}, ], }, - "dataset": {"_name_": "Dataset", "data": [1, 2], "transform": "@transform"}, + "dataset": {"_target_": "Dataset", "data": [1, 2], "transform": "@transform"}, "dataloader": { - "_name_": "DataLoader", + "_target_": "DataLoader", "dataset": "@dataset", "batch_size": 2, "collate_fn": "monai.data.list_data_collate", @@ -64,9 +64,9 @@ def __call__(self, a, b): "cls_func": "$TestClass.cls_compute", "lambda_static_func": "$lambda x, y: TestClass.compute(x, y)", "lambda_cls_func": "$lambda x, y: TestClass.cls_compute(x, y)", - "compute": {"_name_": "tests.test_config_parser.TestClass.compute", "func": "@basic_func"}, - "cls_compute": {"_name_": "tests.test_config_parser.TestClass.cls_compute", "func": "@basic_func"}, - "call_compute": {"_name_": "tests.test_config_parser.TestClass"}, + "compute": {"_target_": "tests.test_config_parser.TestClass.compute", "func": "@basic_func"}, + "cls_compute": {"_target_": "tests.test_config_parser.TestClass.cls_compute", "func": "@basic_func"}, + "call_compute": {"_target_": "tests.test_config_parser.TestClass"}, "error_func": "$TestClass.__call__", "": "$lambda x, y: x + y", } @@ -75,17 +75,17 @@ def __call__(self, a, b): class TestConfigComponent(unittest.TestCase): def test_config_content(self): - test_config = {"preprocessing": [{"_name_": "LoadImage"}], "dataset": {"_name_": "Dataset"}} + test_config = {"preprocessing": [{"_target_": "LoadImage"}], "dataset": {"_target_": "Dataset"}} parser = ConfigParser(config=test_config) # test `get`, `set`, `__getitem__`, `__setitem__` self.assertEqual(str(parser.get()), str(test_config)) parser.set(config=test_config) self.assertListEqual(parser["preprocessing"], test_config["preprocessing"]) - parser["dataset"] = {"_name_": "CacheDataset"} - self.assertEqual(parser["dataset"]["_name_"], "CacheDataset") + parser["dataset"] = {"_target_": "CacheDataset"} + self.assertEqual(parser["dataset"]["_target_"], "CacheDataset") # test nested ids - parser["dataset#_name_"] = "Dataset" - self.assertEqual(parser["dataset#_name_"], "Dataset") + parser["dataset#_target_"] = "Dataset" + self.assertEqual(parser["dataset#_target_"], "Dataset") # test int id parser.set(["test1", "test2", "test3"]) parser[1] = "test4" diff --git a/tests/test_reference_resolver.py b/tests/test_reference_resolver.py index 66b8655402..e6b01c05f4 100644 --- a/tests/test_reference_resolver.py +++ b/tests/test_reference_resolver.py @@ -27,8 +27,8 @@ TEST_CASE_1 = [ { # all the recursively parsed config items - "transform#1": {"_name_": "LoadImaged", "keys": ["image"]}, - "transform#1#_name_": "LoadImaged", + "transform#1": {"_target_": "LoadImaged", "keys": ["image"]}, + "transform#1#_target_": "LoadImaged", "transform#1#keys": ["image"], "transform#1#keys#0": "image", }, @@ -39,12 +39,12 @@ TEST_CASE_2 = [ { # some the recursively parsed config items - "dataloader": {"_name_": "DataLoader", "dataset": "@dataset", "collate_fn": "$monai.data.list_data_collate"}, - "dataset": {"_name_": "Dataset", "data": [1, 2]}, - "dataloader#_name_": "DataLoader", + "dataloader": {"_target_": "DataLoader", "dataset": "@dataset", "collate_fn": "$monai.data.list_data_collate"}, + "dataset": {"_target_": "Dataset", "data": [1, 2]}, + "dataloader#_target_": "DataLoader", "dataloader#dataset": "@dataset", "dataloader#collate_fn": "$monai.data.list_data_collate", - "dataset#_name_": "Dataset", + "dataset#_target_": "Dataset", "dataset#data": [1, 2], "dataset#data#0": 1, "dataset#data#1": 2, @@ -56,8 +56,8 @@ TEST_CASE_3 = [ { # all the recursively parsed config items - "transform#1": {"_name_": "RandTorchVisiond", "keys": "image", "name": "ColorJitter", "brightness": 0.25}, - "transform#1#_name_": "RandTorchVisiond", + "transform#1": {"_target_": "RandTorchVisiond", "keys": "image", "name": "ColorJitter", "brightness": 0.25}, + "transform#1#_target_": "RandTorchVisiond", "transform#1#keys": "image", "transform#1#name": "ColorJitter", "transform#1#brightness": 0.25, diff --git a/tests/testing_data/inference.json b/tests/testing_data/inference.json index 608a62a97d..b96968496d 100644 --- a/tests/testing_data/inference.json +++ b/tests/testing_data/inference.json @@ -1,7 +1,7 @@ { "device": "$torch.device('cuda' if torch.cuda.is_available() else 'cpu')", "network_def": { - "_name_": "UNet", + "_target_": "UNet", "spatial_dims": 3, "in_channels": 1, "out_channels": 2, @@ -23,45 +23,45 @@ }, "network": "need override", "preprocessing": { - "_name_": "Compose", + "_target_": "Compose", "transforms": [ { - "_name_": "LoadImaged", + "_target_": "LoadImaged", "keys": "image" }, { - "_name_": "EnsureChannelFirstd", + "_target_": "EnsureChannelFirstd", "keys": "image" }, { - "_name_": "ScaleIntensityd", + "_target_": "ScaleIntensityd", "keys": "image" }, { - "_name_": "RandRotated", + "_target_": "RandRotated", "_disabled_": true, "keys": "image" }, { - "_name_": "EnsureTyped", + "_target_": "EnsureTyped", "keys": "image" } ] }, "dataset": { - "_name_": "need override", + "_target_": "need override", "data": "@_meta_#datalist", "transform": "@preprocessing" }, "dataloader": { - "_name_": "DataLoader", + "_target_": "DataLoader", "dataset": "@dataset", "batch_size": 1, "shuffle": false, "num_workers": 4 }, "inferer": { - "_name_": "SlidingWindowInferer", + "_target_": "SlidingWindowInferer", "roi_size": [ 96, 96, @@ -71,20 +71,20 @@ "overlap": 0.5 }, "postprocessing": { - "_name_": "Compose", + "_target_": "Compose", "transforms": [ { - "_name_": "Activationsd", + "_target_": "Activationsd", "keys": "pred", "softmax": true }, { - "_name_": "AsDiscreted", + "_target_": "AsDiscreted", "keys": "pred", "argmax": true }, { - "_name_": "SaveImaged", + "_target_": "SaveImaged", "keys": "pred", "meta_keys": "image_meta_dict", "output_dir": "@_meta_#output_dir" @@ -92,7 +92,7 @@ ] }, "evaluator": { - "_name_": "SupervisedEvaluator", + "_target_": "SupervisedEvaluator", "device": "@device", "val_data_loader": "@dataloader", "network": "@network", diff --git a/tests/testing_data/inference.yaml b/tests/testing_data/inference.yaml index 325d69e749..58eeca8191 100644 --- a/tests/testing_data/inference.yaml +++ b/tests/testing_data/inference.yaml @@ -1,7 +1,7 @@ --- device: "$torch.device('cuda' if torch.cuda.is_available() else 'cpu')" network_def: - _name_: UNet + _target_: UNet spatial_dims: 3 in_channels: 1 out_channels: 2 @@ -20,31 +20,31 @@ network_def: norm: batch network: need override preprocessing: - _name_: Compose + _target_: Compose transforms: - - _name_: LoadImaged + - _target_: LoadImaged keys: image - - _name_: EnsureChannelFirstd + - _target_: EnsureChannelFirstd keys: image - - _name_: ScaleIntensityd + - _target_: ScaleIntensityd keys: image - - _name_: RandRotated + - _target_: RandRotated _disabled_: true keys: image - - _name_: EnsureTyped + - _target_: EnsureTyped keys: image dataset: - _name_: need override + _target_: need override data: "@_meta_#datalist" transform: "@preprocessing" dataloader: - _name_: DataLoader + _target_: DataLoader dataset: "@dataset" batch_size: 1 shuffle: false num_workers: 4 inferer: - _name_: SlidingWindowInferer + _target_: SlidingWindowInferer roi_size: - 96 - 96 @@ -52,20 +52,20 @@ inferer: sw_batch_size: 4 overlap: 0.5 postprocessing: - _name_: Compose + _target_: Compose transforms: - - _name_: Activationsd + - _target_: Activationsd keys: pred softmax: true - - _name_: AsDiscreted + - _target_: AsDiscreted keys: pred argmax: true - - _name_: SaveImaged + - _target_: SaveImaged keys: pred meta_keys: image_meta_dict output_dir: "@_meta_#output_dir" evaluator: - _name_: SupervisedEvaluator + _target_: SupervisedEvaluator device: "@device" val_data_loader: "@dataloader" network: "@network" From 194f21e05cb11fe1f868daf3517e4297032581e6 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 11 Mar 2022 07:46:28 +0000 Subject: [PATCH 76/76] fixes typos Signed-off-by: Wenqi Li --- monai/bundle/config_item.py | 2 +- monai/bundle/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/bundle/config_item.py b/monai/bundle/config_item.py index cc6f1ed01e..807b369f5d 100644 --- a/monai/bundle/config_item.py +++ b/monai/bundle/config_item.py @@ -164,7 +164,7 @@ class ConfigComponent(ConfigItem, Instantiable): Subclass of :py:class:`monai.bundle.ConfigItem`, this class uses a dictionary with string keys to represent a component of `class` or `function` and supports instantiation. - Currently, four special keys (strings surrounded by ``_``) are defined and interpreted beyond the regular literals: + Currently, two special keys (strings surrounded by ``_``) are defined and interpreted beyond the regular literals: - class or function identifier of the python module, specified by one of the two keys. - ``"_target_"``: indicates build-in python classes or functions such as "LoadImageDict", diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 75a0ebc2a2..ba5c2729e7 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -14,5 +14,5 @@ ID_REF_KEY = "@" # start of a reference to a ConfigItem ID_SEP_KEY = "#" # separator for the ID of a ConfigItem -EXPR_KEY = "$" # start of an ConfigExpression +EXPR_KEY = "$" # start of a ConfigExpression MACRO_KEY = "%" # start of a macro of a config