From 36884d2b924c12b8a24d5194a2df2c5da619401f Mon Sep 17 00:00:00 2001 From: Chun Cai Date: Wed, 27 Nov 2024 12:52:18 +0800 Subject: [PATCH 1/8] Perf: remove redundant checks on data integrity --- deepmd/utils/data_system.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/deepmd/utils/data_system.py b/deepmd/utils/data_system.py index 0e960d0ba1..0aea12e2b1 100644 --- a/deepmd/utils/data_system.py +++ b/deepmd/utils/data_system.py @@ -758,20 +758,9 @@ def process_systems(systems: Union[str, list[str]]) -> list[str]: help_msg = "Please check your setting for data systems" # check length of systems if len(systems) == 0: - msg = "cannot find valid a data system" + msg = "Can not find any valid data systems" log.fatal(msg) raise OSError(msg, help_msg) - # roughly check all items in systems are valid - for ii in systems: - ii = DPPath(ii) - if not ii.is_dir(): - msg = f"dir {ii} is not a valid dir" - log.fatal(msg) - raise OSError(msg, help_msg) - if not (ii / "type.raw").is_file(): - msg = f"dir {ii} is not a valid data system dir" - log.fatal(msg) - raise OSError(msg, help_msg) return systems From beab41ca42028c052cbab5331f88c1dd114c83c3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 27 Nov 2024 04:58:26 +0000 Subject: [PATCH 2/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- deepmd/utils/data_system.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/deepmd/utils/data_system.py b/deepmd/utils/data_system.py index 0aea12e2b1..fb3521be20 100644 --- a/deepmd/utils/data_system.py +++ b/deepmd/utils/data_system.py @@ -28,9 +28,6 @@ from deepmd.utils.out_stat import ( compute_stats_from_redu, ) -from deepmd.utils.path import ( - DPPath, -) log = logging.getLogger(__name__) From 0e1944a94aeb69b2d7369f5ba86e469f8371e110 Mon Sep 17 00:00:00 2001 From: Chun Cai Date: Wed, 27 Nov 2024 14:44:19 +0800 Subject: [PATCH 3/8] Update deepmd/utils/data_system.py Co-authored-by: Jinzhe Zeng Signed-off-by: Chun Cai --- deepmd/utils/data_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepmd/utils/data_system.py b/deepmd/utils/data_system.py index fb3521be20..709bc3c06b 100644 --- a/deepmd/utils/data_system.py +++ b/deepmd/utils/data_system.py @@ -755,7 +755,7 @@ def process_systems(systems: Union[str, list[str]]) -> list[str]: help_msg = "Please check your setting for data systems" # check length of systems if len(systems) == 0: - msg = "Can not find any valid data systems" + msg = "Cannot find any valid data systems" log.fatal(msg) raise OSError(msg, help_msg) return systems From b093fd73f1c98fea30e41eb6913c791ccdca6c6b Mon Sep 17 00:00:00 2001 From: caic99 Date: Fri, 29 Nov 2024 16:32:24 +0800 Subject: [PATCH 4/8] add error message --- deepmd/utils/data_system.py | 8 ++------ deepmd/utils/path.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/deepmd/utils/data_system.py b/deepmd/utils/data_system.py index 709bc3c06b..7b2e711435 100644 --- a/deepmd/utils/data_system.py +++ b/deepmd/utils/data_system.py @@ -100,6 +100,7 @@ def __init__( del rcut self.system_dirs = systems self.nsystems = len(self.system_dirs) + assert self.nsystems > 0, "No systems provided" self.data_systems = [] for ii in self.system_dirs: self.data_systems.append( @@ -752,12 +753,7 @@ def process_systems(systems: Union[str, list[str]]) -> list[str]: systems = expand_sys_str(systems) elif isinstance(systems, list): systems = systems.copy() - help_msg = "Please check your setting for data systems" - # check length of systems - if len(systems) == 0: - msg = "Cannot find any valid data systems" - log.fatal(msg) - raise OSError(msg, help_msg) + assert systems, "No systems provided" return systems diff --git a/deepmd/utils/path.py b/deepmd/utils/path.py index 6c52caac1d..87e6940682 100644 --- a/deepmd/utils/path.py +++ b/deepmd/utils/path.py @@ -14,6 +14,7 @@ from typing import ( ClassVar, Optional, + Union, ) import h5py @@ -157,19 +158,18 @@ class DPOSPath(DPPath): Parameters ---------- - path : str + path : Union[str, Path] path mode : str, optional mode, by default "r" """ - def __init__(self, path: str, mode: str = "r") -> None: + def __init__(self, path: Union[str, Path], mode: str = "r") -> None: super().__init__() self.mode = mode - if isinstance(path, Path): - self.path = path - else: - self.path = Path(path) + self.path = Path(path) + if not self.path.exists(): + raise FileNotFoundError(f"{self.path} not found") def load_numpy(self) -> np.ndarray: """Load NumPy array. @@ -300,6 +300,8 @@ def __init__(self, path: str, mode: str = "r") -> None: # so we do not support file names containing #... s = path.split("#") self.root_path = s[0] + if not os.path.isfile(self.root_path): + raise FileNotFoundError(f"{self.root_path} not found") self.root = self._load_h5py(s[0], mode) # h5 path: default is the root path self._name = s[1] if len(s) > 1 else "/" From 90ffb614feb16d4925dcfe6f3363fe91cd3aa38e Mon Sep 17 00:00:00 2001 From: Chun Cai Date: Sat, 30 Nov 2024 11:01:44 +0800 Subject: [PATCH 5/8] Update deepmd/utils/data_system.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: Chun Cai --- deepmd/utils/data_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepmd/utils/data_system.py b/deepmd/utils/data_system.py index 7b2e711435..1df386367b 100644 --- a/deepmd/utils/data_system.py +++ b/deepmd/utils/data_system.py @@ -753,7 +753,7 @@ def process_systems(systems: Union[str, list[str]]) -> list[str]: systems = expand_sys_str(systems) elif isinstance(systems, list): systems = systems.copy() - assert systems, "No systems provided" + return systems return systems From 73314728f37aa6c7f8ced322787c08d8d1fcf35a Mon Sep 17 00:00:00 2001 From: Chun Cai Date: Sat, 30 Nov 2024 11:03:02 +0800 Subject: [PATCH 6/8] remove redundant checks Signed-off-by: Chun Cai --- deepmd/utils/data_system.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deepmd/utils/data_system.py b/deepmd/utils/data_system.py index 1df386367b..15371cf1a8 100644 --- a/deepmd/utils/data_system.py +++ b/deepmd/utils/data_system.py @@ -754,7 +754,6 @@ def process_systems(systems: Union[str, list[str]]) -> list[str]: elif isinstance(systems, list): systems = systems.copy() return systems - return systems def get_data( From fd59449b3a085e7e4bca8a7d87f3b62b6fb1a9bd Mon Sep 17 00:00:00 2001 From: Chun Cai Date: Sat, 30 Nov 2024 12:12:33 +0800 Subject: [PATCH 7/8] Update deepmd/utils/data_system.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: Chun Cai --- deepmd/utils/data_system.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deepmd/utils/data_system.py b/deepmd/utils/data_system.py index 15371cf1a8..445461b387 100644 --- a/deepmd/utils/data_system.py +++ b/deepmd/utils/data_system.py @@ -100,7 +100,8 @@ def __init__( del rcut self.system_dirs = systems self.nsystems = len(self.system_dirs) - assert self.nsystems > 0, "No systems provided" + if self.nsystems <= 0: + raise ValueError("No systems provided") self.data_systems = [] for ii in self.system_dirs: self.data_systems.append( From 89a38824e9331860178a46b40ac03e89b36e6541 Mon Sep 17 00:00:00 2001 From: caic99 Date: Mon, 2 Dec 2024 11:02:27 +0800 Subject: [PATCH 8/8] fix ut --- deepmd/utils/data.py | 2 ++ deepmd/utils/path.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deepmd/utils/data.py b/deepmd/utils/data.py index 493a9d8d54..39af73cab3 100644 --- a/deepmd/utils/data.py +++ b/deepmd/utils/data.py @@ -60,6 +60,8 @@ def __init__( ) -> None: """Constructor.""" root = DPPath(sys_path) + if not root.is_dir(): + raise FileNotFoundError(f"System {sys_path} is not found!") self.dirs = root.glob(set_prefix + ".*") if not len(self.dirs): raise FileNotFoundError(f"No {set_prefix}.* is found in {sys_path}") diff --git a/deepmd/utils/path.py b/deepmd/utils/path.py index 87e6940682..c542ccf661 100644 --- a/deepmd/utils/path.py +++ b/deepmd/utils/path.py @@ -168,8 +168,6 @@ def __init__(self, path: Union[str, Path], mode: str = "r") -> None: super().__init__() self.mode = mode self.path = Path(path) - if not self.path.exists(): - raise FileNotFoundError(f"{self.path} not found") def load_numpy(self) -> np.ndarray: """Load NumPy array.