From f015ea120e241330b6ad12636656846c8cc157fe Mon Sep 17 00:00:00 2001 From: Han Wang Date: Sat, 11 Jan 2020 17:21:11 +0800 Subject: [PATCH 01/22] fix bug of ndim when nframe==1 --- data/raw/raw_to_set.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/data/raw/raw_to_set.sh b/data/raw/raw_to_set.sh index 58f2ab4ce5..8e4b917adb 100755 --- a/data/raw/raw_to_set.sh +++ b/data/raw/raw_to_set.sh @@ -36,8 +36,8 @@ do test -f fparam.raw$pi && mv fparam.raw$pi set.$pi/fparam.raw cd set.$pi - python -c 'import numpy as np; data = np.loadtxt("box.raw" ); data = data.astype (np.float32); np.save ("box", data)' - python -c 'import numpy as np; data = np.loadtxt("coord.raw" ); data = data.astype (np.float32); np.save ("coord", data)' + python -c 'import numpy as np; data = np.loadtxt("box.raw" , ndmin = 2); data = data.astype (np.float32); np.save ("box", data)' + python -c 'import numpy as np; data = np.loadtxt("coord.raw" , ndmin = 2); data = data.astype (np.float32); np.save ("coord", data)' python -c \ 'import numpy as np; import os.path; if os.path.isfile("energy.raw"): @@ -48,28 +48,28 @@ if os.path.isfile("energy.raw"): python -c \ 'import numpy as np; import os.path; if os.path.isfile("force.raw" ): - data = np.loadtxt("force.raw" ); + data = np.loadtxt("force.raw", ndmin = 2); data = data.astype (np.float32); np.save ("force", data) ' python -c \ 'import numpy as np; import os.path; if os.path.isfile("virial.raw"): - data = np.loadtxt("virial.raw"); + data = np.loadtxt("virial.raw", ndmin = 2); data = data.astype (np.float32); np.save ("virial", data) ' python -c \ 'import numpy as np; import os.path; if os.path.isfile("atom_ener.raw"): - data = np.loadtxt("atom_ener.raw"); + data = np.loadtxt("atom_ener.raw", ndmin = 2); data = data.astype (np.float32); np.save ("atom_ener", data) ' python -c \ 'import numpy as np; import os.path; if os.path.isfile("fparam.raw"): - data = np.loadtxt("fparam.raw"); + data = np.loadtxt("fparam.raw", ndmin = 2); data = data.astype (np.float32); np.save ("fparam", data) ' From 5f770195991ee3cbb5a741e6020b35503aeb3436 Mon Sep 17 00:00:00 2001 From: Han Wang Date: Wed, 22 Jan 2020 10:16:49 +0800 Subject: [PATCH 02/22] fix bug of test data --- source/train/test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/train/test.py b/source/train/test.py index 83f882e32b..47ecaa9128 100755 --- a/source/train/test.py +++ b/source/train/test.py @@ -6,7 +6,6 @@ import argparse import numpy as np -from deepmd.Data import DataSets from deepmd.Data import DeepmdData from deepmd import DeepEval from deepmd import DeepPot @@ -35,13 +34,15 @@ def test_ener (args) : if args.rand_seed is not None : np.random.seed(args.rand_seed % (2**32)) - data = DataSets (args.system, args.set_prefix, shuffle_test = args.shuffle_test) + dp = DeepPot(args.model) + data = DeepmdData(args.system, args.set_prefix, shuffle_test = args.shuffle_test, type_map = dp.get_type_map()) + test_data = data.get_test () numb_test = args.numb_test natoms = len(test_data["type"][0]) nframes = test_data["box"].shape[0] numb_test = min(nframes, numb_test) - dp = DeepPot(args.model) + coord = test_data["coord"][:numb_test].reshape([numb_test, -1]) box = test_data["box"][:numb_test] atype = test_data["type"][0] From 815d716cf5adadab03dea3607aa9b3ad2cca69cd Mon Sep 17 00:00:00 2001 From: Lu Date: Mon, 6 Jan 2020 20:06:49 +0800 Subject: [PATCH 03/22] fix bug of "-l-ldeepmd" --- source/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index d83ffe9e0e..4c43fc9227 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -173,7 +173,7 @@ if (BUILD_CPP_IF) if (USE_CUDA_TOOLKIT) set (LIB_DEEPMD_OP_CUDA "deepmd_op_cuda") else() - set (LIB_DEEPMD_OP_CUDA "") + set (LIB_DEEPMD_OP_CUDA "deepmd_op") endif() if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 4.9) set (LIB_DEEPMD_NATIVE "deepmd_native_md") From 98ec02b0bd3e95e2a8f7c7519e8f9f3da2524a4c Mon Sep 17 00:00:00 2001 From: Han Wang Date: Thu, 23 Jan 2020 07:42:32 +0800 Subject: [PATCH 04/22] fix bug of undefined get_rcut and get_ntypes --- source/lib/include/NNPInter.h | 8 ++++---- source/lib/src/NNPInter.cc | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/lib/include/NNPInter.h b/source/lib/include/NNPInter.h index 20e9bd8aaa..92d0587eea 100644 --- a/source/lib/include/NNPInter.h +++ b/source/lib/include/NNPInter.h @@ -112,8 +112,8 @@ class NNPInter GraphDef graph_def; bool inited; template VT get_scalar(const string & name) const; - VALUETYPE get_rcut () const; - int get_ntypes () const; + // VALUETYPE get_rcut () const; + // int get_ntypes () const; VALUETYPE rcut; VALUETYPE cell_size; int ntypes; @@ -210,8 +210,8 @@ class NNPInterModelDevi vector graph_defs; bool inited; template VT get_scalar(const string name) const; - VALUETYPE get_rcut () const; - int get_ntypes () const; + // VALUETYPE get_rcut () const; + // int get_ntypes () const; VALUETYPE rcut; VALUETYPE cell_size; int ntypes; diff --git a/source/lib/src/NNPInter.cc b/source/lib/src/NNPInter.cc index 47a66893c3..ae72db8487 100644 --- a/source/lib/src/NNPInter.cc +++ b/source/lib/src/NNPInter.cc @@ -849,8 +849,8 @@ init (const string & model, const int & gpu_rank) ntypes = get_scalar("descrpt_attr/ntypes"); dfparam = get_scalar("fitting_attr/dfparam"); daparam = get_scalar("fitting_attr/daparam"); - assert(rcut == get_rcut()); - assert(ntypes == get_ntypes()); + // assert(rcut == get_rcut()); + // assert(ntypes == get_ntypes()); if (dfparam < 0) dfparam = 0; if (daparam < 0) daparam = 0; inited = true; @@ -880,8 +880,8 @@ init (const string & model, const int & gpu_rank) ntypes = get_scalar("descrpt_attr/ntypes"); dfparam = get_scalar("fitting_attr/dfparam"); daparam = get_scalar("fitting_attr/daparam"); - assert(rcut == get_rcut()); - assert(ntypes == get_ntypes()); + // assert(rcut == get_rcut()); + // assert(ntypes == get_ntypes()); if (dfparam < 0) dfparam = 0; if (daparam < 0) daparam = 0; // rcut = get_rcut(); From 3b82e23878ffc281b3e1db5688251c7b3066c65b Mon Sep 17 00:00:00 2001 From: Han Wang Date: Thu, 23 Jan 2020 07:46:21 +0800 Subject: [PATCH 05/22] do not upgrade pip --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 48aa742722..21b69b37ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,7 +72,7 @@ matrix: - CXX=g++-8 - TENSORFLOW_VERSION=2.0 before_install: - - pip install --upgrade pip +# - pip install --upgrade pip - pip install --upgrade setuptools - pip install tensorflow==$TENSORFLOW_VERSION install: From ac6888189f259484027352789bb53c3f2213c403 Mon Sep 17 00:00:00 2001 From: Lu Date: Thu, 23 Jan 2020 15:05:48 +0800 Subject: [PATCH 06/22] fix bug of 0 output when sec_a.back() is lager than 1024 --- source/op/cuda/descrpt_se_r.cu | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/source/op/cuda/descrpt_se_r.cu b/source/op/cuda/descrpt_se_r.cu index cc7dd4e904..2a4a126166 100644 --- a/source/op/cuda/descrpt_se_r.cu +++ b/source/op/cuda/descrpt_se_r.cu @@ -209,14 +209,16 @@ __global__ void compute_descriptor_se_r (VALUETYPE* descript, const VALUETYPE* coord, const VALUETYPE rmin, const VALUETYPE rmax, - compute_t* sel_diff_dev) + compute_t* sel_diff_dev, + const int sec_size) { // <<>> - const unsigned int idx = blockIdx.x; - const unsigned int idy = threadIdx.x; + const unsigned int idx = blockIdx.y; + const unsigned int idy = blockIdx.x * blockDim.x + threadIdx.x; const int idx_deriv = idy * 3; // 1 components time 3 directions const int idx_value = idy; // 1 components - + if (idy >= sec_size) {return;} + // else {return;} VALUETYPE * row_descript = descript + idx * ndescrpt; VALUETYPE * row_descript_deriv = descript_deriv + idx * descript_deriv_size; @@ -324,7 +326,9 @@ void DescrptSeRLauncher(const VALUETYPE* coord, nei_iter ); } - compute_descriptor_se_r<<>> ( + const int nblock_ = (sec.back() + LEN -1) / LEN; + dim3 block_grid(nblock_, nloc); + compute_descriptor_se_r<<>> ( descript, ndescrpt, descript_deriv, @@ -339,6 +343,7 @@ void DescrptSeRLauncher(const VALUETYPE* coord, coord, rcut_smth, rcut, - sel_diff + sel_diff, + sec.back() ); } \ No newline at end of file From d1fa044e375ada16a9d889da83aceed028ea8f5f Mon Sep 17 00:00:00 2001 From: Lu Date: Thu, 23 Jan 2020 13:02:33 +0800 Subject: [PATCH 07/22] fix bug of 0 output when sec_a.back() is lager than 1024 --- source/op/cuda/descrpt_se_a.cu | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/source/op/cuda/descrpt_se_a.cu b/source/op/cuda/descrpt_se_a.cu index 8893e8a00a..8b6b3ee575 100644 --- a/source/op/cuda/descrpt_se_a.cu +++ b/source/op/cuda/descrpt_se_a.cu @@ -208,13 +208,15 @@ __global__ void compute_descriptor_se_a (VALUETYPE* descript, const VALUETYPE* coord, const VALUETYPE rmin, const VALUETYPE rmax, - compute_t* sel_a_diff_dev) + compute_t* sel_a_diff_dev, + const int sec_a_size) { // <<>> - const unsigned int idx = blockIdx.x; - const unsigned int idy = threadIdx.x; + const unsigned int idx = blockIdx.y; + const unsigned int idy = blockIdx.x * blockDim.x + threadIdx.x; const int idx_deriv = idy * 4 * 3; // 4 components time 3 directions const int idx_value = idy * 4; // 4 components + if (idy >= sec_a_size) {return;} // else {return;} VALUETYPE * row_descript = descript + idx * ndescrpt; @@ -355,7 +357,9 @@ void DescrptSeALauncher(const VALUETYPE* coord, ); } - compute_descriptor_se_a<<>> ( + const int nblock_ = (sec_a.back() + LEN -1) / LEN; + dim3 block_grid(nblock_, nloc); + compute_descriptor_se_a<<>> ( descript, ndescrpt, descript_deriv, @@ -370,7 +374,8 @@ void DescrptSeALauncher(const VALUETYPE* coord, coord, rcut_r_smth, rcut_r, - sel_a_diff + sel_a_diff, + sec_a.back() ); //// // res = cudaFree(sec_a_dev); cudaErrcheck(res); From 0d1a3bc1bd4e68114f7c1467bcf9ab4eb8132fb1 Mon Sep 17 00:00:00 2001 From: Han Wang Date: Fri, 31 Jan 2020 10:47:43 +0800 Subject: [PATCH 08/22] fix bug #165 --- source/train/test.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) mode change 100755 => 100644 source/train/test.py diff --git a/source/train/test.py b/source/train/test.py old mode 100755 new mode 100644 index 47ecaa9128..a5a273de08 --- a/source/train/test.py +++ b/source/train/test.py @@ -36,13 +36,19 @@ def test_ener (args) : dp = DeepPot(args.model) data = DeepmdData(args.system, args.set_prefix, shuffle_test = args.shuffle_test, type_map = dp.get_type_map()) + data.add('energy', 1, atomic=False, must=False, high_prec=True) + data.add('force', 3, atomic=True, must=False, high_prec=False) + data.add('virial', 9, atomic=False, must=False, high_prec=False) + if dp.get_dim_fparam() > 0: + data.add('fparam', dp.get_dim_fparam(), atomic=False, must=True, high_prec=False) + if dp.get_dim_aparam() > 0: + data.add('aparam', dp.get_dim_aparam(), atomic=True, must=True, high_prec=False) test_data = data.get_test () - numb_test = args.numb_test natoms = len(test_data["type"][0]) nframes = test_data["box"].shape[0] + numb_test = args.numb_test numb_test = min(nframes, numb_test) - coord = test_data["coord"][:numb_test].reshape([numb_test, -1]) box = test_data["box"][:numb_test] atype = test_data["type"][0] @@ -54,6 +60,7 @@ def test_ener (args) : aparam = test_data["aparam"][:numb_test] else : aparam = None + energy, force, virial, ae, av = dp.eval(coord, box, atype, fparam = fparam, aparam = aparam, atomic = True) energy = energy.reshape([numb_test,1]) force = force.reshape([numb_test,-1]) From cb25563ca19c4027a1a730ee8673e7bfff503947 Mon Sep 17 00:00:00 2001 From: Han Wang Date: Sat, 1 Feb 2020 10:24:26 +0800 Subject: [PATCH 09/22] std should be set to 1 if its value vanishes --- source/train/DescrptSeA.py | 5 ++++- source/train/DescrptSeR.py | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/source/train/DescrptSeA.py b/source/train/DescrptSeA.py index 527dff2cce..fb914ea378 100644 --- a/source/train/DescrptSeA.py +++ b/source/train/DescrptSeA.py @@ -289,7 +289,10 @@ def _compute_dstats_sys_smth (self, def _compute_std (self,sumv2, sumv, sumn) : - return np.sqrt(sumv2/sumn - np.multiply(sumv/sumn, sumv/sumn)) + val = np.sqrt(sumv2/sumn - np.multiply(sumv/sumn, sumv/sumn)) + if val == 0: + val = 1 + return val def _filter(self, diff --git a/source/train/DescrptSeR.py b/source/train/DescrptSeR.py index aaa91ca220..5c60916d86 100644 --- a/source/train/DescrptSeR.py +++ b/source/train/DescrptSeR.py @@ -246,7 +246,11 @@ def _compute_dstats_sys_se_r (self, def _compute_std (self,sumv2, sumv, sumn) : - return np.sqrt(sumv2/sumn - np.multiply(sumv/sumn, sumv/sumn)) + val = np.sqrt(sumv2/sumn - np.multiply(sumv/sumn, sumv/sumn)) + if val == 0: + val = 1 + return val + def _filter_r(self, inputs, From b29d6d3c96ac7bea8cd99bcce70f922006fb1ec9 Mon Sep 17 00:00:00 2001 From: Han Wang Date: Sat, 1 Feb 2020 10:31:00 +0800 Subject: [PATCH 10/22] protect the descrpt to 1e-2 rather than set to 1 --- source/train/DescrptSeA.py | 4 ++-- source/train/DescrptSeR.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/train/DescrptSeA.py b/source/train/DescrptSeA.py index fb914ea378..dfe7329445 100644 --- a/source/train/DescrptSeA.py +++ b/source/train/DescrptSeA.py @@ -290,8 +290,8 @@ def _compute_dstats_sys_smth (self, def _compute_std (self,sumv2, sumv, sumn) : val = np.sqrt(sumv2/sumn - np.multiply(sumv/sumn, sumv/sumn)) - if val == 0: - val = 1 + if np.abs(val) < 1e-2: + val = 1e-2 return val diff --git a/source/train/DescrptSeR.py b/source/train/DescrptSeR.py index 5c60916d86..f7f8147a6c 100644 --- a/source/train/DescrptSeR.py +++ b/source/train/DescrptSeR.py @@ -247,8 +247,8 @@ def _compute_dstats_sys_se_r (self, def _compute_std (self,sumv2, sumv, sumn) : val = np.sqrt(sumv2/sumn - np.multiply(sumv/sumn, sumv/sumn)) - if val == 0: - val = 1 + if np.abs(val) < 1e-2: + val = 1e-2 return val From e015eac083f6c4628fd79626e5e212493c48b1fe Mon Sep 17 00:00:00 2001 From: Han Wang Date: Sat, 23 May 2020 21:44:35 +0800 Subject: [PATCH 11/22] fix bug in test --- source/train/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/train/test.py b/source/train/test.py index 2674e9c01e..ef71b684da 100644 --- a/source/train/test.py +++ b/source/train/test.py @@ -50,7 +50,7 @@ def test (args): err_coll.append(err) siz_coll.append(siz) avg_err = weighted_average(err_coll, siz_coll) - if len(all_sys) != len(err): + if len(all_sys) != len(err_coll): print('Not all systems are tested! Check if the systems are valid') if len(all_sys) > 1: print ("# ----------weighted average of errors----------- ") From 567bcff2a99ab50b9b1ba5066575da1a99a4c1b8 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 23 May 2020 12:44:21 -0400 Subject: [PATCH 12/22] cmake is a requirement only if it's not installed --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8c6a335ad4..34172a67ee 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ tf_install_dir = imp.find_module('tensorflow', [site_packages_path])[1] install_requires=['numpy', 'scipy'] -setup_requires=['setuptools_scm', 'scikit-build', 'cmake'] +setup_requires=['setuptools_scm', 'scikit-build'] # add cmake as a build requirement if cmake>3.0 is not installed try: From 507062eaead115783873a310bdfe7d8c017ee9fe Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 5 Sep 2020 03:18:08 -0400 Subject: [PATCH 13/22] Update README.md --- README.md | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 178758f5db..37ab751c7f 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ - [Deep Potential in a nutshell](#deep-potential-in-a-nutshell) - [Download and install](#download-and-install) - [Easy installation methods](#easy-installation-methods) + - [Offline packages](#offline-packages) - [With Docker](#with-docker) - [With conda](#with-conda) - - [Offline packages](#offline-packages) - [Install the python interaction](#install-the-python-interface) - [Install the Tensorflow's python interface](#install-the-tensorflows-python-interface) - [Install the DeePMD-kit's python interface](#install-the-deepmd-kits-python-interface) @@ -90,8 +90,10 @@ Please follow our [github](https://github.com/deepmodeling/deepmd-kit) webpage t ## Easy installation methods There various easy methods to install DeePMD-kit. Choose one that you prefer. If you want to build by yourself, jump to the next two sections. -### With Docker -A docker for installing the DeePMD-kit on CentOS 7 is available [here](https://github.com/frankhan91/deepmd-kit_docker). +After your easy installation, DeePMD-kit (`dp`) and LAMMPS (`lmp`) will be available to execute. You can try `dp -h` and `lmp -h` to see the help. `mpirun` is also available considering you may want to run LAMMPS in parallel. + +### Offline packages +Both CPU and GPU version offline packages are avaiable in [the Releases page](https://github.com/deepmodeling/deepmd-kit/releases). ### With conda DeePMD-kit is avaiable with [conda](https://github.com/conda/conda). Install [Anaconda](https://www.anaconda.com/distribution/#download-section) or [Miniconda](https://docs.conda.io/en/latest/miniconda.html) first. @@ -101,13 +103,23 @@ To install the CPU version: conda install deepmd-kit=*=*cpu lammps-dp=*=*cpu -c deepmodeling ``` -To install the GPU version containing [CUDA 10.0](https://docs.nvidia.com/deploy/cuda-compatibility/index.html#binary-compatibility__table-toolkit-driver): +To install the GPU version containing [CUDA 10.1](https://docs.nvidia.com/deploy/cuda-compatibility/index.html#binary-compatibility__table-toolkit-driver): ```bash conda install deepmd-kit=*=*gpu lammps-dp=*=*gpu -c deepmodeling ``` -### Offline packages -Both CPU and GPU version offline package are avaiable in [the Releases page](https://github.com/deepmodeling/deepmd-kit/releases). +### With Docker +A docker for installing the DeePMD-kit is available [here](https://hub.docker.com/r/deepmodeling/deepmd-kit). + +To pull the CPU version: +```bash +docker pull deepmodeling/deepmd-kit:1.2.0_cpu +``` + +To pull the GPU version: +```bash +docker pull deepmodeling/deepmd-kit:1.2.0_cuda10.1_gpu +``` ## Install the python interface ### Install the Tensorflow's python interface From 9dabb31bc4571b07095b9958c55555e4bd476ab7 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 5 Sep 2020 23:48:17 -0400 Subject: [PATCH 14/22] use GitHub docker container --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 37ab751c7f..5520daa57f 100644 --- a/README.md +++ b/README.md @@ -109,16 +109,16 @@ conda install deepmd-kit=*=*gpu lammps-dp=*=*gpu -c deepmodeling ``` ### With Docker -A docker for installing the DeePMD-kit is available [here](https://hub.docker.com/r/deepmodeling/deepmd-kit). +A docker for installing the DeePMD-kit is available [here](https://github.com/orgs/deepmodeling/packages/container/deepmd-kit). To pull the CPU version: ```bash -docker pull deepmodeling/deepmd-kit:1.2.0_cpu +docker pull ghcr.io/deepmodeling/deepmd-kit:1.2.0_cpu ``` To pull the GPU version: ```bash -docker pull deepmodeling/deepmd-kit:1.2.0_cuda10.1_gpu +docker pull ghcr.io/deepmodeling/deepmd-kit:1.2.0_cuda10.1_gpu ``` ## Install the python interface From 143afce2e05ba90382ed67ece785790938969d02 Mon Sep 17 00:00:00 2001 From: Lu Date: Mon, 14 Sep 2020 13:25:41 +0800 Subject: [PATCH 15/22] fix bug of error compilation when using float precision --- source/op/cuda/descrpt_se_a.cu | 41 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/source/op/cuda/descrpt_se_a.cu b/source/op/cuda/descrpt_se_a.cu index 1636df8ff5..1dd255fff2 100644 --- a/source/op/cuda/descrpt_se_a.cu +++ b/source/op/cuda/descrpt_se_a.cu @@ -24,8 +24,6 @@ limitations under the License. typedef float VALUETYPE; #endif -typedef double compute_t; - typedef unsigned long long int_64; #define cudaErrcheck(res) { cudaAssert((res), __FILE__, __LINE__); } @@ -77,19 +75,20 @@ __device__ inline T dev_dot(T * arr1, T * arr2) { return arr1[0] * arr2[0] + arr1[1] * arr2[1] + arr1[2] * arr2[2]; } -__device__ inline void spline5_switch(compute_t & vv, - compute_t & dd, - compute_t & xx, - const compute_t & rmin, - const compute_t & rmax) +template +__device__ inline void spline5_switch(FPTYPE & vv, + FPTYPE & dd, + FPTYPE & xx, + const float & rmin, + const float & rmax) { if (xx < rmin) { dd = 0; vv = 1; } else if (xx < rmax) { - compute_t uu = (xx - rmin) / (rmax - rmin) ; - compute_t du = 1. / (rmax - rmin) ; + FPTYPE uu = (xx - rmin) / (rmax - rmin) ; + FPTYPE du = 1. / (rmax - rmin) ; vv = uu*uu*uu * (-6 * uu*uu + 15 * uu - 10) + 1; dd = ( 3 * uu*uu * (-6 * uu*uu + 15 * uu - 10) + uu*uu*uu * (-12 * uu + 15) ) * du; } @@ -133,12 +132,12 @@ __global__ void format_nlist_fill_a_se_a(const VALUETYPE * coord, int_64 * key_in = key + idx * MAGIC_NUMBER; - compute_t diff[3]; + VALUETYPE diff[3]; const int & j_idx = nei_idx[idy]; for (int dd = 0; dd < 3; dd++) { diff[dd] = coord[j_idx * 3 + dd] - coord[idx * 3 + dd]; } - compute_t rr = sqrt(dev_dot(diff, diff)); + VALUETYPE rr = sqrt(dev_dot(diff, diff)); if (rr <= rcut) { key_in[idy] = type[j_idx] * 1E15+ (int_64)(rr * 1.0E13) / 100000 * 100000 + j_idx; } @@ -192,8 +191,8 @@ __global__ void compute_descriptor_se_a (VALUETYPE* descript, int* nlist, const int nlist_size, const VALUETYPE* coord, - const VALUETYPE rmin, - const VALUETYPE rmax, + const float rmin, + const float rmax, const int sec_a_size) { // <<>> @@ -214,14 +213,14 @@ __global__ void compute_descriptor_se_a (VALUETYPE* descript, for (int kk = 0; kk < 3; kk++) { row_rij[idy * 3 + kk] = coord[j_idx * 3 + kk] - coord[idx * 3 + kk]; } - const compute_t * rr = &row_rij[idy * 3 + 0]; - compute_t nr2 = dev_dot(rr, rr); - compute_t inr = 1./sqrt(nr2); - compute_t nr = nr2 * inr; - compute_t inr2 = inr * inr; - compute_t inr4 = inr2 * inr2; - compute_t inr3 = inr4 * nr; - compute_t sw, dsw; + const VALUETYPE * rr = &row_rij[idy * 3 + 0]; + VALUETYPE nr2 = dev_dot(rr, rr); + VALUETYPE inr = 1./sqrt(nr2); + VALUETYPE nr = nr2 * inr; + VALUETYPE inr2 = inr * inr; + VALUETYPE inr4 = inr2 * inr2; + VALUETYPE inr3 = inr4 * nr; + VALUETYPE sw, dsw; spline5_switch(sw, dsw, nr, rmin, rmax); row_descript[idx_value + 0] = (1./nr) ;//* sw; row_descript[idx_value + 1] = (rr[0] / nr2) ;//* sw; From 5061492268ef53fc5b9bbcf94dc13f0c0b0b0f4f Mon Sep 17 00:00:00 2001 From: Han Wang Date: Wed, 16 Sep 2020 13:32:53 +0800 Subject: [PATCH 16/22] global_polar: print loss not normalized by sqrt(natoms). add dp test for global_polar --- source/train/Loss.py | 7 ++++--- source/train/test.py | 27 +++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/source/train/Loss.py b/source/train/Loss.py index d939273f26..1f336325a3 100644 --- a/source/train/Loss.py +++ b/source/train/Loss.py @@ -301,11 +301,12 @@ def build (self, polar_hat = label_dict[self.label_name] polar = model_dict[self.tensor_name] l2_loss = tf.reduce_mean( tf.square(self.scale*(polar - polar_hat)), name='l2_'+suffix) + more_loss = {'nonorm': l2_loss} if not self.atomic : atom_norm = 1./ global_cvt_2_tf_float(natoms[0]) l2_loss = l2_loss * atom_norm self.l2_l = l2_loss - more_loss = {} + self.l2_more = more_loss['nonorm'] return l2_loss, more_loss @@ -321,10 +322,10 @@ def print_on_training(self, feed_dict_test, feed_dict_batch) : error_test\ - = sess.run([self.l2_l], \ + = sess.run([self.l2_more], \ feed_dict=feed_dict_test) error_train\ - = sess.run([self.l2_l], \ + = sess.run([self.l2_more], \ feed_dict=feed_dict_batch) print_str = "" prop_fmt = " %9.2e %9.2e" diff --git a/source/train/test.py b/source/train/test.py index d8639020ce..c01b81d4d0 100644 --- a/source/train/test.py +++ b/source/train/test.py @@ -12,6 +12,7 @@ from deepmd import DeepPot from deepmd import DeepDipole from deepmd import DeepPolar +from deepmd import DeepGlobalPolar from deepmd import DeepWFC from tensorflow.python.framework import ops @@ -28,6 +29,8 @@ def test (args): dp = DeepDipole(args.model) elif de.model_type == 'polar': dp = DeepPolar(args.model) + elif de.model_type == 'global_polar': + dp = DeepGlobalPolar(args.model) elif de.model_type == 'wfc': dp = DeepWFC(args.model) else : @@ -41,7 +44,9 @@ def test (args): elif de.model_type == 'dipole': err, siz = test_dipole(dp, args) elif de.model_type == 'polar': - err, siz = test_polar(dp, args) + err, siz = test_polar(dp, args, global_polar=False) + elif de.model_type == 'global_polar': + err, siz = test_polar(dp, args, global_polar=True) elif de.model_type == 'wfc': err, siz = test_wfc(dp, args) else : @@ -61,6 +66,8 @@ def test (args): print_dipole_sys_avg(avg_err) elif de.model_type == 'polar': print_polar_sys_avg(avg_err) + elif de.model_type == 'global_polar': + print_polar_sys_avg(avg_err) elif de.model_type == 'wfc': print_wfc_sys_avg(avg_err) else : @@ -223,12 +230,15 @@ def print_wfc_sys_avg(avg): print ("WFC L2err : %e eV/A" % avg[0]) -def test_polar (dp, args) : +def test_polar (dp, args, global_polar = False) : if args.rand_seed is not None : np.random.seed(args.rand_seed % (2**32)) data = DeepmdData(args.system, args.set_prefix, shuffle_test = args.shuffle_test) - data.add('polarizability', 9, atomic=True, must=True, high_prec=False, type_sel = dp.get_sel_type()) + if not global_polar: + data.add('polarizability', 9, atomic=True, must=True, high_prec=False, type_sel = dp.get_sel_type()) + else: + data.add('polarizability', 9, atomic=False, must=True, high_prec=False, type_sel = dp.get_sel_type()) test_data = data.get_test () numb_test = args.numb_test natoms = len(test_data["type"][0]) @@ -239,12 +249,21 @@ def test_polar (dp, args) : box = test_data["box"][:numb_test] atype = test_data["type"][0] polar = dp.eval(coord, box, atype) + sel_type = dp.get_sel_type() + sel_natoms = 0 + for ii in sel_type: + sel_natoms += sum(atype == ii) polar = polar.reshape([numb_test,-1]) l2f = (l2err (polar - test_data["polarizability"] [:numb_test])) + l2fs = l2f/np.sqrt(sel_natoms) + l2fa = l2f/sel_natoms print ("# number of test data : %d " % numb_test) - print ("Polarizability L2err : %e eV/A" % l2f) + print ("Polarizability L2err : %e eV/A" % l2f) + if global_polar: + print ("Polarizability L2err/sqrtN : %e eV/A" % l2fs) + print ("Polarizability L2err/N : %e eV/A" % l2fa) detail_file = args.detail_file if detail_file is not None : From e110ec462fb32ace1f0ca26493bd885a2c3c54b0 Mon Sep 17 00:00:00 2001 From: Han Wang Date: Thu, 17 Sep 2020 14:47:40 +0800 Subject: [PATCH 17/22] correct way of getting site package path --- setup.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 99a86d3da8..2e147160e1 100644 --- a/setup.py +++ b/setup.py @@ -4,17 +4,16 @@ from setuptools_scm import get_version from packaging.version import LegacyVersion from os import path, makedirs -import imp, sys, platform +import os, imp, sys, platform, sysconfig def get_dp_install_path() : - site_packages_path = path.join(path.dirname(path.__file__), 'site-packages') - dp_scm_version = get_version(root="./", relative_to=__file__) + site_packages_path = sysconfig.get_paths()['purelib'] + dp_scm_version = get_version(root=".", relative_to=__file__) python_version = 'py' + str(sys.version_info.major + sys.version_info.minor * 0.1) os_info = sys.platform machine_info = platform.machine() - dp_pip_install_path = site_packages_path + '/deepmd' - dp_setup_install_path = site_packages_path + '/deepmd_kit-' + dp_scm_version + '-' + python_version + '-' + os_info + '-' + machine_info + '.egg/deepmd' - + dp_pip_install_path = os.path.join(site_packages_path, 'deepmd') + dp_setup_install_path = os.path.join(site_packages_path, 'deepmd_kit-' + dp_scm_version + '-' + python_version + '-' + os_info + '-' + machine_info + '.egg', 'deepmd') return dp_pip_install_path, dp_setup_install_path readme_file = path.join(path.dirname(path.abspath(__file__)), 'README.md') From 45849aac04519dedafd18d60d43759b882625db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mari=C3=A1n=20Rynik?= Date: Wed, 7 Oct 2020 16:45:07 +0200 Subject: [PATCH 18/22] add ability to parse yaml config files --- setup.py | 2 +- source/tests/test_compat_input_v0_v1.py | 15 ++++++--------- source/tests/test_data_modifier.py | 7 +++---- source/tests/test_fitting_stat.py | 8 ++++---- source/tests/test_model_loc_frame.py | 7 +++---- source/tests/test_model_se_a.py | 8 ++++---- source/tests/test_model_se_a_aparam.py | 7 +++---- source/tests/test_model_se_a_fparam.py | 8 ++++---- source/tests/test_model_se_a_srtab.py | 8 ++++---- source/tests/test_model_se_r.py | 8 ++++---- source/tests/test_polar_se_a.py | 8 ++++---- source/tests/test_wfc.py | 8 ++++---- source/train/common.py | 14 +++++++++++++- source/train/print_old_model.py | 7 +++---- source/train/train.py | 7 +++---- 15 files changed, 63 insertions(+), 59 deletions(-) diff --git a/setup.py b/setup.py index 98bcfded26..172904a506 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ def get_dp_install_path() : site_packages_path = path.join(path.dirname(path.__file__), 'site-packages') tf_install_dir = imp.find_module('tensorflow', [site_packages_path])[1] -install_requires=['numpy', 'scipy'] +install_requires=['numpy', 'scipy', 'pyyaml'] setup_requires=['setuptools_scm', 'scikit-build'] # add cmake as a build requirement if cmake>3.0 is not installed diff --git a/source/tests/test_compat_input_v0_v1.py b/source/tests/test_compat_input_v0_v1.py index 6890a89301..7c4a46c361 100644 --- a/source/tests/test_compat_input_v0_v1.py +++ b/source/tests/test_compat_input_v0_v1.py @@ -1,23 +1,20 @@ -import os,sys,json +import os,sys import numpy as np import unittest from deepmd.compat import convert_input_v0_v1 +from deepmd.common import j_loader class TestConvertInputV0V1 (unittest.TestCase) : def test_convert_smth(self): - with open(os.path.join('compat_inputs', 'water_se_a_v0.json')) as fp: - jdata0 = json.load(fp) - with open(os.path.join('compat_inputs', 'water_se_a_v1.json')) as fp: - jdata1 = json.load(fp) + jdata0 = j_loader(os.path.join('compat_inputs', 'water_se_a_v0.json')) + jdata1 = j_loader(os.path.join('compat_inputs', 'water_se_a_v1.json')) jdata = convert_input_v0_v1(jdata0, warning = False, dump = None) self.assertEqual(jdata, jdata1) def test_convert_nonsmth(self): - with open(os.path.join('compat_inputs', 'water_v0.json')) as fp: - jdata0 = json.load(fp) - with open(os.path.join('compat_inputs', 'water_v1.json')) as fp: - jdata1 = json.load(fp) + jdata0 = j_loader(os.path.join('compat_inputs', 'water_v0.json')) + jdata1 = j_loader(os.path.join('compat_inputs', 'water_v1.json')) jdata = convert_input_v0_v1(jdata0, warning = False, dump = None) self.assertEqual(jdata, jdata1) diff --git a/source/tests/test_data_modifier.py b/source/tests/test_data_modifier.py index 4e7b43663e..f71e31377d 100644 --- a/source/tests/test_data_modifier.py +++ b/source/tests/test_data_modifier.py @@ -1,9 +1,9 @@ -import os,sys,platform,json +import os,sys,platform import numpy as np import unittest from deepmd.env import tf -from deepmd.common import j_must_have, data_requirement +from deepmd.common import j_must_have, data_requirement, j_loader from deepmd.RunOptions import RunOptions from deepmd.Trainer import NNPTrainer from deepmd.DataSystem import DeepmdDataSystem @@ -45,8 +45,7 @@ def tearDown(self): def _setUp(self): args = Args() run_opt = RunOptions(args, False) - with open (args.INPUT, 'r') as fp: - jdata = json.load (fp) + jdata = j_loader(args.INPUT) # init model model = NNPTrainer (jdata, run_opt = run_opt) diff --git a/source/tests/test_fitting_stat.py b/source/tests/test_fitting_stat.py index 0cbd693ae1..1e9c48b30b 100644 --- a/source/tests/test_fitting_stat.py +++ b/source/tests/test_fitting_stat.py @@ -1,10 +1,11 @@ -import os,sys,json +import os,sys import numpy as np import unittest from collections import defaultdict from deepmd.DescrptSeA import DescrptSeA from deepmd.Fitting import EnerFitting +from deepmd.common import j_loader input_json = 'water_se_a_afparam.json' @@ -57,9 +58,8 @@ def _brute_aparam(data, ndim): class TestEnerFittingStat (unittest.TestCase) : def test (self) : - with open(input_json) as fp: - jdata = json.load(fp) - jdata = jdata['model'] + jdata = j_loader(input_json) + jdata = jdata['model'] descrpt = DescrptSeA(jdata['descriptor']) fitting = EnerFitting(jdata['fitting_net'], descrpt) avgs = [0, 10] diff --git a/source/tests/test_model_loc_frame.py b/source/tests/test_model_loc_frame.py index b651862885..e79e59de1a 100644 --- a/source/tests/test_model_loc_frame.py +++ b/source/tests/test_model_loc_frame.py @@ -1,4 +1,4 @@ -import dpdata,os,sys,json,unittest +import dpdata,os,sys,unittest import numpy as np from deepmd.env import tf from common import Data,gen_data @@ -8,7 +8,7 @@ from deepmd.DescrptLocFrame import DescrptLocFrame from deepmd.Fitting import EnerFitting from deepmd.Model import Model -from deepmd.common import j_must_have, j_must_have_d, j_have +from deepmd.common import j_must_have, j_must_have_d, j_have, j_loader global_ener_float_precision = tf.float64 global_tf_float_precision = tf.float64 @@ -21,8 +21,7 @@ def setUp(self) : def test_model(self): jfile = 'water.json' - with open(jfile) as fp: - jdata = json.load (fp) + jdata = j_loader(jfile) run_opt = RunOptions(None) systems = j_must_have(jdata, 'systems') set_pfx = j_must_have(jdata, 'set_prefix') diff --git a/source/tests/test_model_se_a.py b/source/tests/test_model_se_a.py index 0d54f14c5f..2d32d89e45 100644 --- a/source/tests/test_model_se_a.py +++ b/source/tests/test_model_se_a.py @@ -1,5 +1,5 @@ -import dpdata,os,sys,json,unittest +import dpdata,os,sys,unittest import numpy as np from deepmd.env import tf from common import Data,gen_data @@ -9,7 +9,7 @@ from deepmd.DescrptSeA import DescrptSeA from deepmd.Fitting import EnerFitting from deepmd.Model import Model -from deepmd.common import j_must_have, j_must_have_d, j_have +from deepmd.common import j_must_have, j_must_have_d, j_have, j_loader global_ener_float_precision = tf.float64 global_tf_float_precision = tf.float64 @@ -21,8 +21,8 @@ def setUp(self) : def test_model(self): jfile = 'water_se_a.json' - with open(jfile) as fp: - jdata = json.load (fp) + jdata = j_loader(jfile) + run_opt = RunOptions(None) systems = j_must_have(jdata, 'systems') set_pfx = j_must_have(jdata, 'set_prefix') diff --git a/source/tests/test_model_se_a_aparam.py b/source/tests/test_model_se_a_aparam.py index 58b060225c..f22629ca19 100644 --- a/source/tests/test_model_se_a_aparam.py +++ b/source/tests/test_model_se_a_aparam.py @@ -1,4 +1,4 @@ -import dpdata,os,sys,json,unittest +import dpdata,os,sys,unittest import numpy as np from deepmd.env import tf from common import Data,gen_data @@ -8,7 +8,7 @@ from deepmd.DescrptSeA import DescrptSeA from deepmd.Fitting import EnerFitting from deepmd.Model import Model -from deepmd.common import j_must_have, j_must_have_d, j_have +from deepmd.common import j_must_have, j_must_have_d, j_have, j_loader global_ener_float_precision = tf.float64 global_tf_float_precision = tf.float64 @@ -20,8 +20,7 @@ def setUp(self) : def test_model(self): jfile = 'water_se_a_aparam.json' - with open(jfile) as fp: - jdata = json.load (fp) + jdata = j_loader(jfile) run_opt = RunOptions(None) systems = j_must_have(jdata, 'systems') set_pfx = j_must_have(jdata, 'set_prefix') diff --git a/source/tests/test_model_se_a_fparam.py b/source/tests/test_model_se_a_fparam.py index ec4a46c7d4..7c5ca2dfc6 100644 --- a/source/tests/test_model_se_a_fparam.py +++ b/source/tests/test_model_se_a_fparam.py @@ -1,4 +1,4 @@ -import dpdata,os,sys,json,unittest +import dpdata,os,sys,unittest import numpy as np from deepmd.env import tf from common import Data,gen_data @@ -8,7 +8,7 @@ from deepmd.DescrptSeA import DescrptSeA from deepmd.Fitting import EnerFitting from deepmd.Model import Model -from deepmd.common import j_must_have, j_must_have_d, j_have +from deepmd.common import j_must_have, j_must_have_d, j_have, j_loader global_ener_float_precision = tf.float64 global_tf_float_precision = tf.float64 @@ -20,8 +20,8 @@ def setUp(self) : def test_model(self): jfile = 'water_se_a_fparam.json' - with open(jfile) as fp: - jdata = json.load (fp) + jdata = j_loader(jfile) + run_opt = RunOptions(None) systems = j_must_have(jdata, 'systems') set_pfx = j_must_have(jdata, 'set_prefix') diff --git a/source/tests/test_model_se_a_srtab.py b/source/tests/test_model_se_a_srtab.py index c2950fe788..2eeda45b50 100644 --- a/source/tests/test_model_se_a_srtab.py +++ b/source/tests/test_model_se_a_srtab.py @@ -1,4 +1,4 @@ -import dpdata,os,sys,json,unittest +import dpdata,os,sys,unittest import numpy as np from deepmd.env import tf from common import Data,gen_data @@ -8,7 +8,7 @@ from deepmd.DescrptSeA import DescrptSeA from deepmd.Fitting import EnerFitting from deepmd.Model import Model -from deepmd.common import j_must_have, j_must_have_d, j_have +from deepmd.common import j_must_have, j_must_have_d, j_have, j_loader global_ener_float_precision = tf.float64 global_tf_float_precision = tf.float64 @@ -30,8 +30,8 @@ def setUp(self) : def test_model(self): jfile = 'water_se_a.json' - with open(jfile) as fp: - jdata = json.load (fp) + jdata = j_loader(jfile) + run_opt = RunOptions(None) systems = j_must_have(jdata, 'systems') set_pfx = j_must_have(jdata, 'set_prefix') diff --git a/source/tests/test_model_se_r.py b/source/tests/test_model_se_r.py index d3607a9164..32e3276760 100644 --- a/source/tests/test_model_se_r.py +++ b/source/tests/test_model_se_r.py @@ -1,4 +1,4 @@ -import dpdata,os,sys,json,unittest +import dpdata,os,sys,unittest import numpy as np from deepmd.env import tf from common import Data,gen_data @@ -8,7 +8,7 @@ from deepmd.DescrptSeR import DescrptSeR from deepmd.Fitting import EnerFitting from deepmd.Model import Model -from deepmd.common import j_must_have, j_must_have_d, j_have +from deepmd.common import j_must_have, j_must_have_d, j_have, j_loader global_ener_float_precision = tf.float64 global_tf_float_precision = tf.float64 @@ -20,8 +20,8 @@ def setUp(self) : def test_model(self): jfile = 'water_se_r.json' - with open(jfile) as fp: - jdata = json.load (fp) + jdata = j_loader(jfile) + run_opt = RunOptions(None) systems = j_must_have(jdata, 'systems') set_pfx = j_must_have(jdata, 'set_prefix') diff --git a/source/tests/test_polar_se_a.py b/source/tests/test_polar_se_a.py index 275b4fa707..ad2168dcb5 100644 --- a/source/tests/test_polar_se_a.py +++ b/source/tests/test_polar_se_a.py @@ -1,4 +1,4 @@ -import dpdata,os,sys,json,unittest +import dpdata,os,sys,unittest import numpy as np from deepmd.env import tf from common import Data,gen_data @@ -8,7 +8,7 @@ from deepmd.DescrptSeA import DescrptSeA from deepmd.Fitting import PolarFittingSeA from deepmd.Model import PolarModel -from deepmd.common import j_must_have, j_must_have_d, j_have +from deepmd.common import j_must_have, j_must_have_d, j_have, j_loader global_ener_float_precision = tf.float64 global_tf_float_precision = tf.float64 @@ -20,8 +20,8 @@ def setUp(self) : def test_model(self): jfile = 'polar_se_a.json' - with open(jfile) as fp: - jdata = json.load (fp) + jdata = j_loader(jfile) + run_opt = RunOptions(None) systems = j_must_have(jdata, 'systems') set_pfx = j_must_have(jdata, 'set_prefix') diff --git a/source/tests/test_wfc.py b/source/tests/test_wfc.py index d4b408cd60..876f4dba0a 100644 --- a/source/tests/test_wfc.py +++ b/source/tests/test_wfc.py @@ -1,4 +1,4 @@ -import dpdata,os,sys,json,unittest +import dpdata,os,sys,unittest import numpy as np from deepmd.env import tf from common import Data,gen_data @@ -8,7 +8,7 @@ from deepmd.DescrptLocFrame import DescrptLocFrame from deepmd.Fitting import WFCFitting from deepmd.Model import WFCModel -from deepmd.common import j_must_have, j_must_have_d, j_have +from deepmd.common import j_must_have, j_must_have_d, j_have, j_loader global_ener_float_precision = tf.float64 global_tf_float_precision = tf.float64 @@ -20,8 +20,8 @@ def setUp(self) : def test_model(self): jfile = 'wfc.json' - with open(jfile) as fp: - jdata = json.load (fp) + jdata = j_loader(jfile) + run_opt = RunOptions(None) systems = j_must_have(jdata, 'systems') set_pfx = j_must_have(jdata, 'set_prefix') diff --git a/source/train/common.py b/source/train/common.py index 887669a278..a68f8d27b8 100644 --- a/source/train/common.py +++ b/source/train/common.py @@ -4,6 +4,8 @@ from deepmd.env import tf from deepmd.env import op_module from deepmd.RunOptions import global_tf_float_precision +import json +import yaml # def gelu(x): # """Gaussian Error Linear Unit. @@ -163,7 +165,17 @@ def j_must_have_d (jdata, key, deprecated_key) : def j_have (jdata, key) : return key in jdata.keys() - + +def j_loader(filename): + + with open(filename, 'r') as fp: + if filename.endswith("json"): + return json.load(fp) + elif filename.endswith(("yml", "yaml")): + return yaml.safe_load(fp) + else: + raise TypeError("config file must be json, or yaml/yml") + def get_activation_func(activation_fn): if activation_fn not in activation_fn_dict: raise RuntimeError(activation_fn+" is not a valid activation function") diff --git a/source/train/print_old_model.py b/source/train/print_old_model.py index 14719723f9..d125e7f8b6 100644 --- a/source/train/print_old_model.py +++ b/source/train/print_old_model.py @@ -1,4 +1,4 @@ -import dpdata,os,sys,json +import dpdata,os,sys import numpy as np import tensorflow as tf from common import Data @@ -12,7 +12,7 @@ from deepmd.DataSystem import DataSystem from deepmd.Model import NNPModel from deepmd.Model import LearingRate -from deepmd.common import j_must_have +from deepmd.common import j_must_have, j_loader def gen_data() : tmpdata = Data(rand_pert = 0.1, seed = 1) @@ -32,8 +32,7 @@ def gen_data() : np.save('system/set.000/fparam.npy', tmpdata.fparam) def compute_efv(jfile): - fp = open (jfile, 'r') - jdata = json.load (fp) + jdata = j_loader(jfile) run_opt = RunOptions(None) systems = j_must_have(jdata, 'systems') set_pfx = j_must_have(jdata, 'set_prefix') diff --git a/source/train/train.py b/source/train/train.py index c89760fa4d..3e7ba2955b 100755 --- a/source/train/train.py +++ b/source/train/train.py @@ -4,13 +4,12 @@ import sys import time import numpy as np -import json from deepmd.env import tf from deepmd.compat import convert_input_v0_v1 from deepmd.RunOptions import RunOptions from deepmd.DataSystem import DeepmdDataSystem from deepmd.Trainer import NNPTrainer -from deepmd.common import data_requirement, expand_sys_str +from deepmd.common import data_requirement, expand_sys_str, j_loader from deepmd.DataModifier import DipoleChargeModifier def create_done_queue(cluster_spec, task_index): @@ -49,8 +48,8 @@ def j_must_have (jdata, key) : def train (args) : # load json database - with open (args.INPUT, 'r') as fp: - jdata = json.load (fp) + jdata = j_loader(args.INPUT) + if not 'model' in jdata.keys(): jdata = convert_input_v0_v1(jdata, warning = True, From faae175e52da30ed93cbf6fe969200beae4ed7b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mari=C3=A1n=20Rynik?= Date: Wed, 7 Oct 2020 21:36:02 +0200 Subject: [PATCH 19/22] added script to convert json to yaml --- data/raw/json2yaml.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 data/raw/json2yaml.py diff --git a/data/raw/json2yaml.py b/data/raw/json2yaml.py new file mode 100644 index 0000000000..8f47da40dc --- /dev/null +++ b/data/raw/json2yaml.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +import argparse +from pathlib import Path +import json +import yaml + + +def _main(): + parser = argparse.ArgumentParser( + description="convert json config file to yaml", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + # get all json files in dir + jsons = [p for p in Path.cwd().glob("*.json")] + # use the newest as autosuggestion + jsons.sort(key=lambda x: x.stat().st_mtime, reverse=True) + jfile = jsons[0] + yfile = jfile.with_suffix(".yaml") + + parser.add_argument("INPUT", default=jfile, type=Path, nargs="?", + help="input json file") + parser.add_argument("OUTPUT", default=yfile, type=Path, nargs="?", + help="output yaml file") + args = parser.parse_args() + + with args.INPUT.open("r") as infile, args.OUTPUT.open("w") as outfile: + yaml.dump(json.load(infile), outfile, default_flow_style=False, + sort_keys=False) + +if __name__ == "__main__": + _main() From 9806168fe1cb2a7a7d1be0202f5e987ff42b1b1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mari=C3=A1n=20Rynik?= Date: Thu, 8 Oct 2020 13:01:56 +0200 Subject: [PATCH 20/22] added test for json yaml equality --- data/{raw => json}/json2yaml.py | 7 ++- ...at_input_v0_v1.py => test_compat_input.py} | 18 +++++- source/tests/yaml_inputs/water_se_a_v1.json | 55 +++++++++++++++++++ source/tests/yaml_inputs/water_se_a_v1.yaml | 50 +++++++++++++++++ source/tests/yaml_inputs/water_v1.json | 51 +++++++++++++++++ source/tests/yaml_inputs/water_v1.yaml | 48 ++++++++++++++++ source/train/common.py | 11 ++-- 7 files changed, 232 insertions(+), 8 deletions(-) rename data/{raw => json}/json2yaml.py (86%) rename source/tests/{test_compat_input_v0_v1.py => test_compat_input.py} (54%) create mode 100644 source/tests/yaml_inputs/water_se_a_v1.json create mode 100644 source/tests/yaml_inputs/water_se_a_v1.yaml create mode 100644 source/tests/yaml_inputs/water_v1.json create mode 100644 source/tests/yaml_inputs/water_v1.yaml diff --git a/data/raw/json2yaml.py b/data/json/json2yaml.py similarity index 86% rename from data/raw/json2yaml.py rename to data/json/json2yaml.py index 8f47da40dc..f601928427 100644 --- a/data/raw/json2yaml.py +++ b/data/json/json2yaml.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 import argparse -from pathlib import Path import json +from pathlib import Path +from warnings import warn + import yaml @@ -28,5 +30,8 @@ def _main(): yaml.dump(json.load(infile), outfile, default_flow_style=False, sort_keys=False) + warn("The order of the keys won't be preserved!", SyntaxWarning) + warn("_comment keys will also be lostt in the conversion") + if __name__ == "__main__": _main() diff --git a/source/tests/test_compat_input_v0_v1.py b/source/tests/test_compat_input.py similarity index 54% rename from source/tests/test_compat_input_v0_v1.py rename to source/tests/test_compat_input.py index 7c4a46c361..61eb16faf3 100644 --- a/source/tests/test_compat_input_v0_v1.py +++ b/source/tests/test_compat_input.py @@ -3,9 +3,11 @@ import unittest from deepmd.compat import convert_input_v0_v1 -from deepmd.common import j_loader +sys.path.append("/mnt/md0/OneDrive/dizertacka/code/deepmd-kit/source") +from train.common import j_loader +#from deepmd.common import j_loader -class TestConvertInputV0V1 (unittest.TestCase) : +class TestConvertInput (unittest.TestCase) : def test_convert_smth(self): jdata0 = j_loader(os.path.join('compat_inputs', 'water_se_a_v0.json')) jdata1 = j_loader(os.path.join('compat_inputs', 'water_se_a_v1.json')) @@ -18,3 +20,15 @@ def test_convert_nonsmth(self): jdata = convert_input_v0_v1(jdata0, warning = False, dump = None) self.assertEqual(jdata, jdata1) + def test_json_yaml_equal(self): + + inputs = ("water_v1", "water_se_a_v1") + + for i in inputs: + jdata = j_loader(os.path.join('yaml_inputs', f'{i}.json')) + ydata = j_loader(os.path.join('yaml_inputs', f'{i}.yaml')) + self.assertEqual(jdata, ydata) + + with self.assertRaises(TypeError): + j_loader("path_with_wrong.extension") + diff --git a/source/tests/yaml_inputs/water_se_a_v1.json b/source/tests/yaml_inputs/water_se_a_v1.json new file mode 100644 index 0000000000..402da962ca --- /dev/null +++ b/source/tests/yaml_inputs/water_se_a_v1.json @@ -0,0 +1,55 @@ +{ + "model": { + "descriptor" :{ + "type": "se_a", + "sel": [46, 92], + "rcut_smth": 5.80, + "rcut": 6.00, + "neuron": [25, 50, 100], + "axis_neuron": 16, + "resnet_dt": false, + "seed": 1 + }, + "fitting_net" : { + "neuron": [240, 240, 240], + "resnet_dt": true, + "seed": 1 + } + }, + + "learning_rate" :{ + "type": "exp", + "decay_steps": 5000, + "decay_rate": 0.95, + "start_lr": 0.001 + }, + + "loss" :{ + "start_pref_e": 0.02, + "limit_pref_e": 1, + "start_pref_f": 1000, + "limit_pref_f": 1, + "start_pref_v": 0, + "limit_pref_v": 0 + }, + + "training" : { + "systems": ["../data/"], + "set_prefix": "set", + "stop_batch": 1000000, + "batch_size": [1], + + "seed": 1, + + "disp_file": "lcurve.out", + "disp_freq": 100, + "numb_test": 10, + "save_freq": 1000, + "save_ckpt": "model.ckpt", + "disp_training":true, + "time_training":true, + "profiling": true, + "profiling_file":"timeline.json" + } +} + diff --git a/source/tests/yaml_inputs/water_se_a_v1.yaml b/source/tests/yaml_inputs/water_se_a_v1.yaml new file mode 100644 index 0000000000..55580daf1e --- /dev/null +++ b/source/tests/yaml_inputs/water_se_a_v1.yaml @@ -0,0 +1,50 @@ +model: + descriptor: + type: se_a + sel: + - 46 + - 92 + rcut_smth: 5.8 + rcut: 6.0 + neuron: + - 25 + - 50 + - 100 + axis_neuron: 16 + resnet_dt: false + seed: 1 + fitting_net: + neuron: + - 240 + - 240 + - 240 + resnet_dt: true + seed: 1 +learning_rate: + type: exp + decay_steps: 5000 + decay_rate: 0.95 + start_lr: 0.001 +loss: + start_pref_e: 0.02 + limit_pref_e: 1 + start_pref_f: 1000 + limit_pref_f: 1 + start_pref_v: 0 + limit_pref_v: 0 +training: + systems: ['../data/'] + set_prefix: set + stop_batch: 1000000 + batch_size: + - 1 + seed: 1 + disp_file: lcurve.out + disp_freq: 100 + numb_test: 10 + save_freq: 1000 + save_ckpt: model.ckpt + disp_training: true + time_training: true + profiling: true + profiling_file: timeline.json diff --git a/source/tests/yaml_inputs/water_v1.json b/source/tests/yaml_inputs/water_v1.json new file mode 100644 index 0000000000..e5f2032ea2 --- /dev/null +++ b/source/tests/yaml_inputs/water_v1.json @@ -0,0 +1,51 @@ +{ + "with_distrib": false, + "model":{ + "descriptor": { + "type": "loc_frame", + "sel_a": [16, 32], + "sel_r": [30, 60], + "rcut": 6.00, + "axis_rule": [0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0] + }, + "fitting_net": { + "neuron": [240, 120, 60, 30, 10], + "resnet_dt": true, + "seed": 1 + } + }, + + "learning_rate" :{ + "type": "exp", + "decay_steps": 5000, + "decay_rate": 0.95, + "start_lr": 0.001 + }, + + "loss" : { + "start_pref_e": 0.02, + "limit_pref_e": 8, + "start_pref_f": 1000, + "limit_pref_f": 1, + "start_pref_v": 0, + "limit_pref_v": 0 + }, + + "training": { + "systems": ["../data/"], + "set_prefix": "set", + "stop_batch": 1000000, + "batch_size": [4], + + "seed": 1, + + "disp_file": "lcurve.out", + "disp_freq": 100, + "numb_test": 10, + "save_freq": 1000, + "save_ckpt": "model.ckpt", + "disp_training":true, + "time_training":true + } +} + diff --git a/source/tests/yaml_inputs/water_v1.yaml b/source/tests/yaml_inputs/water_v1.yaml new file mode 100644 index 0000000000..5121a961b0 --- /dev/null +++ b/source/tests/yaml_inputs/water_v1.yaml @@ -0,0 +1,48 @@ +with_distrib: false +model: + descriptor: + type: loc_frame + sel_a: + - 16 + - 32 + sel_r: + - 30 + - 60 + rcut: 6.0 + axis_rule: [0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0] + fitting_net: + neuron: + - 240 + - 120 + - 60 + - 30 + - 10 + resnet_dt: true + seed: 1 +learning_rate: + type: exp + decay_steps: 5000 + decay_rate: 0.95 + start_lr: 0.001 +loss: + start_pref_e: 0.02 + limit_pref_e: 8 + start_pref_f: 1000 + limit_pref_f: 1 + start_pref_v: 0 + limit_pref_v: 0 +training: + systems: + - ../data/ + set_prefix: set + stop_batch: 1000000 + batch_size: + - 4 + seed: 1 + disp_file: lcurve.out + disp_freq: 100 + numb_test: 10 + save_freq: 1000 + save_ckpt: model.ckpt + disp_training: true + time_training: true diff --git a/source/train/common.py b/source/train/common.py index a68f8d27b8..83f5e6ecf3 100644 --- a/source/train/common.py +++ b/source/train/common.py @@ -168,13 +168,14 @@ def j_have (jdata, key) : def j_loader(filename): - with open(filename, 'r') as fp: - if filename.endswith("json"): + if filename.endswith("json"): + with open(filename, 'r') as fp: return json.load(fp) - elif filename.endswith(("yml", "yaml")): + elif filename.endswith(("yml", "yaml")): + with open(filename, 'r') as fp: return yaml.safe_load(fp) - else: - raise TypeError("config file must be json, or yaml/yml") + else: + raise TypeError("config file must be json, or yaml/yml") def get_activation_func(activation_fn): if activation_fn not in activation_fn_dict: From 865a1f46c198e4061744aedb63fd192bd2d00c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mari=C3=A1n=20Rynik?= Date: Thu, 8 Oct 2020 13:30:40 +0200 Subject: [PATCH 21/22] minor bugfix --- source/tests/test_compat_input.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/tests/test_compat_input.py b/source/tests/test_compat_input.py index 61eb16faf3..c0a29283dd 100644 --- a/source/tests/test_compat_input.py +++ b/source/tests/test_compat_input.py @@ -3,9 +3,7 @@ import unittest from deepmd.compat import convert_input_v0_v1 -sys.path.append("/mnt/md0/OneDrive/dizertacka/code/deepmd-kit/source") -from train.common import j_loader -#from deepmd.common import j_loader +from deepmd.common import j_loader class TestConvertInput (unittest.TestCase) : def test_convert_smth(self): From d248f8a29a5c31d6ce10c3cde558b43180f00352 Mon Sep 17 00:00:00 2001 From: Lu Date: Fri, 9 Oct 2020 18:32:58 +0800 Subject: [PATCH 22/22] fix conflict between master and devel branch --- setup.py | 14 +----------- source/op/CMakeLists.txt | 2 +- source/op/cuda/descrpt_se_a.cu | 41 ++++++++++++++++++---------------- 3 files changed, 24 insertions(+), 33 deletions(-) diff --git a/setup.py b/setup.py index 98bcfded26..858f84aa5a 100644 --- a/setup.py +++ b/setup.py @@ -6,15 +6,6 @@ from os import path, makedirs import os, imp, sys, platform, sysconfig -def get_dp_install_path() : - site_packages_path = sysconfig.get_paths()['purelib'] - dp_scm_version = get_version(root=".", relative_to=__file__) - python_version = 'py' + str(sys.version_info.major + sys.version_info.minor * 0.1) - os_info = sys.platform - machine_info = platform.machine() - dp_pip_install_path = os.path.join(site_packages_path, 'deepmd') - dp_setup_install_path = os.path.join(site_packages_path, 'deepmd_kit-' + dp_scm_version + '-' + python_version + '-' + os_info + '-' + machine_info + '.egg', 'deepmd') - return dp_pip_install_path, dp_setup_install_path readme_file = path.join(path.dirname(path.abspath(__file__)), 'README.md') try: @@ -31,7 +22,7 @@ def get_dp_install_path() : tf_install_dir = imp.find_module('tensorflow', [site_packages_path])[1] install_requires=['numpy', 'scipy'] -setup_requires=['setuptools_scm', 'scikit-build'] +setup_requires=['setuptools_scm', 'scikit-build', 'cmake'] # add cmake as a build requirement if cmake>3.0 is not installed try: @@ -45,7 +36,6 @@ def get_dp_install_path() : except OSError: pass -dp_pip_install_path, dp_setup_install_path = get_dp_install_path() setup( name="deepmd-kit", @@ -69,8 +59,6 @@ def get_dp_install_path() : '-DBUILD_PY_IF:BOOL=TRUE', '-DBUILD_CPP_IF:BOOL=FALSE', '-DFLOAT_PREC:STRING=high', - '-DDP_PIP_INSTALL_PATH=%s' % dp_pip_install_path, - '-DDP_SETUP_INSTALL_PATH=%s' % dp_setup_install_path, ], cmake_source_dir='source', cmake_minimum_required_version='3.0', diff --git a/source/op/CMakeLists.txt b/source/op/CMakeLists.txt index 993a1b6fd4..e73e5dcb49 100644 --- a/source/op/CMakeLists.txt +++ b/source/op/CMakeLists.txt @@ -25,7 +25,7 @@ endif (BUILD_CPP_IF) if (BUILD_PY_IF) set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) - set(CMAKE_INSTALL_RPATH DESTINATION ${DP_PIP_INSTALL_PATH} ${DP_SETUP_INSTALL_PATH} ${CMAKE_BINARY_DIR}/op/cuda) + set(CMAKE_INSTALL_RPATH $ORIGIN) if (USE_CUDA_TOOLKIT) add_library(op_abi SHARED ${OP_PY_CUDA_SRC} ${OP_LIB}) add_library(op_grads SHARED ${OP_GRADS_SRC}) diff --git a/source/op/cuda/descrpt_se_a.cu b/source/op/cuda/descrpt_se_a.cu index 1dd255fff2..09b2ed2638 100644 --- a/source/op/cuda/descrpt_se_a.cu +++ b/source/op/cuda/descrpt_se_a.cu @@ -24,6 +24,8 @@ limitations under the License. typedef float VALUETYPE; #endif +typedef double compute_t; + typedef unsigned long long int_64; #define cudaErrcheck(res) { cudaAssert((res), __FILE__, __LINE__); } @@ -132,12 +134,12 @@ __global__ void format_nlist_fill_a_se_a(const VALUETYPE * coord, int_64 * key_in = key + idx * MAGIC_NUMBER; - VALUETYPE diff[3]; + compute_t diff[3]; const int & j_idx = nei_idx[idy]; for (int dd = 0; dd < 3; dd++) { diff[dd] = coord[j_idx * 3 + dd] - coord[idx * 3 + dd]; } - VALUETYPE rr = sqrt(dev_dot(diff, diff)); + compute_t rr = sqrt(dev_dot(diff, diff)); if (rr <= rcut) { key_in[idy] = type[j_idx] * 1E15+ (int_64)(rr * 1.0E13) / 100000 * 100000 + j_idx; } @@ -179,18 +181,19 @@ __global__ void format_nlist_fill_b_se_a(int * nlist, } //it's ok! -__global__ void compute_descriptor_se_a (VALUETYPE* descript, +template +__global__ void compute_descriptor_se_a (FPTYPE* descript, const int ndescrpt, - VALUETYPE* descript_deriv, + FPTYPE* descript_deriv, const int descript_deriv_size, - VALUETYPE* rij, + FPTYPE* rij, const int rij_size, const int* type, - const VALUETYPE* avg, - const VALUETYPE* std, + const FPTYPE* avg, + const FPTYPE* std, int* nlist, const int nlist_size, - const VALUETYPE* coord, + const FPTYPE* coord, const float rmin, const float rmax, const int sec_a_size) @@ -203,9 +206,9 @@ __global__ void compute_descriptor_se_a (VALUETYPE* descript, if (idy >= sec_a_size) {return;} // else {return;} - VALUETYPE * row_descript = descript + idx * ndescrpt; - VALUETYPE * row_descript_deriv = descript_deriv + idx * descript_deriv_size; - VALUETYPE * row_rij = rij + idx * rij_size; + FPTYPE * row_descript = descript + idx * ndescrpt; + FPTYPE * row_descript_deriv = descript_deriv + idx * descript_deriv_size; + FPTYPE * row_rij = rij + idx * rij_size; int * row_nlist = nlist + idx * nlist_size; if (row_nlist[idy] >= 0) { @@ -213,14 +216,14 @@ __global__ void compute_descriptor_se_a (VALUETYPE* descript, for (int kk = 0; kk < 3; kk++) { row_rij[idy * 3 + kk] = coord[j_idx * 3 + kk] - coord[idx * 3 + kk]; } - const VALUETYPE * rr = &row_rij[idy * 3 + 0]; - VALUETYPE nr2 = dev_dot(rr, rr); - VALUETYPE inr = 1./sqrt(nr2); - VALUETYPE nr = nr2 * inr; - VALUETYPE inr2 = inr * inr; - VALUETYPE inr4 = inr2 * inr2; - VALUETYPE inr3 = inr4 * nr; - VALUETYPE sw, dsw; + const FPTYPE * rr = &row_rij[idy * 3 + 0]; + FPTYPE nr2 = dev_dot(rr, rr); + FPTYPE inr = 1./sqrt(nr2); + FPTYPE nr = nr2 * inr; + FPTYPE inr2 = inr * inr; + FPTYPE inr4 = inr2 * inr2; + FPTYPE inr3 = inr4 * nr; + FPTYPE sw, dsw; spline5_switch(sw, dsw, nr, rmin, rmax); row_descript[idx_value + 0] = (1./nr) ;//* sw; row_descript[idx_value + 1] = (rr[0] / nr2) ;//* sw;