From 6d86c7bb328b4ff3c2ce10c5be4c7432eb9924b2 Mon Sep 17 00:00:00 2001 From: akashdhruv Date: Wed, 14 Dec 2022 01:19:34 -0600 Subject: [PATCH 01/26] error handling --- boxkit/library/_action.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/boxkit/library/_action.py b/boxkit/library/_action.py index e6607a84..f46d5919 100644 --- a/boxkit/library/_action.py +++ b/boxkit/library/_action.py @@ -1,5 +1,5 @@ """Module with implementation of Action utility""" - +from types import GeneratorType import copy from .. import library @@ -68,9 +68,15 @@ def execute(self, *args, **kwargs): """Custom call signature""" toparg, args = Action.toparg(*args) + + if not isinstance(toparg, GeneratorType) and not isinstance(toparg, list): + raise ValueError( + "[boxkit.library.Action] First argument " + + f"must be {GeneratorType!r} or {list!r} not {type(toparg)!r}" + ) + obj_list = list(toparg) del toparg - self.__class__.chk_obj_list(obj_list) return library.exectask(self, obj_list, *args, **kwargs) @@ -79,11 +85,6 @@ def execute(self, *args, **kwargs): def chk_obj_list(obj_list): """Check if obj_list matches the parallel_obj type""" - if not isinstance(obj_list, list): - raise ValueError( - "[boxkit.library.Action] Top argument must be a list of parallel_objs" - ) - first_obj = obj_list[0] for index, parallel_obj in enumerate(obj_list): if not isinstance(parallel_obj, type(first_obj)): From 07834fb435f02eb1ade6433b73b0578e57909712 Mon Sep 17 00:00:00 2001 From: akashdhruv Date: Wed, 14 Dec 2022 01:27:38 -0600 Subject: [PATCH 02/26] fix trivial.py --- tests/cache/trivial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cache/trivial.py b/tests/cache/trivial.py index a1cd7567..7c6a39fc 100644 --- a/tests/cache/trivial.py +++ b/tests/cache/trivial.py @@ -2,7 +2,7 @@ from boxkit.library import Action, Timer -@Action(parallel_obj=float, backend="serial", nthreads=1, monitor=True) +@Action(backend="serial", nthreads=1, monitor=True) def multiply(parallel_obj): return parallel_obj * 10 From 98ddcdbf8c5653625c9d3e651687cad564a8ae75 Mon Sep 17 00:00:00 2001 From: akashdhruv Date: Wed, 14 Dec 2022 14:16:56 -0600 Subject: [PATCH 03/26] add design document --- DESIGN.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 DESIGN.md diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 00000000..cf9ab9c0 --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,13 @@ +## Software Design and Developer's Guide for BoxKit + +### API +BoxKit API details here + +### Library +BoxKit library details here + +### C-Box +C++ backend detailes here + +### Miscellaneous +Miscellaneous details about how to develop frontend methods, etc. From 695a0c98e7d90fff8a1c0419ad84bbcaceb39b27 Mon Sep 17 00:00:00 2001 From: akashdhruv Date: Wed, 14 Dec 2022 15:22:33 -0600 Subject: [PATCH 04/26] update design doc --- DESIGN.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index cf9ab9c0..54d6bbd5 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,13 +1,13 @@ -## Software Design and Developer's Guide for BoxKit +# Software Design and Developer's Guide for BoxKit -### API +## API BoxKit API details here -### Library +## Library BoxKit library details here -### C-Box +## C-Box C++ backend detailes here -### Miscellaneous +## Miscellaneous Miscellaneous details about how to develop frontend methods, etc. From ee42d8ba00b0673493d18e7614cd30a45535fc0c Mon Sep 17 00:00:00 2001 From: akashdhruv Date: Thu, 15 Dec 2022 22:18:23 -0600 Subject: [PATCH 05/26] update design.md --- DESIGN.md | 12 +++++++----- media/book.svg | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 media/book.svg diff --git a/DESIGN.md b/DESIGN.md index 54d6bbd5..dd2649bb 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,13 +1,15 @@ -# Software Design and Developer's Guide for BoxKit +

-## API +##

Software Design and Developer's Guide for BoxKit

+ +### 1. API BoxKit API details here -## Library +### 2. Library BoxKit library details here -## C-Box +### 3. C-Box C++ backend detailes here -## Miscellaneous +### 4. Miscellaneous Miscellaneous details about how to develop frontend methods, etc. diff --git a/media/book.svg b/media/book.svg new file mode 100644 index 00000000..a440aa83 --- /dev/null +++ b/media/book.svg @@ -0,0 +1 @@ + \ No newline at end of file From d5d1e249f0584575f94c8807a114a3b5db134370 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Thu, 15 Dec 2022 22:30:54 -0600 Subject: [PATCH 06/26] Update DESIGN.md --- DESIGN.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index dd2649bb..2111659f 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,5 +1,3 @@ -

- ##

Software Design and Developer's Guide for BoxKit

### 1. API From 71bf1dffa25d2fc69a87d8f92333549ea0275734 Mon Sep 17 00:00:00 2001 From: akashdhruv Date: Thu, 15 Dec 2022 22:44:21 -0600 Subject: [PATCH 07/26] format rst files --- .gitignore | 1 + DESIGN.rst | 30 ++++++++++++++ README.rst | 114 ++++++++++++++++++++++++++++++++--------------------- 3 files changed, 101 insertions(+), 44 deletions(-) create mode 100644 DESIGN.rst diff --git a/.gitignore b/.gitignore index d10fc33a..a569f3e8 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ boxkit/depends/boost/* .sphinx/build/* !.sphinx/source/index.rst !README.rst +!DESIGN.rst diff --git a/DESIGN.rst b/DESIGN.rst new file mode 100644 index 00000000..d27b758e --- /dev/null +++ b/DESIGN.rst @@ -0,0 +1,30 @@ +.. |icon| image:: ./media/book.svg + :width: 30 + +######################################################### + |icon| Software Design and Developer's Guide for BoxKit +######################################################### + +***** + API +***** + +BoxKit API details here + +********* + Library +********* + +BoxKit library details here + +******* + C-Box +******* + +C++ backend detailes here + +*************** + Miscellaneous +*************** + +Miscellaneous details about how to develop frontend methods, etc. diff --git a/README.rst b/README.rst index 354b01af..10040e51 100644 --- a/README.rst +++ b/README.rst @@ -1,87 +1,113 @@ .. |icon| image:: ./media/icon.svg - :width: 30 - -============= -|icon| BoxKit -============= + :width: 30 + +############### + |icon| BoxKit +############### |Code style: black| |FlashX| |FlowX| |Minimal| |Publish| -BoxKit is a library that provides building blocks to parallelize and scale data science, high performance computing, and machine learning applications for block-structured datasets. Spatial data from simulations and experiments can be accessed and managed using tools available in this library when working with more data analysis oriented packages like SciKit (https://github.com/scikit-learn/scikit-learn) and FlowNet (https://github.com/NVIDIA/flownet2-pytorch) +BoxKit is a library that provides building blocks to parallelize and +scale data science, high performance computing, and machine learning +applications for block-structured datasets. Spatial data from +simulations and experiments can be accessed and managed using tools +available in this library when working with more data analysis oriented +packages like SciKit (https://github.com/scikit-learn/scikit-learn) and +FlowNet (https://github.com/NVIDIA/flownet2-pytorch) -Installation -============ +************** + Installation +************** -Stable releases of BoxKit are hosted on Python Package Index website (``_) and can be installed by executing, +Stable releases of BoxKit are hosted on Python Package Index website +(https://pypi.org/project/BoxKit/) and can be installed by executing, -:: +.. code:: pip install BoxKit --user - + export CXX=$(CPP_COMPILER) pip install BoxKit --user --install-option="--enable-testing" --install-option="--with-cbox" - -Note that ``pip`` should point to ``python3+`` installation package ``pip3``. -Upgrading and uninstallation is easily managed through this interface using, +Note that ``pip`` should point to ``python3+`` installation package +``pip3``. + +Upgrading and uninstallation is easily managed through this interface +using, -:: +.. code:: pip install --upgrade BoxKit --user pip install --upgrade BoxKit --user --install-option="--enable-testing" --install-option="--with-cbox" pip uninstall BoxKit -There maybe situations where users may want to install BoxKit in development mode $\\textemdash$ to design new features, debug, or customize classes/methods to their needs. This can be easily accomplished using the ``setup`` script located in the project root directory and executing, +There maybe situations where users may want to install BoxKit in +development mode $\\textemdash$ to design new features, debug, or +customize classes/methods to their needs. This can be easily +accomplished using the ``setup`` script located in the project root +directory and executing, -:: +.. code:: ./setup develop -Development mode enables testing of features/updates directly from the source code and is an effective method for debugging. Note that the ``setup`` script relies on ``click``, which can be installed using, +Development mode enables testing of features/updates directly from the +source code and is an effective method for debugging. Note that the +``setup`` script relies on ``click``, which can be installed using, -:: +.. code:: - pip install click - -Usage -===== + pip install click -BoxKit is undergoing active development and therefore design changes are frequent, however, the library is divided into two broad categories: +******* + Usage +******* -- **Create**: Containing interface for classes/methods to store spatial data in a rectangular/cubic frame, along with auxillary tools to manage irregular geometries composed of unstructured triangular mesh. +BoxKit is undergoing active development and therefore design changes are +frequent, however, the library is divided into two broad categories: -- **Utilities**: Containing interface for classes/method to improve memory managment of data on NUMA nodes. +- **Create**: Containing interface for classes/methods to store spatial + data in a rectangular/cubic frame, along with auxillary tools to + manage irregular geometries composed of unstructured triangular mesh. -We are currently setting up use cases for BoxKit, and will update this section when we are able to demonstrate proof-of-concept. +- **Utilities**: Containing interface for classes/method to improve + memory managment of data on NUMA nodes. -Citation -======== +We are currently setting up use cases for BoxKit, and will update this +section when we are able to demonstrate proof-of-concept. -:: +********** + Citation +********** - @software{akash_dhruv_2022_7255632, - author = {Akash Dhruv}, - title = {akashdhruv/BoxKit: October 2022}, - month = oct, - year = 2022, - publisher = {Zenodo}, - version = {22.10}, - doi = {10.5281/zenodo.7255632}, - url = {https://doi.org/10.5281/zenodo.7255632} - } +.. code:: -Help & Support -============== + @software{akash_dhruv_2022_7255632, + author = {Akash Dhruv}, + title = {akashdhruv/BoxKit: October 2022}, + month = oct, + year = 2022, + publisher = {Zenodo}, + version = {22.10}, + doi = {10.5281/zenodo.7255632}, + url = {https://doi.org/10.5281/zenodo.7255632} + } -Please file an issue on the repository page +**************** + Help & Support +**************** +Please file an issue on the repository page .. |Code style: black| image:: https://img.shields.io/badge/code%20style-black-000000.svg :target: https://github.com/psf/black - + .. |FlashX| image:: https://github.com/akashdhruv/BoxKit/workflows/FlashX/badge.svg + .. |FlowX| image:: https://github.com/akashdhruv/BoxKit/workflows/FlowX/badge.svg + .. |Minimal| image:: https://github.com/akashdhruv/BoxKit/workflows/Minimal/badge.svg + .. |Publish| image:: https://github.com/akashdhruv/BoxKit/workflows/Publish/badge.svg From 954e08e564341cc9eb9dcc75ec575e7f4af63ec6 Mon Sep 17 00:00:00 2001 From: akashdhruv Date: Thu, 15 Dec 2022 22:46:04 -0600 Subject: [PATCH 08/26] fix merging issues --- DESIGN.md | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 DESIGN.md diff --git a/DESIGN.md b/DESIGN.md deleted file mode 100644 index 2111659f..00000000 --- a/DESIGN.md +++ /dev/null @@ -1,13 +0,0 @@ -##

Software Design and Developer's Guide for BoxKit

- -### 1. API -BoxKit API details here - -### 2. Library -BoxKit library details here - -### 3. C-Box -C++ backend detailes here - -### 4. Miscellaneous -Miscellaneous details about how to develop frontend methods, etc. From ed12a8c55b92fcacf12bbc16575bdb6e043fa9aa Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Thu, 15 Dec 2022 22:50:02 -0600 Subject: [PATCH 09/26] Update README.rst --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 10040e51..5c5f92d3 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,3 @@ -.. |icon| image:: ./media/icon.svg - :width: 30 - ############### |icon| BoxKit ############### @@ -111,3 +108,6 @@ Please file an issue on the repository page .. |Minimal| image:: https://github.com/akashdhruv/BoxKit/workflows/Minimal/badge.svg .. |Publish| image:: https://github.com/akashdhruv/BoxKit/workflows/Publish/badge.svg + +.. |icon| image:: ./media/icon.svg + :width: 30 From d7c6ce8d4c705d15942c7387b8f46b9f6f36b5b5 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Thu, 15 Dec 2022 22:51:05 -0600 Subject: [PATCH 10/26] Update DESIGN.rst --- DESIGN.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DESIGN.rst b/DESIGN.rst index d27b758e..b9d3b766 100644 --- a/DESIGN.rst +++ b/DESIGN.rst @@ -1,6 +1,3 @@ -.. |icon| image:: ./media/book.svg - :width: 30 - ######################################################### |icon| Software Design and Developer's Guide for BoxKit ######################################################### @@ -28,3 +25,6 @@ C++ backend detailes here *************** Miscellaneous details about how to develop frontend methods, etc. + +.. |icon| image:: ./media/book.svg + :width: 30 From 2b16980fad761f79b99f96d5c890a2bad0fa1afa Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Fri, 16 Dec 2022 14:21:42 -0600 Subject: [PATCH 11/26] Update DESIGN.rst --- DESIGN.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESIGN.rst b/DESIGN.rst index b9d3b766..586df7a8 100644 --- a/DESIGN.rst +++ b/DESIGN.rst @@ -27,4 +27,4 @@ C++ backend detailes here Miscellaneous details about how to develop frontend methods, etc. .. |icon| image:: ./media/book.svg - :width: 30 + :width: 25 From 7282d30f96abc1639d2ed2e9ac38b7caf87314f7 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Wed, 18 Jan 2023 22:32:05 -0600 Subject: [PATCH 12/26] some left over updates --- DESIGN.rst | 6 +++--- boxkit/api/_mean.py | 16 ++++++++++------ boxkit/api/_mergeblocks.py | 4 ---- boxkit/library/_data.py | 27 ++++++++++++++++++++++++--- setup.py | 10 +++++----- 5 files changed, 42 insertions(+), 21 deletions(-) diff --git a/DESIGN.rst b/DESIGN.rst index 586df7a8..d26b3eec 100644 --- a/DESIGN.rst +++ b/DESIGN.rst @@ -1,6 +1,6 @@ -######################################################### - |icon| Software Design and Developer's Guide for BoxKit -######################################################### +############################################## + |icon| Software Design and Developer's Guide +############################################## ***** API diff --git a/boxkit/api/_mean.py b/boxkit/api/_mean.py index 9a30310f..bcba9bad 100644 --- a/boxkit/api/_mean.py +++ b/boxkit/api/_mean.py @@ -33,6 +33,13 @@ def mean_temporal(datasets, varlist, backend="serial", nthreads=1, monitor=False # Create an mean dataset mean_dataset = datasets[0].clone(storage="numpy-memmap") + # loop over varlist append values to + # add it to the mean dataset and perform mean + for varkey in varlist: + mean_dataset.addvar(varkey) + + sample_size = len(datasets) + # Create a block list for reduction, first add # blocks from average_dataset and then loop over # datasets to add blocks from their respective blocklist @@ -42,12 +49,8 @@ def mean_temporal(datasets, varlist, backend="serial", nthreads=1, monitor=False for block, blk_list in zip(dataset.blocklist, blk_reduce_list): blk_list.append(block) - # loop over varlist append values to - # add it to the mean dataset and perform mean for varkey in varlist: - mean_dataset.addvar(varkey) - for varkey in varlist: if monitor: time_reduction = library.Timer("[boxkit.mean_temporal.mean_blk_list]") @@ -58,7 +61,9 @@ def mean_temporal(datasets, varlist, backend="serial", nthreads=1, monitor=False # pass list of parallel objects (blk_list for blk_list in blk_reduce_list), varkey, + sample_size, ) + if monitor: del time_reduction @@ -68,12 +73,11 @@ def mean_temporal(datasets, varlist, backend="serial", nthreads=1, monitor=False return mean_dataset -def mean_blk_list(blk_list, varkey): +def mean_blk_list(blk_list, varkey, sample_size): """ Reduce dataset / compute average """ mean_blk = blk_list[0] - sample_size = len(blk_list[1:]) for work_blk in blk_list[1:]: mean_blk[varkey] = mean_blk[varkey] + work_blk[varkey] / sample_size diff --git a/boxkit/api/_mergeblocks.py b/boxkit/api/_mergeblocks.py index ca079f52..234001aa 100644 --- a/boxkit/api/_mergeblocks.py +++ b/boxkit/api/_mergeblocks.py @@ -76,10 +76,6 @@ def mergeblocks( if monitor: resources.display() - print( - f"[mem_dataset]: {round(sys.getsizeof(dataset._data.variables[varlist[0]][:])/(2**20),2)} MB" - ) - map_blk_to_merged_dset.nthreads = nthreads map_blk_to_merged_dset.monitor = monitor map_blk_to_merged_dset.backend = backend diff --git a/boxkit/library/_data.py b/boxkit/library/_data.py index efc0bd99..753660ad 100644 --- a/boxkit/library/_data.py +++ b/boxkit/library/_data.py @@ -4,7 +4,7 @@ import string import random import shutil - +import h5py import h5pickle import numpy @@ -211,7 +211,7 @@ def _create_h5_datasets(self): pass for varkey in emptykeys: - outputfile = h5pickle.File(os.path.join(self.boxmem, varkey), "w") + outputfile = h5py.File(os.path.join(self.boxmem, varkey), "w") outputshape = [ self.nblocks, @@ -224,9 +224,10 @@ def _create_h5_datasets(self): varkey, data=numpy.zeros(outputshape, dtype=self.dtype[varkey]) ) outputfile.close() + del outputfile self.outfile[varkey] = h5pickle.File( - os.path.join(self.boxmem, varkey), "r+", skip_cache=False + os.path.join(self.boxmem, varkey), "r+", skip_cache=True ) self.variables[varkey] = self.outfile[varkey][varkey] @@ -265,6 +266,12 @@ def _create_numpy_memmap(self): self.variables[varkey] = numpy.memmap( outputfile, dtype=self.dtype[varkey], shape=outputshape, mode="w+" ) + self.variables[varkey].flush() + del self.variables[varkey] + + self.variables[varkey] = numpy.memmap( + outputfile, dtype=self.dtype[varkey], shape=outputshape, mode="r+" + ) self.dtype[varkey] = type(self.variables[varkey]) @@ -312,6 +319,20 @@ def _create_zarr_objects(self): ), dtype=self.dtype[varkey], ) + del self.variables[varkey] + + self.variables[varkey] = zarr.open( + outputfile, + mode="r+", + shape=outputshape, + chunks=( + 1, + self.nzb + 2 * self.zguard, + self.nyb + 2 * self.yguard, + self.nxb + 2 * self.xguard, + ), + dtype=self.dtype[varkey], + ) self.dtype[varkey] = type(self.variables[varkey]) diff --git a/setup.py b/setup.py index 0981ff80..66015a57 100644 --- a/setup.py +++ b/setup.py @@ -11,16 +11,16 @@ # Import bin from current working directory # sys.path.insert makes sure that current file path is searched # first to find this module -import bin.cmd as bin_cmd +import bin.cmd as bin_cmd # pylint: disable=wrong-import-position # Parse README and get long # description -with open("README.rst", mode="r") as readme: +with open("README.rst", mode="r", encoding="ascii") as readme: long_description = readme.read() # Open metadata file to extract package information -with open("boxkit/__meta__.py", mode="r") as source: +with open("boxkit/__meta__.py", mode="r", encoding="ascii") as source: content = source.read().strip() metadata = { key: re.search(key + r'\s*=\s*[\'"]([^\'"]*)[\'"]', content).group(1) @@ -34,8 +34,8 @@ } # core dependancies for the package -with open('requirements/core.txt') as reqs: - DEPENDENCIES = reqs.read() +with open("requirements/core.txt", mode="r", encoding="ascii") as core_reqs: + DEPENDENCIES = core_reqs.read() # Call setup command with necessary arguments # replace existing build and develop commands From daa8b4c32a6d4994d8476af3f142b4f6933e4bd9 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Tue, 28 Mar 2023 16:37:38 -0500 Subject: [PATCH 13/26] bug fix in relative location calculation --- boxkit/api/_mergeblocks.py | 6 +++--- boxkit/library/_action.py | 2 +- boxkit/library/_block.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/boxkit/api/_mergeblocks.py b/boxkit/api/_mergeblocks.py index 234001aa..07e6d71e 100644 --- a/boxkit/api/_mergeblocks.py +++ b/boxkit/api/_mergeblocks.py @@ -34,9 +34,9 @@ def mergeblocks( ) # Compute number of blocks in each direction - nblockx = int((dataset.xmax - dataset.xmin) / dataset.blocklist[0].dx / dataset.nxb) - nblocky = int((dataset.ymax - dataset.ymin) / dataset.blocklist[0].dy / dataset.nyb) - nblockz = int((dataset.zmax - dataset.zmin) / dataset.blocklist[0].dz / dataset.nzb) + nblockx = round((dataset.xmax - dataset.xmin) / dataset.blocklist[0].dx / dataset.nxb) + nblocky = round((dataset.ymax - dataset.ymin) / dataset.blocklist[0].dy / dataset.nyb) + nblockz = round((dataset.zmax - dataset.zmin) / dataset.blocklist[0].dz / dataset.nzb) nblockx, nblocky, nblockz = [ value if value > 0 else 1 for value in [nblockx, nblocky, nblockz] diff --git a/boxkit/library/_action.py b/boxkit/library/_action.py index f46d5919..94dad9a7 100644 --- a/boxkit/library/_action.py +++ b/boxkit/library/_action.py @@ -89,6 +89,6 @@ def chk_obj_list(obj_list): for index, parallel_obj in enumerate(obj_list): if not isinstance(parallel_obj, type(first_obj)): raise ValueError( - "[boxkit.library.Action] Inconsistent type at index " + f"[boxkit.library.Action] Inconsistent type {type(parallel_obj)} vs {type(first_obj)} at index " + f'"{index}" in parallel object list' ) diff --git a/boxkit/library/_block.py b/boxkit/library/_block.py index 6d5fb0c5..86e399e2 100644 --- a/boxkit/library/_block.py +++ b/boxkit/library/_block.py @@ -252,9 +252,9 @@ def get_relative_loc(self, origin): Get offset from origin """ iloc, jloc, kloc = [ - math.ceil((self.xmin - origin[0]) / (self.xmax - self.xmin + 1e-13)), - math.ceil((self.ymin - origin[1]) / (self.ymax - self.ymin + 1e-13)), - math.ceil((self.zmin - origin[2]) / (self.zmax - self.zmin + 1e-13)), + round((self.xmin - origin[0]) / (self.xmax - self.xmin + 1e-13)), + round((self.ymin - origin[1]) / (self.ymax - self.ymin + 1e-13)), + round((self.zmin - origin[2]) / (self.zmax - self.zmin + 1e-13)), ] return iloc, jloc, kloc From fbd8b6cb8fbfc2ebb91fcfb7d44d6e409fe4b844 Mon Sep 17 00:00:00 2001 From: akashdhruv Date: Thu, 15 Jun 2023 15:22:16 -0500 Subject: [PATCH 14/26] add JOSS paper folder --- paper/Makefile | 2 + paper/paper.bib | 59 ++++++++++++++++++++++++++++ paper/paper.md | 100 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 paper/Makefile create mode 100644 paper/paper.bib create mode 100644 paper/paper.md diff --git a/paper/Makefile b/paper/Makefile new file mode 100644 index 00000000..a0c457f4 --- /dev/null +++ b/paper/Makefile @@ -0,0 +1,2 @@ +all: + docker run --rm --volume $PWD:/data --user $(id -u):$(id -g) --env JOURNAL=joss openjournals/inara diff --git a/paper/paper.bib b/paper/paper.bib new file mode 100644 index 00000000..4e4544a4 --- /dev/null +++ b/paper/paper.bib @@ -0,0 +1,59 @@ +@article{Pearson:2017, + url = {http://adsabs.harvard.edu/abs/2017arXiv170304627P}, + Archiveprefix = {arXiv}, + Author = {{Pearson}, S. and {Price-Whelan}, A.~M. and {Johnston}, K.~V.}, + Eprint = {1703.04627}, + Journal = {ArXiv e-prints}, + Keywords = {Astrophysics - Astrophysics of Galaxies}, + Month = mar, + Title = {{Gaps in Globular Cluster Streams: Pal 5 and the Galactic Bar}}, + Year = 2017 +} + +@book{Binney:2008, + url = {http://adsabs.harvard.edu/abs/2008gady.book.....B}, + Author = {{Binney}, J. and {Tremaine}, S.}, + Booktitle = {Galactic Dynamics: Second Edition, by James Binney and Scott Tremaine.~ISBN 978-0-691-13026-2 (HB).~Published by Princeton University Press, Princeton, NJ USA, 2008.}, + Publisher = {Princeton University Press}, + Title = {{Galactic Dynamics: Second Edition}}, + Year = 2008 +} + +@article{gaia, + author = {{Gaia Collaboration}}, + title = "{The Gaia mission}", + journal = {Astronomy and Astrophysics}, + archivePrefix = "arXiv", + eprint = {1609.04153}, + primaryClass = "astro-ph.IM", + keywords = {space vehicles: instruments, Galaxy: structure, astrometry, parallaxes, proper motions, telescopes}, + year = 2016, + month = nov, + volume = 595, + doi = {10.1051/0004-6361/201629272}, + url = {http://adsabs.harvard.edu/abs/2016A%26A...595A...1G}, +} + +@article{astropy, + author = {{Astropy Collaboration}}, + title = "{Astropy: A community Python package for astronomy}", + journal = {Astronomy and Astrophysics}, + archivePrefix = "arXiv", + eprint = {1307.6212}, + primaryClass = "astro-ph.IM", + keywords = {methods: data analysis, methods: miscellaneous, virtual observatory tools}, + year = 2013, + month = oct, + volume = 558, + doi = {10.1051/0004-6361/201322068}, + url = {http://adsabs.harvard.edu/abs/2013A%26A...558A..33A} +} + +@misc{fidgit, + author = {A. M. Smith and K. Thaney and M. Hahnel}, + title = {Fidgit: An ungodly union of GitHub and Figshare}, + year = {2020}, + publisher = {GitHub}, + journal = {GitHub repository}, + url = {https://github.com/arfon/fidgit} +} diff --git a/paper/paper.md b/paper/paper.md new file mode 100644 index 00000000..ac575f33 --- /dev/null +++ b/paper/paper.md @@ -0,0 +1,100 @@ +--- +title: 'BoxKit' +tags: + - Python + - block structured datasets + - data analysis + - machine learning + - performance optimization +authors: + - name: Akash Dhruv + orcid: 0000-0003-4997-321X + equal-contrib: true + affiliation: 1 +affiliations: + - name: Argonne National Laboratory, USA + index: 1 +date: 15 June 2023 +bibliography: paper.bib +--- + +# Summary + +The forces on stars, galaxies, and dark matter under external gravitational +fields lead to the dynamical evolution of structures in the universe. The orbits +of these bodies are therefore key to understanding the formation, history, and +future state of galaxies. The field of "galactic dynamics," which aims to model +the gravitating components of galaxies to study their structure and evolution, +is now well-established, commonly taught, and frequently used in astronomy. +Aside from toy problems and demonstrations, the majority of problems require +efficient numerical tools, many of which require the same base code (e.g., for +performing numerical orbit integration). + +# Statement of need + +`Gala` is an Astropy-affiliated Python package for galactic dynamics. Python +enables wrapping low-level languages (e.g., C) for speed without losing +flexibility or ease-of-use in the user-interface. The API for `Gala` was +designed to provide a class-based and user-friendly interface to fast (C or +Cython-optimized) implementations of common operations such as gravitational +potential and force evaluation, orbit integration, dynamical transformations, +and chaos indicators for nonlinear dynamics. `Gala` also relies heavily on and +interfaces well with the implementations of physical units and astronomical +coordinate systems in the `Astropy` package [@astropy] (`astropy.units` and +`astropy.coordinates`). + +`Gala` was designed to be used by both astronomical researchers and by +students in courses on gravitational dynamics or astronomy. It has already been +used in a number of scientific publications [@Pearson:2017] and has also been +used in graduate courses on Galactic dynamics to, e.g., provide interactive +visualizations of textbook material [@Binney:2008]. The combination of speed, +design, and support for Astropy functionality in `Gala` will enable exciting +scientific explorations of forthcoming data releases from the *Gaia* mission +[@gaia] by students and experts alike. + +# Mathematics + +Single dollars ($) are required for inline mathematics e.g. $f(x) = e^{\pi/x}$ + +Double dollars make self-standing equations: + +$$\Theta(x) = \left\{\begin{array}{l} +0\textrm{ if } x < 0\cr +1\textrm{ else} +\end{array}\right.$$ + +You can also use plain \LaTeX for equations +\begin{equation}\label{eq:fourier} +\hat f(\omega) = \int_{-\infty}^{\infty} f(x) e^{i\omega x} dx +\end{equation} +and refer to \autoref{eq:fourier} from text. + +# Citations + +Citations to entries in paper.bib should be in +[rMarkdown](http://rmarkdown.rstudio.com/authoring_bibliographies_and_citations.html) +format. + +If you want to cite a software repository URL (e.g. something on GitHub without a preferred +citation) then you can do it with the example BibTeX entry below for @fidgit. + +For a quick reference, the following citation commands can be used: +- `@author:2001` -> "Author et al. (2001)" +- `[@author:2001]` -> "(Author et al., 2001)" +- `[@author1:2001; @author2:2001]` -> "(Author1 et al., 2001; Author2 et al., 2002)" + +# Figures + +Figures can be included like this: +![Caption for example figure.\label{fig:example}](figure.png) +and referenced from text using \autoref{fig:example}. + +Figure sizes can be customized by adding an optional second parameter: +![Caption for example figure.](figure.png){ width=20% } + +# Acknowledgements + +We acknowledge contributions from Brigitta Sipocz, Syrtis Major, and Semyeong +Oh, and support from Kathryn Johnston during the genesis of this project. + +# References From 06f906b46cfd13817f1908b21eba81d1d6745536 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Thu, 15 Jun 2023 15:37:19 -0500 Subject: [PATCH 15/26] update paper --- paper/.gitignore | 2 ++ paper/Makefile | 2 +- paper/paper.md | 5 ++--- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 paper/.gitignore diff --git a/paper/.gitignore b/paper/.gitignore new file mode 100644 index 00000000..af2a6303 --- /dev/null +++ b/paper/.gitignore @@ -0,0 +1,2 @@ +paper.pdf +paper.jats diff --git a/paper/Makefile b/paper/Makefile index a0c457f4..c762cda7 100644 --- a/paper/Makefile +++ b/paper/Makefile @@ -1,2 +1,2 @@ all: - docker run --rm --volume $PWD:/data --user $(id -u):$(id -g) --env JOURNAL=joss openjournals/inara + docker run --rm --volume $(PWD):/data --user $(id -u):$(id -g) --env JOURNAL=joss openjournals/inara diff --git a/paper/paper.md b/paper/paper.md index ac575f33..da2b80fd 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -1,15 +1,14 @@ --- -title: 'BoxKit' +title: 'BoxKit: A Python library to manage analysis of block-structured simulation datasets' tags: - Python - block structured datasets - - data analysis + - simulation analysis - machine learning - performance optimization authors: - name: Akash Dhruv orcid: 0000-0003-4997-321X - equal-contrib: true affiliation: 1 affiliations: - name: Argonne National Laboratory, USA From cc09407a1286bb6bab313a6b98f1aaa9cbce1b19 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Thu, 15 Jun 2023 23:02:11 -0500 Subject: [PATCH 16/26] remove cyclic imports --- boxkit/library/_action.py | 2 +- boxkit/library/_data.py | 2 +- boxkit/library/_dataset.py | 6 +++--- boxkit/library/_execute.py | 2 +- boxkit/library/_monitor.py | 2 +- boxkit/library/_slice.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/boxkit/library/_action.py b/boxkit/library/_action.py index 94dad9a7..6f2f1c3d 100644 --- a/boxkit/library/_action.py +++ b/boxkit/library/_action.py @@ -2,7 +2,7 @@ from types import GeneratorType import copy -from .. import library +from boxkit import library class Action: # pylint: disable=too-many-arguments diff --git a/boxkit/library/_data.py b/boxkit/library/_data.py index 753660ad..0f12e894 100644 --- a/boxkit/library/_data.py +++ b/boxkit/library/_data.py @@ -8,7 +8,7 @@ import h5pickle import numpy -from .. import options +from boxkit import options if options.dask: import dask.array as dsarray diff --git a/boxkit/library/_dataset.py b/boxkit/library/_dataset.py index 80e4b329..ae8a7961 100644 --- a/boxkit/library/_dataset.py +++ b/boxkit/library/_dataset.py @@ -1,8 +1,8 @@ """Module with implemenetation of Dataset class""" -from . import Block -from . import Data -from . import Action +from boxkit.library import Block +from boxkit.library import Data +from boxkit.library import Action class Dataset: # pylint: disable=too-many-instance-attributes diff --git a/boxkit/library/_execute.py b/boxkit/library/_execute.py index 6c875fd7..103ed41b 100644 --- a/boxkit/library/_execute.py +++ b/boxkit/library/_execute.py @@ -5,7 +5,7 @@ import joblib import tqdm -from .. import options +from boxkit import options if options.dask: import dask # pylint: disable=unused-import diff --git a/boxkit/library/_monitor.py b/boxkit/library/_monitor.py index b7382360..3915c74d 100644 --- a/boxkit/library/_monitor.py +++ b/boxkit/library/_monitor.py @@ -2,7 +2,7 @@ from progress.bar import Bar, ChargingBar # pylint: disable=unused-import -from .. import options +from boxkit import options if options.cbox: from ..cbox.lib import boost as cbox # pylint: disable=c-extension-no-member diff --git a/boxkit/library/_slice.py b/boxkit/library/_slice.py index 851003dd..a7b31ff2 100644 --- a/boxkit/library/_slice.py +++ b/boxkit/library/_slice.py @@ -1,6 +1,6 @@ """Module with implementation of the Slice class.""" -from . import Region +from boxkit.library import Region class Slice(Region): # pylint: disable=too-few-public-methods From 3b436ccdb6d936f7553d8db6aea549b0af5983e8 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Thu, 15 Jun 2023 23:05:34 -0500 Subject: [PATCH 17/26] update action --- boxkit/library/_action.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/boxkit/library/_action.py b/boxkit/library/_action.py index 6f2f1c3d..61272ce3 100644 --- a/boxkit/library/_action.py +++ b/boxkit/library/_action.py @@ -89,6 +89,7 @@ def chk_obj_list(obj_list): for index, parallel_obj in enumerate(obj_list): if not isinstance(parallel_obj, type(first_obj)): raise ValueError( - f"[boxkit.library.Action] Inconsistent type {type(parallel_obj)} vs {type(first_obj)} at index " + f"[boxkit.library.Action] Inconsistent type " + + f"{type(parallel_obj)} vs {type(first_obj)} at index " + f'"{index}" in parallel object list' ) From 6d8229f1deacc4293c3ffee3ccae8589c3d021d0 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Fri, 16 Jun 2023 17:31:58 -0500 Subject: [PATCH 18/26] apply linting exceptions --- .pylintrc | 3 +++ boxkit/api/_mean.py | 2 +- boxkit/api/_mergeblocks.py | 22 +++++++++++++--------- boxkit/api/_read.py | 2 +- boxkit/api/_regionprops.py | 4 ++-- boxkit/api/_resfilter.py | 6 +++--- boxkit/library/_action.py | 4 ++-- boxkit/library/_block.py | 1 - boxkit/library/_dataset.py | 6 +++--- boxkit/library/_slice.py | 4 ++-- boxkit/resources/read/_sample.py | 8 ++++---- 11 files changed, 34 insertions(+), 28 deletions(-) create mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..05fefd3e --- /dev/null +++ b/.pylintrc @@ -0,0 +1,3 @@ +[DESIGN] +disable= + duplicate-code diff --git a/boxkit/api/_mean.py b/boxkit/api/_mean.py index bcba9bad..e029e5b7 100644 --- a/boxkit/api/_mean.py +++ b/boxkit/api/_mean.py @@ -17,7 +17,7 @@ def mean_temporal(datasets, varlist, backend="serial", nthreads=1, monitor=False if isinstance(varlist, str): varlist = [varlist] - # TODO: Add more error handling here to account + # TODO: Add more error handling here to account # pylint: disable=fixme # for consistency between multiple datasets # # Handle errors, compute level of the first diff --git a/boxkit/api/_mergeblocks.py b/boxkit/api/_mergeblocks.py index 07e6d71e..25710b54 100644 --- a/boxkit/api/_mergeblocks.py +++ b/boxkit/api/_mergeblocks.py @@ -1,15 +1,13 @@ """Module with implemenetation of api methods""" -import sys - -from .. import library -from .. import api -from ..library import Action +from boxkit import library # pylint: disable=cyclic-import +from boxkit import api # pylint: disable=cyclic-import +from boxkit.library import Action # pylint: disable=cyclic-import def mergeblocks( dataset, varlist, nthreads=1, batch="auto", monitor=False, backend="serial" -): +): # pylint: disable=too-many-arguments disable=too-many-locals """ Reshaped dataset at a level """ @@ -34,9 +32,15 @@ def mergeblocks( ) # Compute number of blocks in each direction - nblockx = round((dataset.xmax - dataset.xmin) / dataset.blocklist[0].dx / dataset.nxb) - nblocky = round((dataset.ymax - dataset.ymin) / dataset.blocklist[0].dy / dataset.nyb) - nblockz = round((dataset.zmax - dataset.zmin) / dataset.blocklist[0].dz / dataset.nzb) + nblockx = round( + (dataset.xmax - dataset.xmin) / dataset.blocklist[0].dx / dataset.nxb + ) + nblocky = round( + (dataset.ymax - dataset.ymin) / dataset.blocklist[0].dy / dataset.nyb + ) + nblockz = round( + (dataset.zmax - dataset.zmin) / dataset.blocklist[0].dz / dataset.nzb + ) nblockx, nblocky, nblockz = [ value if value > 0 else 1 for value in [nblockx, nblocky, nblockz] diff --git a/boxkit/api/_read.py b/boxkit/api/_read.py index 1abcdef3..41881b67 100644 --- a/boxkit/api/_read.py +++ b/boxkit/api/_read.py @@ -13,7 +13,7 @@ def read_dataset( force_memmap=False, nthreads=1, backend="serial", -): +): # pylint: disable=too-many-arguments disable=too-many-locals """ Create a dataset from a file diff --git a/boxkit/api/_regionprops.py b/boxkit/api/_regionprops.py index 826b5d63..004c59c4 100644 --- a/boxkit/api/_regionprops.py +++ b/boxkit/api/_regionprops.py @@ -3,8 +3,8 @@ import itertools import skimage.measure as skimage_measure -from .. import api -from ..library import Action +from boxkit import api # pylint: disable=cyclic-import +from boxkit.library import Action # pylint: disable=cyclic-import def regionprops(dataset, lsetkey, backend="serial", nthreads=1, monitor=False): diff --git a/boxkit/api/_resfilter.py b/boxkit/api/_resfilter.py index d41be69f..493e8e1d 100644 --- a/boxkit/api/_resfilter.py +++ b/boxkit/api/_resfilter.py @@ -1,12 +1,12 @@ """Module with implemenetation of api reshape methods""" -from .. import library -from .. import api +from boxkit import library # pylint: disable=cyclic-import +from boxkit import api # pylint: disable=cyclic-import def resfilter( dataset, varlist=None, level=1, nthreads=1, monitor=False, backend="serial" -): +): # pylint: disable=too-many-arguments disable=too-many-locals """ Build a pseudo UG dataset from AMR dataset at a level diff --git a/boxkit/library/_action.py b/boxkit/library/_action.py index 61272ce3..fcffd628 100644 --- a/boxkit/library/_action.py +++ b/boxkit/library/_action.py @@ -2,7 +2,7 @@ from types import GeneratorType import copy -from boxkit import library +from boxkit import library # pylint: disable=cyclic-import class Action: # pylint: disable=too-many-arguments @@ -89,7 +89,7 @@ def chk_obj_list(obj_list): for index, parallel_obj in enumerate(obj_list): if not isinstance(parallel_obj, type(first_obj)): raise ValueError( - f"[boxkit.library.Action] Inconsistent type " + "[boxkit.library.Action] Inconsistent type " + f"{type(parallel_obj)} vs {type(first_obj)} at index " + f'"{index}" in parallel object list' ) diff --git a/boxkit/library/_block.py b/boxkit/library/_block.py index 86e399e2..7159624e 100644 --- a/boxkit/library/_block.py +++ b/boxkit/library/_block.py @@ -1,6 +1,5 @@ """Module with implementation of the Block class.""" -import math import numpy import pymorton diff --git a/boxkit/library/_dataset.py b/boxkit/library/_dataset.py index ae8a7961..c72f544a 100644 --- a/boxkit/library/_dataset.py +++ b/boxkit/library/_dataset.py @@ -1,8 +1,8 @@ """Module with implemenetation of Dataset class""" -from boxkit.library import Block -from boxkit.library import Data -from boxkit.library import Action +from boxkit.library import Block # pylint: disable=cyclic-import +from boxkit.library import Data # pylint: disable=cyclic-import +from boxkit.library import Action # pylint: disable=cyclic-import class Dataset: # pylint: disable=too-many-instance-attributes diff --git a/boxkit/library/_slice.py b/boxkit/library/_slice.py index a7b31ff2..af5a829d 100644 --- a/boxkit/library/_slice.py +++ b/boxkit/library/_slice.py @@ -1,6 +1,6 @@ """Module with implementation of the Slice class.""" -from boxkit.library import Region +from boxkit.library import Region # pylint: disable=cyclic-import class Slice(Region): # pylint: disable=too-few-public-methods @@ -8,7 +8,7 @@ class Slice(Region): # pylint: disable=too-few-public-methods type_ = "derived" - def __init__(self, blocklist, **attributes): + def __init__(self, blocklist, **attributes): # pylint: disable=W0235 """Initialize the Slice object and allocate the data. Parameters diff --git a/boxkit/resources/read/_sample.py b/boxkit/resources/read/_sample.py index f967c419..196c99f1 100644 --- a/boxkit/resources/read/_sample.py +++ b/boxkit/resources/read/_sample.py @@ -3,7 +3,7 @@ import h5pickle as h5py -def read_test_sample(filename, server): +def read_test_sample(filename, server): # pylint: disable=too-many-locals """ Read dataset from BoxKit test sample @@ -34,9 +34,9 @@ def read_test_sample(filename, server): xmax = inputfile["boundbox/max"][:, 0] ymax = inputfile["boundbox/max"][:, 1] zmax = inputfile["boundbox/max"][:, 2] - dx = inputfile["deltas"][0] - dy = inputfile["deltas"][1] - dz = inputfile["deltas"][2] + dx = inputfile["deltas"][0] # pylint: disable=invalid-name + dy = inputfile["deltas"][1] # pylint: disable=invalid-name + dz = inputfile["deltas"][2] # pylint: disable=invalid-name variables = {} variables.update(inputfile["quantities"]) From 6dc65f2687bbdd69ab316e16ab007465a76e2f22 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Fri, 16 Jun 2023 17:32:31 -0500 Subject: [PATCH 19/26] update version --- boxkit/__meta__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxkit/__meta__.py b/boxkit/__meta__.py index 65489fe7..14ef0ddc 100644 --- a/boxkit/__meta__.py +++ b/boxkit/__meta__.py @@ -1,7 +1,7 @@ """Metadata for BoxKit package""" __pkgname__ = "BoxKit" -__version__ = "3.0" +__version__ = "2023.06" __authors__ = "Akash Dhruv" __license__ = "Apache License" __copyright__ = "Copyright (c) Akash Dhruv 2021. All Rights Reserved." From 389227dd4a56db7ab005fb3a2105abcf0589afdf Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Fri, 16 Jun 2023 17:39:05 -0500 Subject: [PATCH 20/26] final linting fixes --- boxkit/api/_resfilter.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/boxkit/api/_resfilter.py b/boxkit/api/_resfilter.py index 493e8e1d..6793b2be 100644 --- a/boxkit/api/_resfilter.py +++ b/boxkit/api/_resfilter.py @@ -1,12 +1,13 @@ """Module with implemenetation of api reshape methods""" from boxkit import library # pylint: disable=cyclic-import -from boxkit import api # pylint: disable=cyclic-import + +# from boxkit import api # pylint: disable=cyclic-import def resfilter( dataset, varlist=None, level=1, nthreads=1, monitor=False, backend="serial" -): # pylint: disable=too-many-arguments disable=too-many-locals +): # pylint: disable=too-many-arguments disable=too-many-locals disable=unused-argument """ Build a pseudo UG dataset from AMR dataset at a level @@ -51,20 +52,21 @@ def resfilter( for value in [nblockx_level, nblocky_level, nblockz_level] ] - filtered_dataset = api.create_dataset( - nblockx=nblockx_level, - nblocky=nblocky_level, - nblockz=nblockz_level, - nxb=dataset.nxb, - nyb=dataset.nyb, - nzb=dataset.nzb, - xmin=dataset.xmin, - ymin=dataset.ymin, - zmin=dataset.zmin, - xmax=dataset.xmax, - ymax=dataset.ymax, - zmax=dataset.zmax, - ) + # TODO: uncomment this # pylint: disable=fixme + # filtered_dataset = api.create_dataset( + # nblockx=nblockx_level, + # nblocky=nblocky_level, + # nblockz=nblockz_level, + # nxb=dataset.nxb, + # nyb=dataset.nyb, + # nzb=dataset.nzb, + # xmin=dataset.xmin, + # ymin=dataset.ymin, + # zmin=dataset.zmin, + # xmax=dataset.xmax, + # ymax=dataset.ymax, + # zmax=dataset.zmax, + # ) if monitor: del time_resfilter From dec78aae8150cf993d61b2657a0fcc1e02da093e Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Fri, 16 Jun 2023 17:51:29 -0500 Subject: [PATCH 21/26] update paper --- paper/paper.bib | 72 +++++++++++++------------------------------------ paper/paper.md | 37 +++++++------------------ 2 files changed, 27 insertions(+), 82 deletions(-) diff --git a/paper/paper.bib b/paper/paper.bib index 4e4544a4..3c529537 100644 --- a/paper/paper.bib +++ b/paper/paper.bib @@ -1,59 +1,23 @@ -@article{Pearson:2017, - url = {http://adsabs.harvard.edu/abs/2017arXiv170304627P}, - Archiveprefix = {arXiv}, - Author = {{Pearson}, S. and {Price-Whelan}, A.~M. and {Johnston}, K.~V.}, - Eprint = {1703.04627}, - Journal = {ArXiv e-prints}, - Keywords = {Astrophysics - Astrophysics of Galaxies}, - Month = mar, - Title = {{Gaps in Globular Cluster Streams: Pal 5 and the Galactic Bar}}, - Year = 2017 +@article{DUBEY2022, + Title = {Flash-{X}: A multiphysics simulation software instrument}, + Journal = {SoftwareX}, + Volume = {19}, + Pages = {101168}, + Year = {2022}, + issn = {2352-7110}, + doi = {https://doi.org/10.1016/j.softx.2022.101168}, + url = {https://www.sciencedirect.com/science/article/pii/S2352711022001030}, + Author = {Anshu Dubey and Klaus Weide and Jared O’Neal and Akash Dhruv and Sean Couch and J. Austin Harris and Tom Klosterman and Rajeev Jain and Johann Rudi and Bronson Messer and Michael Pajkos and Jared Carlson and Ran Chu and Mohamed Wahib and Saurabh Chawdhary and Paul M. Ricker and Dongwook Lee and Katie Antypas and Katherine M. Riley and Christopher Daley and Murali Ganapathy and Francis X. Timmes and Dean M. Townsley and Marcos Vanella and John Bachan and Paul M. Rich and Shravan Kumar and Eirik Endeve and W. Raphael Hix and Anthony Mezzacappa and Thomas Papatheodore} } -@book{Binney:2008, - url = {http://adsabs.harvard.edu/abs/2008gady.book.....B}, - Author = {{Binney}, J. and {Tremaine}, S.}, - Booktitle = {Galactic Dynamics: Second Edition, by James Binney and Scott Tremaine.~ISBN 978-0-691-13026-2 (HB).~Published by Princeton University Press, Princeton, NJ USA, 2008.}, - Publisher = {Princeton University Press}, - Title = {{Galactic Dynamics: Second Edition}}, - Year = 2008 +@unpublished{DHRUV2023, + Author = {Akash Dhruv}, + Title = {A Vortex Damping Outflow Forcing for Multiphase Flows with Sharp Interfacial Jumps}, + Year = {2023} } -@article{gaia, - author = {{Gaia Collaboration}}, - title = "{The Gaia mission}", - journal = {Astronomy and Astrophysics}, - archivePrefix = "arXiv", - eprint = {1609.04153}, - primaryClass = "astro-ph.IM", - keywords = {space vehicles: instruments, Galaxy: structure, astrometry, parallaxes, proper motions, telescopes}, - year = 2016, - month = nov, - volume = 595, - doi = {10.1051/0004-6361/201629272}, - url = {http://adsabs.harvard.edu/abs/2016A%26A...595A...1G}, -} - -@article{astropy, - author = {{Astropy Collaboration}}, - title = "{Astropy: A community Python package for astronomy}", - journal = {Astronomy and Astrophysics}, - archivePrefix = "arXiv", - eprint = {1307.6212}, - primaryClass = "astro-ph.IM", - keywords = {methods: data analysis, methods: miscellaneous, virtual observatory tools}, - year = 2013, - month = oct, - volume = 558, - doi = {10.1051/0004-6361/201322068}, - url = {http://adsabs.harvard.edu/abs/2013A%26A...558A..33A} -} - -@misc{fidgit, - author = {A. M. Smith and K. Thaney and M. Hahnel}, - title = {Fidgit: An ungodly union of GitHub and Figshare}, - year = {2020}, - publisher = {GitHub}, - journal = {GitHub repository}, - url = {https://github.com/arfon/fidgit} +@unpublished{HASSAN2023, + Author = {Shakeel Hassan and Arthur Feeney and Akash Dhruv and Jihoon Kim and Youngjoon Suh and Jaiyoung Ryu and Yoonjin Won and Aparna Chandramowlishwaran}, + Title = {{B}ubble{ML}: A Multi-Physics Dataset and Benchmarks for Machine Learning}, + Year = {2023} } diff --git a/paper/paper.md b/paper/paper.md index da2b80fd..3649d7c4 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -19,37 +19,18 @@ bibliography: paper.bib # Summary -The forces on stars, galaxies, and dark matter under external gravitational -fields lead to the dynamical evolution of structures in the universe. The orbits -of these bodies are therefore key to understanding the formation, history, and -future state of galaxies. The field of "galactic dynamics," which aims to model -the gravitating components of galaxies to study their structure and evolution, -is now well-established, commonly taught, and frequently used in astronomy. -Aside from toy problems and demonstrations, the majority of problems require -efficient numerical tools, many of which require the same base code (e.g., for -performing numerical orbit integration). +BoxKit is a library that provides building blocks to parallelize and +scale data science, high performance computing, and machine learning +applications for block-structured datasets. Spatial data from +simulations and experiments can be accessed and managed using tools +available in this library when working with more data analysis oriented +packages like SciKit, FlowNet, and OpticalFlow + # Statement of need -`Gala` is an Astropy-affiliated Python package for galactic dynamics. Python -enables wrapping low-level languages (e.g., C) for speed without losing -flexibility or ease-of-use in the user-interface. The API for `Gala` was -designed to provide a class-based and user-friendly interface to fast (C or -Cython-optimized) implementations of common operations such as gravitational -potential and force evaluation, orbit integration, dynamical transformations, -and chaos indicators for nonlinear dynamics. `Gala` also relies heavily on and -interfaces well with the implementations of physical units and astronomical -coordinate systems in the `Astropy` package [@astropy] (`astropy.units` and -`astropy.coordinates`). - -`Gala` was designed to be used by both astronomical researchers and by -students in courses on gravitational dynamics or astronomy. It has already been -used in a number of scientific publications [@Pearson:2017] and has also been -used in graduate courses on Galactic dynamics to, e.g., provide interactive -visualizations of textbook material [@Binney:2008]. The combination of speed, -design, and support for Astropy functionality in `Gala` will enable exciting -scientific explorations of forthcoming data releases from the *Gaia* mission -[@gaia] by students and experts alike. +Details about why there is software addresses references like +[@DUBEY2022], [@DHRUV2023], and [@HASSAN2023] # Mathematics From b5f5994514c9b8c76cb3b2a70e590cae60eb4417 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Fri, 16 Jun 2023 22:44:15 -0500 Subject: [PATCH 22/26] add github action for building paper --- .github/workflows/draft-pdf.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/draft-pdf.yml diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml new file mode 100644 index 00000000..3279d214 --- /dev/null +++ b/.github/workflows/draft-pdf.yml @@ -0,0 +1,23 @@ +on: [push] + +jobs: + paper: + runs-on: ubuntu-latest + name: Paper Draft + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Build draft PDF + uses: openjournals/openjournals-draft-action@master + with: + journal: joss + # This should be the path to the paper within your repo. + paper-path: paper/paper.md + - name: Upload + uses: actions/upload-artifact@v1 + with: + name: paper + # This is the output path where Pandoc will write the compiled + # PDF. Note, this should be the same directory as the input + # paper.md + path: paper.pdf From 4ba006daba9e1cfe59f2a6012e4a23520c529acb Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Fri, 16 Jun 2023 22:45:39 -0500 Subject: [PATCH 23/26] update workflow --- .github/workflows/draft-pdf.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml index 3279d214..c2b7cdd5 100644 --- a/.github/workflows/draft-pdf.yml +++ b/.github/workflows/draft-pdf.yml @@ -2,6 +2,7 @@ on: [push] jobs: paper: + name: "paper-draft" runs-on: ubuntu-latest name: Paper Draft steps: From 2f79d24cd291163eff12e9cacd5876ac681dad6d Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Fri, 16 Jun 2023 22:47:21 -0500 Subject: [PATCH 24/26] update worfklow again --- .github/workflows/draft-pdf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml index c2b7cdd5..2401209f 100644 --- a/.github/workflows/draft-pdf.yml +++ b/.github/workflows/draft-pdf.yml @@ -21,4 +21,4 @@ jobs: # This is the output path where Pandoc will write the compiled # PDF. Note, this should be the same directory as the input # paper.md - path: paper.pdf + path: paper/paper.pdf From 362135f14433bfb6c534e38e454786b13d0dc140 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Fri, 16 Jun 2023 22:48:49 -0500 Subject: [PATCH 25/26] update name --- .github/workflows/draft-pdf.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml index 2401209f..38646414 100644 --- a/.github/workflows/draft-pdf.yml +++ b/.github/workflows/draft-pdf.yml @@ -1,8 +1,9 @@ +name: draft-pdf + on: [push] jobs: paper: - name: "paper-draft" runs-on: ubuntu-latest name: Paper Draft steps: From 64f9b4b724bcfd1b0f38b7cf7a4af5f6bf5eff10 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Fri, 16 Jun 2023 22:57:29 -0500 Subject: [PATCH 26/26] build draft during PR --- .github/workflows/draft-pdf.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml index 38646414..9f738cd9 100644 --- a/.github/workflows/draft-pdf.yml +++ b/.github/workflows/draft-pdf.yml @@ -1,6 +1,10 @@ name: draft-pdf -on: [push] +on: + pull_request: + branches: + - main + - development jobs: paper: