merge update from dev to main

.gitignore

@@ -11,6 +11,7 @@ dask-worker-space
 *.rst
 boxkit/depends/boost/*
 .sphinx/build/*
+.sphinx/source/media/*
 !.sphinx/source/index.rst
-!README.rst
-!DESIGN.rst
+!./README.rst
+!./DESIGN.rst

.sphinx/setup

@@ -1 +1,17 @@
-sphinx-apidoc -f -P -o source ../../BoxKit/boxkit
+# Excluding __meta__.py, options.py, and cbox
+# from current documentation. If the need arises
+# to include them in future releases, please
+# remove them from excluded paths in sphinx-apidoc
+# command
+
+echo Removing artifacts
+rm -rf source/README.rst source/media module.rst boxkit.*
+
+sphinx-apidoc -fPeM --implicit-namespaces -o source ../../BoxKit/boxkit \
+	../../BoxKit/boxkit/__meta__.py \
+	../../BoxKit/boxkit/options.py \
+	../../BoxKit/boxkit/cbox
+
+echo Copying README.rst
+cp ../../BoxKit/README.rst source/.
+cp -r ../../BoxKit/media source/.

.sphinx/source/conf.py

@@ -13,39 +13,39 @@
 import os
 import sys
 
-sys.path.insert(0, os.path.abspath('../..'))
+sys.path.insert(0, os.path.abspath("../.."))
 
 # -- Project information -----------------------------------------------------
 
-project = 'BoxKit'
-copyright = '2022, Akash Dhruv'
-author = 'Akash Dhruv'
+project = "BoxKit"
+copyright = "2022, Akash Dhruv"
+author = "Akash Dhruv"
 
 
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.autodoc','sphinx.ext.napoleon']
+extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = []
+exclude_patterns = ["modules.rst"]
 
 
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ['_static']

.sphinx/source/index.rst

@@ -1,20 +1,24 @@
-.. BoxKit documentation master file, created by
+..
+   BoxKit documentation master file, created by
    sphinx-quickstart on Thu Jan 20 23:56:00 2022.
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to BoxKit's documentation!
-=====================================
+####################################
+ Welcome to BoxKit's documentation!
+####################################
 
 .. toctree::
    :maxdepth: 2
    :caption: Contents:
 
-   modules
+   README
+   boxkit
 
-Indices and tables
-==================
+####################
+ Indices and tables
+####################
 
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
+-  :ref:`genindex`
+-  :ref:`modindex`
+-  :ref:`search`

README.rst

@@ -9,11 +9,17 @@
 **********
  Overview
 **********
-
-An overview of BoxKit is available in ``paper/paper.md`` that provides a
-summary and statement of need for the package. You can compile it into a
-pdf by running ``make`` in the ``paper`` directory. Please note that the
-``Makefile`` requires Docker to be installed and running on the machine.
+BoxKit is a library that provides building blocks to parallelize and 
+scale data science, statistical analysis, and machine learning
+applications for block-structured simulation datasets. Spatial data
+from simulations can be accessed and managed using tools available 
+in this library to interface with packages like SciKit, PyTorch, and 
+OpticalFlow for post-processing and analysis.
+
+The library provides a Python interface to efficiently access Adaptive 
+Mesh Refinement (AMR) data typical of simulation outputs, and leverages
+multiprocessing libraries like JobLib and Dask to scale analysis on 
+Non-Uniform Memory Access (NUMA) and distributed computing architectures.
 
 **************
  Installation
@@ -173,9 +179,31 @@ a dataset's ``blocklist``
    for block in dset.blocklist:
        print(block["soln"])
 
-For instructions on using parallelization wrapper please read
-``paper/paper.md``. Detailed information on full functionality is
-availabe in documentation (https://akashdhruv.github.io/BoxKit/).
+BoxKit also offers wrappers to scale the process of deploying workflows
+on NUMA and distributed computing architectures by providing decorators
+that can parallelize Python operations over a single data structure to
+operate over a list,
+
+.. code:: python
+
+   from boxkit.library import Action
+
+   # Decorate function on a block with desired configuration for parallelization
+   @Action(num_procs, parallel_backend)
+   def operation_on_block(block, *args):
+       pass
+
+   # Call the function with list of blocks as the first argument
+   operation_on_block((block for block in list_of_blocks), *args)
+
+The ``Action`` wrapper converts the function, ``operation_on_block``,
+into a parallel method which can be deployed on a multinode cluster with
+the desired backend (JobLib/Dask). BoxKit does not interfere with
+parallelization schema of target applications like SciKit, OpticalFlow,
+and PyTorch which function independently using available resources.
+
+Detailed information on full functionality is availabe in documentation
+(https://akashdhruv.github.io/BoxKit/).
 
 **************
  Contribution

bin/cmd.py

@@ -111,12 +111,12 @@ def run(self, user):
 
         with open("boxkit/options.py", "w", encoding="ascii") as optfile:
 
-            optfile.write(f"cbox={self.with_cbox}\n")
-            optfile.write(f"pyarrow={self.with_pyarrow}\n")
-            optfile.write(f"zarr={self.with_zarr}\n")
-            optfile.write(f"dask={self.with_dask}\n")
-            optfile.write(f"server={self.with_server}\n")
-            optfile.write(f"testing={self.enable_testing}\n")
+            optfile.write(f"CBOX={self.with_cbox}\n")
+            optfile.write(f"PYARROW={self.with_pyarrow}\n")
+            optfile.write(f"ZARR={self.with_zarr}\n")
+            optfile.write(f"DASK={self.with_dask}\n")
+            optfile.write(f"SERVER={self.with_server}\n")
+            optfile.write(f"TESTING={self.enable_testing}\n")
 
 
 # replaces the default build command for setup.py

boxkit/__init__.py

@@ -5,5 +5,5 @@
 from . import options
 from . import resources
 
-if options.cbox:
+if options.CBOX:
     from . import cbox

boxkit/__meta__.py

@@ -4,5 +4,6 @@
 __version__ = "2023.06"
 __authors__ = "Akash Dhruv"
 __license__ = "Apache License"
-__copyright__ = "Copyright (c) Akash Dhruv 2021. All Rights Reserved."
-__description__ = "BoxKit Utility"
+__copyright__ = "Copyright (c) Akash Dhruv 2023. All Rights Reserved."
+__description__ = "BoxKit utility"
+

boxkit/api/__init__.py

@@ -10,5 +10,5 @@
 from ._resfilter import resfilter
 from ._mean import mean_temporal
 
-if options.testing:
+if options.TESTING:
     from ._regionprops import regionprops

boxkit/cbox/__init__.py

@@ -1 +1,2 @@
+"""Module with implementation of Cbox library"""
 from . import lib

boxkit/library/__init__.py

@@ -12,5 +12,5 @@
 from ._resources import Resources
 from ._monitor import Monitor
 
-if options.server:
+if options.SERVER:
     from ._server import Server

boxkit/library/_action.py

@@ -12,7 +12,7 @@ class Action:  # pylint: disable=too-many-arguments
 
     @staticmethod
     def toparg(*args):
-        """Method to get top argument from *args"""
+        """Method to get top argument from args"""
 
         top = args[0]
 

boxkit/library/_data.py

@@ -10,16 +10,16 @@
 
 from boxkit import options
 
-if options.dask:
+if options.DASK:
     import dask.array as dsarray
 
-if options.pyarrow:
+if options.PYARROW:
     import pyarrow
 
-if options.zarr:
+if options.ZARR:
     import zarr
 
-if options.cbox:
+if options.CBOX:
     from ..cbox.lib import boost as cbox
 
     _DataBase = cbox.library.Data  # pylint: disable=c-extension-no-member
@@ -280,7 +280,7 @@ def _create_zarr_objects(self):
         Create zarr objects
         """
 
-        if options.zarr:
+        if options.ZARR:
             emptykeys = [
                 key
                 for key, value in self.variables.items()
@@ -370,7 +370,7 @@ def _create_dask_objects(self):
         """
         Create dask array representation of data
         """
-        if options.dask:
+        if options.DASK:
             emptykeys = [
                 key
                 for key, value in self.variables.items()
@@ -402,7 +402,7 @@ def _create_pyarrow_objects(self):
         """
         Create a pyarrow tensor objects
         """
-        if options.pyarrow:
+        if options.PYARROW:
             emptykeys = [
                 key
                 for key, value in self.variables.items()

boxkit/library/_execute.py

@@ -7,11 +7,11 @@
 
 from boxkit import options
 
-if options.dask:
+if options.DASK:
     import dask  # pylint: disable=unused-import
     from dask import distributed
 
-if options.cbox:
+if options.CBOX:
     import ctypes  # pylint: disable=unused-import
     from ..cbox.lib import extern as cbox  # pylint: disable=unused-import
 
@@ -23,10 +23,10 @@ def exectask(action, obj_list, *args, **kwargs):
     action   : action object contains following attributes
 
                target   : function/action operates on a,
-                          parallel_obj ---> def target(parallel_obj, *args)
+                          parallel_obj ---> def target(parallel_obj, args)
 
                           actual call passes,
-                          obj_list ---> target(obj_list, *args)
+                          obj_list ---> target(obj_list, args)
 
                nthreads : number of nthreads (only relevant for parallel operations)
                monitor  : flag (True or False) to show progress bar for action
@@ -101,7 +101,7 @@ def execute_cbox(action, obj_list, *args, **kwargs):
     Wrapper takes in obj_list and additional arguments and
     then applies target operations to individual parallel_objs using boxlib
     """
-    # if options.cbox:
+    # if options.CBOX:
     #    cbox.utilities.execute_pyTask.argtypes = [ctypes.py_object] * 3
     #    cbox.utilities.execute_pyTask.restype = ctypes.py_object
     #
@@ -123,7 +123,7 @@ def execute_dask(action, obj_list, *args, **kwargs):
 
     nthreads = 1 or None reverts to serial mode
     """
-    if options.dask:
+    if options.DASK:
         with distributed.LocalCluster(
             threads_per_worker=None, n_workers=None, processes=False
         ) as cluster, distributed.Client(cluster) as client:

boxkit/library/_monitor.py

@@ -4,7 +4,7 @@
 
 from boxkit import options
 
-if options.cbox:
+if options.CBOX:
     from ..cbox.lib import boost as cbox  # pylint: disable=c-extension-no-member
 
 
@@ -17,7 +17,7 @@ def __init__(self, msg="", iters=1):
         """
         Initialize and create object
         """
-        if options.cbox:
+        if options.CBOX:
             self._bar = cbox.library.Monitor(  # pylint: disable=c-extension-no-member
                 "action"
             )
@@ -31,7 +31,7 @@ def update(self):
         """
         update monitor
         """
-        if options.cbox:
+        if options.CBOX:
             self._bar._update(self.msg, 0)  # pylint: disable=protected-access
         else:
             self._bar.next()
@@ -40,5 +40,5 @@ def finish(self):
         """
         finish
         """
-        if not options.cbox:
+        if not options.CBOX:
             self._bar.finish()