CosmoStat · sfarrens · Sep 16, 2022 · Jun 22, 2022 · Jun 24, 2022 · Jun 28, 2022
diff --git a/example/cfis/config.mask b/example/cfis/config.mask
@@ -1,4 +1,4 @@
-# Mask module configuration file
+# Mask module configuration file for single-exposure images
 
 ## Paths to executables
 [PROGRAM_PATH]
@@ -51,12 +51,23 @@ SPIKE_REG_FILE = spike.reg
 
 MESSIER_MAKE = True
 
-MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.npy
+MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits
 MESSIER_SIZE_PLUS = 0.
 MESSIER_FLAG_VALUE = 16
 
 
-# -------------------------- Missing data parameters ------------------------------
+## NGC mask
+[NGC_PARAMETERS]
+
+NGC_MAKE = True
+
+NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits
+NGC_SIZE_PLUS = 0.
+NGC_FLAG_VALUE = 32
+
+
+
+## Missing data parameters
 [MD_PARAMETERS]
 
 MD_MAKE = False
@@ -66,11 +77,10 @@ MD_THRESH_REMOVE = 0.75
 MD_REMOVE = False
 
 
-# ----------------------------- Other parameters ----------------------------------
+## Other parameters
 [OTHER]
 
 TEMP_DIRECTORY = .temp
 
 KEEP_REG_FILE = False
 KEEP_INDIVIDUAL_MASK = False
-
diff --git a/example/cfis/config_GitFeGie_vos.ini b/example/cfis/config_GitFeGie_vos.ini
@@ -18,7 +18,7 @@ RUN_DATETIME = True
 [EXECUTION]
 
 # Module name, single string or comma-separated list of valid module runner names
-MODULE = get_images_runner, find_exposures_runner, get_images_runner
+MODULE = get_images_runner, find_exposures_runner, get_images_runner            
 
 # Parallel processing mode, SMP or MPI
 MODE = SMP
@@ -146,4 +146,3 @@ N_TRY = 3
 
 # Retrieve command options, optional
 RETRIEVE_OPTIONS = --certfile=$VM_HOME/.ssl/cadcproxy.pem
-
diff --git a/example/cfis/config_MaMa.ini b/example/cfis/config_MaMa.ini
@@ -44,7 +44,7 @@ OUTPUT_DIR = $SP_RUN/output
 [JOB]
 
 # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
-SMP_BATCH_SIZE = 4
+SMP_BATCH_SIZE = 8
 
 # Timeout value (optional), default is None, i.e. no timeout limit applied
 TIMEOUT = 96:00:00

diff --git a/example/cfis/config_exp_SpMh.ini b/example/cfis/config_exp_SpMh.ini
@@ -45,7 +45,7 @@ OUTPUT_DIR = $SP_RUN/output
 [JOB]
 
 # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
-SMP_BATCH_SIZE = 4
+SMP_BATCH_SIZE = 16
 
 # Timeout value (optional), default is None, i.e. no timeout limit applied
 TIMEOUT = 96:00:00

diff --git a/example/cfis/config_tile.mask b/example/cfis/config_tile.mask
@@ -1,4 +1,6 @@
-# ----------------------------- Path ----------------------------------
+# Mask module config file for tiles
+
+## Paths to executables
 [PROGRAM_PATH]
 
 WW_PATH = ww
@@ -8,7 +10,7 @@ WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww
 # (e.g. no internet access on run nodes)
 CDSCLIENT_PATH = findgsc2.2
 
-# ----------------------------- Border parameters ----------------------------------
+## Border parameters
 [BORDER_PARAMETERS]
 
 BORDER_MAKE = False
@@ -17,7 +19,7 @@ BORDER_WIDTH = 0
 BORDER_FLAG_VALUE = 4
 
 
-# ----------------------------- Halo parameters ----------------------------------
+## Halo parameters
 [HALO_PARAMETERS]
 
 HALO_MAKE = True
@@ -30,7 +32,7 @@ HALO_FLAG_VALUE = 2
 HALO_REG_FILE = halo.reg
 
 
-# ------------------------- Diffraction pike parameters ------------------------------
+## Diffraction pike parameters
 [SPIKE_PARAMETERS]
 
 SPIKE_MAKE = True
@@ -43,24 +45,33 @@ SPIKE_FLAG_VALUE = 128
 SPIKE_REG_FILE = spike.reg
 
 
-# ---------------------------- Messier parameters ---------------------------------
+## Messier parameters
 [MESSIER_PARAMETERS]
 
 MESSIER_MAKE = True
 
-MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.npy
+MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits
 MESSIER_PIXEL_SCALE = 0.187
 MESSIER_SIZE_PLUS = 0.
 MESSIER_FLAG_VALUE = 16
 
+## NGC mask                                                                     
+[NGC_PARAMETERS]                                                                
+
+NGC_MAKE = True                                                                 
+
+NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits                              
+NGC_SIZE_PLUS = 0.                                                              
+NGC_FLAG_VALUE = 32
+
 
-# -------------------------------- External flag ----------------------------------
+## External flag
 [EXTERNAL_FLAG]
 
 EF_MAKE = False
 
 
-# -------------------------- Missing data parameters ------------------------------
+## Missing data parameters
 [MD_PARAMETERS]
 
 MD_MAKE = False
@@ -70,7 +81,7 @@ MD_THRESH_REMOVE = 0.75
 MD_REMOVE = False
 
 
-# ----------------------------- Other parameters ----------------------------------
+## Other parameters
 [OTHER]
 
 KEEP_REG_FILE = False

diff --git a/example/cfis/mask_default/Messier_catalog_updated.fits b/example/cfis/mask_default/Messier_catalog_updated.fits
diff --git a/example/cfis/mask_default/Messier_catalog_updated.npy b/example/cfis/mask_default/Messier_catalog_updated.npy
diff --git a/example/cfis/mask_default/ngc_cat.fits b/example/cfis/mask_default/ngc_cat.fits
diff --git a/scripts/python/cfis_field_select.py b/scripts/python/cfis_field_select.py
@@ -26,7 +26,6 @@
 from optparse import OptionParser, IndentedHelpFormatter, OptionGroup
 
 from astropy.io import fits
-from astropy.table import Table, Column
 from astropy import units
 from astropy.coordinates import Angle, SkyCoord
 

diff --git a/shapepipe/modules/get_images_package/__init__.py b/shapepipe/modules/get_images_package/__init__.py
@@ -44,8 +44,8 @@
 INPUT_NUMBERING : str
     Input numbering scheme, python regexp
 CHECK_EXISTING_DIR : str, optional
-    If given, check this directory for existing images, which will then not be
-    downloaded
+    If given, search this directory (recursively) for existing images, which
+    will then not be downloaded
 N_EXPECTED : int, optional
     If ``CHECK_EXISTING_DIR`` is given, this option specifies the number of
     expected images; the default value is ``1``

diff --git a/shapepipe/modules/get_images_package/get_images.py b/shapepipe/modules/get_images_package/get_images.py
@@ -281,9 +281,25 @@ def retrieve(self, all_inputs, all_outputs):
                         f'{self._check_existing_dir}/**/{out_base}',
                         recursive=True,
                     )
-                    if path and len(path) == self._n_expected:
-                        self._w_log.info(f'{path[0]} found, skipping')
-                        continue
+                    if path:
+                        if len(path) == self._n_expected:
+                            self._w_log.info(
+                                f'{path[0]} found, skipping download'
+                            )
+                            continue
+                        else:
+                            self._w_log.info(
+                                f'{len(path)} instead of {self._n_expected} '
+                                + 'existing files found at'
+                                + f' {self._check_existing_dir}'
+                                + ', downloading images'
+                            )
+                    else:
+                        self._w_log.info(
+                            'No existing images found at'
+                            + f' {self._check_existing_dir},'
+                            + ' downloading images'
+                        )
                 self.retrieve_one(in_per_type[idx], out_per_type[idx])
 
     def retrieve_one(self, in_path, out_path):

diff --git a/shapepipe/modules/mask_package/__init__.py b/shapepipe/modules/mask_package/__init__.py
@@ -14,13 +14,17 @@
 Description
 ===========
 
-This module creates masks for bright stars, diffraction spikes, Messier
-objects, borders, and other artifacts. If a flag file is given as input,
-for example from pre-processing, the mask that is created by this module
-is joined with the mask from this external flag file. In this case
-the config flag ``USE_EXT_FLAG`` needs to be set to ``True``. To distinguish
-the newly created output flag file from the input ones, a prefix can added as
-specificed by the config entry ``PREFIX``.
+This module creates masks for bright stars, diffraction spikes, deep sky
+objects (from the Messier and NGC catalogues), borders, and other artifacts. If
+a flag file is given as input, for example from pre-processing, the mask that
+is created by this module is joined with the mask from this external flag file.
+In this case the config flag ``USE_EXT_FLAG`` needs to be set to ``True``. To
+distinguish the newly created output flag file from the input ones, a prefix
+can added as specificed by the config entry ``PREFIX``.
+
+An NGC catalogue with positions, sizes, and types is provided with
+``shapepipe``,
+`source <www.klima-luft.de/steinicke/ngcic/rev2000/Explan.htm>`_
 
 Masked pixels of different mask types are indicated by integers, which
 conveniently are powers of two such that they can be combined bit-wise.
@@ -137,6 +141,18 @@
 MESSIER_FLAG_VALUE : int
     Messier mask pixel value, power of 2
 
+[NGC_PARAMETERS]
+--------------------
+
+NGC_MAKE : bool
+    Create mask around NGC objects if ``True``
+NGC_CAT_PATH : str
+    Path to NGC catalogue
+NGC_SIZE_PLUS : float
+    Fraction to increase NGC mask
+NGC_FLAG_VALUE : int
+    NGC mask pixel value, power of 2
+
 [MD_PARAMETERS]
 ---------------