spread_model revision, suggestion of common module format

example/cfis/config_tile_SxMiViSmVi.ini

@@ -21,7 +21,7 @@ RUN_NAME = run_sp_SxMiViSmVi
 # Module name, single string or comma-separated list of valid module runner names
 MODULE = sextractor_runner, mccd_interp_runner,
          vignetmaker_runner, spread_model_runner,
-         vignetmaker_runner2
+         vignetmaker_runner
 
 # Parallel processing mode, SMP or MPI
 MODE = SMP

example/cfis/config_tile_SxPiViSmVi.ini

@@ -21,7 +21,7 @@ RUN_NAME = run_sp_SxPsViSmVi
 # Module name, single string or comma-separated list of valid module runner names
 MODULE = sextractor_runner, psfex_interp_runner,
          vignetmaker_runner, spread_model_runner,
-         vignetmaker_runner2
+         vignetmaker_runner
 
 # Parallel processing mode, SMP or MPI
 MODE = SMP

scripts/jupyter/plot_spectro_areas.ipynb

@@ -1352,7 +1352,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.9"
+   "version": "3.8.8"
   }
  },
  "nbformat": 4,

scripts/python/canfar_avail_results.py

@@ -19,15 +19,11 @@
 
 from optparse import OptionParser
 
-from shapepipe.utilities.canfar import vosHandler
+from shapepipe.utilities.canfar import dir_list
 
 import cfis
 
 
-# Global VLS definition
-vls = vosHandler('vls')
-
-
 def params_default():
     """Set default parameter values.
 
@@ -205,25 +201,7 @@ def check_results(ID_files, input_vos, result_base_names, n_complete, verbose=Fa
     cmd = 'vls'
     vos_dir = 'vos:cfis/{}'.format(input_vos)
 
-    sys.argv = []
-    sys.argv.append(cmd)
-    sys.argv.append(vos_dir)
-    #sys.argv.append('> .vls.tmp')
-    #if certfile:
-        #sys.argv.append('--certfile={}'.format(certfile))
-
-    if verbose:
-        print('Getting vos directory content from vls...')
-    f = io.StringIO()
-
-    try:
-        with redirect_stdout(f):
-            vls()
-    except:
-        print('Error during vls command')
-        raise
-
-    vls_out = f.getvalue()
+    vls_out = dir_list(vos_dir)
 
     n_found = {}
     n_IDs = {}

scripts/python/canfar_run_analyse.py

@@ -171,13 +171,13 @@ def get_status(tile_num):
             final_cat_found = False
             with open(out_name) as out_file:
                 for line in out_file:
-                    m = re.match('Upload final_cat to.*(\d+) files', line)
+                    m = re.match('Upload.*final_cat.*return value = (\.*)', line)
                     if m:
                         final_cat_found = True
                         if m[1] == '0':
-                            status = res_noout, '0 final_cat output files'
+                            status = res_ok, 'successful upload of final_cat'
                         else:
-                            status = res_ok, 'success, {} final_cat output files'.format(m[1])
+                            status = res_noout, 'failed upload of final_cat'
                         break
 
             if final_cat_found == False:

scripts/python/create_sample_results.py

@@ -17,7 +17,7 @@
 import io
 from contextlib import redirect_stdout
 from optparse import OptionParser
-import cfis
+from shapepipe.utilities import cfis
 from shapepipe.utilities.file_system import mkdir
 
 
@@ -244,7 +244,7 @@ def main(argv=None):
 
     input_IDs = read_ID_list(param.input_IDs, verbose=param.verbose)
 
-    result_base_names = ['psfex', 'psfex_interp_exp', 'setools_mask', 'setools_stat', 'setools_plot',
+    result_base_names = ['psfex_interp_exp', 'setools_mask', 'setools_stat', 'setools_plot',
                          'pipeline_flag', 'final_cat', 'logs']
 
     if os.path.isdir(param.output_dir):

scripts/python/merge_final_cat.py

@@ -25,7 +25,7 @@
 
 from tqdm import tqdm
 
-import cfis
+from shapepipe.utilities import cfis
 
 
 class param:
@@ -81,14 +81,25 @@ def parse_options(p_def):
     usage  = "%prog [OPTIONS]"
     parser = OptionParser(usage=usage)
 
-    parser.add_option('-i', '--input_path', dest='input_path', type='string',
-                      default=p_def.input_path,
-                      help='input path, default=\'{}\''.format(p_def.input_path))
+    parser.add_option(
+        '-i',
+        '--input_path',
+        dest='input_path',
+        type='string',
+        default=p_def.input_path,
+        help=f'input path, default=\'{p_def.input_path}\''
+    )
     parser.add_option('-p', '--param_path', dest='param_path', type='string',
                       default=None,
                       help='parameter file path, default=None')
 
-    parser.add_option('-v', '--verbose', dest='verbose', action='store_true', help='verbose output')
+    parser.add_option(
+        '-v',
+        '--verbose',
+        dest='verbose',
+        action='store_true',
+        help='verbose output'
+    )
 
     options, args = parser.parse_args()
 
@@ -176,7 +187,7 @@ def read_param_file(path, verbose=False):
 
     if verbose:
         if len(param_list) > 0: 
-            print('Copying {} columns'.format(len(param_list)), end='')
+            print(f'Copying {len(param_list)} columns', end='')
         else:
             print('Copying all columns', end='')
         print(' into final catalogue')
@@ -259,7 +270,7 @@ def main(argv=None):
         lpath.append(os.path.join(path, this_l))
 
     if param.verbose:
-        print('{} final catalog files found'.format(len(lpath)))
+        print(f'{len(lpath)} final catalog files found')
 
     count = 0
 
@@ -270,39 +281,30 @@ def main(argv=None):
     for key in d_tmp.dtype.names:
         d[key] = d_tmp[key]
     count = count + 1
-    print('File \'{}\' copied ({}/{})'.format(lpath[0], count, len(lpath)))
-
-    #new_dt = np.dtype(d_tmp.dtype.descr + [('TILE_ID', '>i4')])
-    #d = np.zeros(d_tmp.shape, dtype=new_dt)
-
-    #if 'TILE_ID' in d_tmp.dtype.names:
-        #d['TILE_ID'].fill(int(''.join(re.findall('\d+', l[0]))))
-
-    # Read all final catalogues and merge
-    #for i in tqdm(lpath[1:], total=len(lpath)-1):
+    if param.verbose:
+        print(f'File \'lpath[0]{}\' copied ({count}/{len(lpath)})')
 
-    for i in lpath[1:]:
-        if ('final_cat' not in i) | ('.npy' in i):
+    for idx in lpath[1:]:
+        if ('final_cat' not in idx) | ('.npy' in idx):
             continue
 
         try:
-            d_tmp = get_data(i, 1, param.param_list)
+            d_tmp = get_data(idx, 1, param.param_list)
 
             dd = np.zeros(d_tmp.shape, dtype=d_tmp.dtype)
             for key in d_tmp.dtype.names:
                 dd[key] = d_tmp[key]
             count = count + 1
-            print('File \'{}\' copied ({}/{})'.format(i, count, len(lpath)))
-
-            #if 'TILE_ID' in d_tmp.dtype.names:
-                #dd['TILE_ID'].fill(int(''.join(re.findall('\d+', i))))
+            if param.verbose:
+                print(f'File \'{idx}\' copied ({count}/{len(lpath)})')
 
             d = np.concatenate((d, dd))
         except:
-            print('Error while copying file \'{}\''.format(i))
+            print(f'Error while copying file \'{idx}\'')
 
     # Save merged catalogue as numpy binary file
-    print('Saving final catalogue')
+    if param.verbose:
+        print('Saving final catalogue')
     np.save('final_cat.npy', d)
 
     msg = f'{count} catalog files merged with success'

scripts/python/merge_star_cat_mccd.py

@@ -175,6 +175,7 @@ def create_full_cat(input_dir, input_file_pattern, output_path, verbose=False):
         print('Found {} files'.format(len(starcat_names)))
 
     # Initialise columns to be saved
+    x, y = [], []
     ra, dec = [], []
     g1_psf, g2_psf, size_psf = [], [], []
     g1, g2, size = [], [], []
@@ -187,20 +188,23 @@ def create_full_cat(input_dir, input_file_pattern, output_path, verbose=False):
         starcat_j = fits.open(name)
 
         # positions
-        ra += list(starcat_j[1].data['RA_LIST'])
-        dec += list(starcat_j[1].data['DEC_LIST'])
+        x += list(starcat_j[1].data['GLOB_POSITION_IMG_LIST'][:, 0])
+        y += list(starcat_j[1].data['GLOB_POSITION_IMG_LIST'][:, 1])
+
+        ra += list(starcat_j[1].data['RA_LIST'][:])
+        dec += list(starcat_j[1].data['DEC_LIST'][:])
 
         # shapes (convert sigmas to R^2)
-        g1_psf += list(starcat_j[1].data['PSF_MOM_LIST'][0])
-        g2_psf += list(starcat_j[1].data['PSF_MOM_LIST'][1])
-        size_psf += list(starcat_j[1].data['PSF_MOM_LIST'][2]**2)
-        g1 += list(starcat_j[1].data['STAR_MOM_LIST'][0])
-        g2 += list(starcat_j[1].data['STAR_MOM_LIST'][1])
-        size += list(starcat_j[1].data['STAR_MOM_LIST'][2]**2)
+        g1_psf += list(starcat_j[1].data['PSF_MOM_LIST'][:,0])
+        g2_psf += list(starcat_j[1].data['PSF_MOM_LIST'][:,1])
+        size_psf += list(starcat_j[1].data['PSF_MOM_LIST'][:,2]**2)
+        g1 += list(starcat_j[1].data['STAR_MOM_LIST'][:,0])
+        g2 += list(starcat_j[1].data['STAR_MOM_LIST'][:,1])
+        size += list(starcat_j[1].data['STAR_MOM_LIST'][:,2]**2)
 
         # flags
-        flag_psf += list(starcat_j[1].data['PSF_MOM_LIST'][3])
-        flag_star += list(starcat_j[1].data['STAR_MOM_LIST'][3])
+        flag_psf += list(starcat_j[1].data['PSF_MOM_LIST'][:,3])
+        flag_star += list(starcat_j[1].data['STAR_MOM_LIST'][:,3])
 
         # CCD number
         ccd_nb = list(starcat_j[1].data['CCD_ID_LIST'])
@@ -211,7 +215,7 @@ def create_full_cat(input_dir, input_file_pattern, output_path, verbose=False):
 
     # Collect columns
     # convert back to sigma for consistency
-    data = {'RA': ra, 'DEC': dec,
+    data = {'X': x, 'Y': y, 'RA': ra, 'DEC': dec,
             'E1_PSF_HSM': g1_psf, 'E2_PSF_HSM': g2_psf, 'SIGMA_PSF_HSM': np.sqrt(size_psf),
             'E1_STAR_HSM': g1, 'E2_STAR_HSM': g2, 'SIGMA_STAR_HSM': np.sqrt(size),
             'FLAG_PSF_HSM': flag_psf, 'FLAG_STAR_HSM': flag_star,

scripts/python/merge_star_cat_psfex.py

@@ -24,7 +24,7 @@
 from optparse import OptionParser
 from astropy.io import fits
 
-import cfis
+from shapepipe.utilities import cfis
 from shapepipe.pipeline import file_io as sc
 
 

scripts/sh/canfar_submit_selection.sh

@@ -11,7 +11,7 @@
 
 # Job file for one tile
 SP_ROOT=$HOME/shapepipe
-sp_job="$SP_ROOT/scripts/sh/canfar_sp.bash"
+sp_job="$SP_ROOT/scripts/sh/job_sp.bash"
 
 
 # Functions

scripts/sh/job_sp.bash

@@ -16,8 +16,9 @@
 ## Default values
 do_env=0
 job=255
-psf='mccd'
+psf='psfex'
 retrieve='vos'
+RESULTS='cosmostat/kilbinger/results'
 nsh_step=3500
 nsh_max=-1
 nsh_jobs=8
@@ -40,6 +41,8 @@ usage="Usage: $(basename "$0") [OPTIONS] TILE_ID_1 [TILE_ID_2 [...]]
     \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n
    -r, --retrieve METHOD\n
    \tmethod to retrieve images, one in ['vos'|'symlink]', default='$retrieve'\n
+   -o, --output_dir\n
+   \toutput (upload) directory on vos:cfis, default='$RESULTS'\n
    --nsh_jobs NJOB\n
    \tnumber of shape measurement parallel jobs, default=$nsh_jobs\n
    --nsh_step NSTEP\n
@@ -129,9 +132,6 @@ fi
 # SExtractor library bug work-around
 export PATH="$PATH:$VM_HOME/bin"
 
-# Results upload subdirectory on vos
-RESULTS=results_$psf
-
 ## Path variables used in shapepipe config files
 
 # Run path and location of input image directories
@@ -242,7 +242,7 @@ function upload() {
       fi
    fi
    tar czf ${base}_${ID}.tgz ${upl[@]}
-   command "$VCP ${base}_${ID}.tgz vos:cfis/cosmostat/kilbinger/$RESULTS" "Upload log tar ball"
+   command "$VCP ${base}_${ID}.tgz vos:cfis/$RESULTS" "Upload log tar ball"
 }
 
 # Upload log files
@@ -361,17 +361,17 @@ if [[ $do_job != 0 ]]; then
 
   ### The following are very a bad hacks to get additional input file paths
   if [ "$psf" == "psfex" ]; then
-    input_psfex=`find . -name star_split_ratio_80-*.psf | head -n 1`
+    input_psfex=`ls -1 ./output/*/psfex_runner/output/star_split_ratio_80*.psf | head -n 1`
     command_sp "ln -s `dirname $input_psfex` input_psfex" "Link psfex output"
   else
     input_psf_mccd=`find . -name "fitted_model*.npy" | head -n 1`
     command_sp "ln -s `dirname $input_psf_mccd` input_psf_mccd" "Link MCCD output"
   fi
 
-  input_split_exp=`find output -name flag-*.fits | head -n 1`
+  input_split_exp=`ls -1 ./output/*/split_exp_runner/output/flag*.fits | head -n 1`
   command_sp "ln -s `dirname $input_split_exp` input_split_exp" "Link split_exp output"
 
-  input_sextractor=`find . -name sexcat_sexcat-*.fits | head -n 1`
+  input_sextractor=`ls -1 ./output/*/sextractor_runner/output/sexcat_sexcat*.fits | head -n 1`
   command_sp "ln -s `dirname $input_sextractor` input_sextractor" "Link sextractor output"
 
 fi

shapepipe/modules/spread_model_package/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+"""VIGNET MAKER PACKAGE
+
+This package contains the module(s) for ``spread_model``.
+
+:Author: Axel Guinot
+
+"""
+
+__all__ = ['spread_model']