spread model optional

example/cfis/config_make_cat_psfex_nosm.ini

@@ -0,0 +1,76 @@
+# ShapePipe post-run configuration file: create final catalogs, with
+# no spread model on input
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = True
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_Mc
+
+# Add date and time to RUN_NAME, optional, default: True
+; RUN_DATETIME = False
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = make_cat_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN
+INPUT_DIR = .
+
+# Output directory
+OUTPUT_DIR = ./output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 8
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+[MAKE_CAT_RUNNER]
+
+# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN
+INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:psfex_interp_runner, last:merge_sep_cats_runner
+
+# Input file pattern(s), list of strings with length matching number of expected input file types
+# Cannot contain wild cards
+FILE_PATTERN = sexcat, galaxy_psf, ngmix
+
+# FILE_EXT (optional) list of string extensions to identify input files
+FILE_EXT = .fits, .sqlite, .fits
+
+# Numbering convention, string that exemplifies a numbering pattern.
+# Matches input single exposures (with 'p' removed)
+# Needs to be given in this section, will be updated in module
+# sections below
+NUMBERING_SCHEME = -000-000
+
+SM_DO_CLASSIFICATION = False
+
+SHAPE_MEASUREMENT_TYPE = ngmix

scripts/sh/curl_canfar_local.sh

@@ -27,8 +27,7 @@ sp_local=0
 test_only=0
 debug_out="-1"
 scratch="-1"
-
-script_version=1.1
+sm=1
 
 pat="- "
 
@@ -47,6 +46,8 @@ usage="Usage: $(basename "$0") -j JOB -[e ID |-f file_IDs] -k KIND [OPTIONS]
     \tmerged header file local (MH=0) or global (MH=1); default is $mh_local\n
    -s, --sp_local SP\n
     \tsplit local run local (SP=1) or global (SP=0); default is SP=$sp_local\n
+   --sm SM\n
+    \tWith (SM=1; default) or without (SM=0) spread model input\n
    -N, --N_SMP N_SMOp\n
     \tnumber of jobs (SMP mode only), default=$N_SMP\n
    -F, --fix FIX\n
@@ -57,8 +58,6 @@ usage="Usage: $(basename "$0") -j JOB -[e ID |-f file_IDs] -k KIND [OPTIONS]
     \tremote command to run on canfar, default='$cmd_remote'\n
    -S, --scratch\n
     \tprocessing scratch directory, default is None ($scratch)\n
-   -B, --batch\n
-    \tbatch size = size of subsamples if number of jobs > batch_max\n 
    -b, --batch_max\n
     \tmaximum batch size = number of jobs run simultaneously, default=$batch_max\n
    --debug_out PATH\n
@@ -98,6 +97,10 @@ while [ $# -gt 0 ]; do
       sp_local="$2"
       shift
       ;;
+    --sm)
+      sm="$2"
+      shift
+      ;;  
     -e|--exclusive)
       ID="$2"
       shift
@@ -180,7 +183,6 @@ if [ "$debug_out" != "-1" ]; then
   echo ${pat}`date` >> $debug_out
 fi
 
-#. /opt/conda/etc/profile.d/conda.sh
 source activate shapepipe
 if [ "$debug_out"  != "-1" ]; then
     echo "${pat}conda prefix = ${CONDA_PREFIX}" >> $debug_out
@@ -201,10 +203,11 @@ function submit_batch() {
   for ID in `cat $path`; do
     IDt=`echo $ID | tr "." "-"`
     my_name="SP-${patch}-J${job}-${IDt}"
-    call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $debug_out $fix $scratch $test_arg
+    call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg
   done
 }
 
+batch=50
 if [ "$batch" -ge "$batch_max" ]; then
   ((batch=batch_max/2))
   echo "Reducing batch size to $batch"
@@ -226,15 +229,15 @@ if [ "$dry_run" == 2 ]; then
     for ID in `cat $file_IDs`; do
       IDt=`echo $ID | tr "." "-"`
       my_name="SP-${patch}-J${job}-${IDt}"
-      call_curl $my_name $job  $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $debug_out $fix $scratch $test_arg
+      call_curl $my_name $job  $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg
     done
 
   else
 
     # Submit image (dry run = 2)
     IDt=`echo $ID | tr "." "-"`
     my_name="SP-${patch}-J${job}-${IDt}"
-    call_curl $my_name $job  $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $debug_out $fix $scratch $test_arg
+    call_curl $my_name $job  $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg
 
   fi
 
@@ -256,20 +259,20 @@ else
       echo "Split '$file_IDs' into $n_split batches of size $batch"
 
       count=1
-      n_running=`stats_jobs_canfar.sh`
+      n_queued=`stats_jobs_canfar.sh -w all`
       for batch in $prefix*; do
-        echo "Number of running jobs = $n_running"
+        echo "Number of queued jobs = $n_queued"
         echo "Submitting batch $batch ($count/$n_split)"
         echo -ne "\033]0;curl patch=$patch job=$job $count/$n_split\007"
         submit_batch $batch
         ((count=count+1))
 
-        n_running=`stats_jobs_canfar.sh`
+        n_queued=`stats_jobs_canfar.sh -w all`
 
-        while [ "$n_running" -gt "$n_thresh" ]; do
-          echo "Wait for #jobs = $n_running jobs to go < $n_thresh ..."
+        while [ "$n_queued" -gt "$n_thresh" ]; do
+          echo "Wait for #jobs = $n_queued jobs to go < $n_thresh ..."
           sleep $sleep
-          n_running=`stats_jobs_canfar.sh`
+          n_queued=`stats_jobs_canfar.sh -w all`
         done
 
       done
@@ -287,7 +290,7 @@ else
     # Submit image
     IDt=`echo $ID | tr "." "-"`
     my_name="SP-${patch}-J${job}-${IDt}"
-    call_curl $my_name $job  $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $debug_out $fix $scratch $test_arg
+    call_curl $my_name $job  $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg
 
   fi
 
@@ -296,5 +299,5 @@ fi
 echo "Done $(basename "$0")" 
 
 if [ "$debug_out" != "-1" ]; then
-  echo "${pat}End $(basename "$0")" $test_arg >> $debug_out
+  echo "${pat}End $(basename "$0") $test_arg" >> $debug_out
 fi

scripts/sh/functions.sh

@@ -1,4 +1,4 @@
-    # Global variables
+# Global variables
 SSL=~/.ssl/cadcproxy.pem
 SESSION=https://ws-uv.canfar.net/skaha/v0/session
 IMAGE=images.canfar.net/unions/shapepipe
@@ -26,15 +26,15 @@ function call_curl() {
   my_dir=$7
   my_mh_local=$8
   my_sp_local=$9
-  my_debug_out=${10}
-  my_fix=${11}
-  my_scratch=${12}
-  my_test_arg=${13}
+  my_sm=${10}
+  my_debug_out=${11}
+  my_fix=${12}
+  my_scratch=${13}
+  my_test_arg=${14}
 
-  my_arg="-j $my_job -p $my_psf -e $my_ID -N $my_N_SMP -n $my_dry_run -d $my_dir -m $my_mh_local -s $my_sp_local --debug_out $my_debug_out -F $my_fix -S $my_scratch $my_test_arg"
+  my_arg="-j $my_job -p $my_psf -e $my_ID -N $my_N_SMP -n $my_dry_run -d $my_dir -m $my_mh_local -s $my_sp_local --sm $my_sm --debug_out $my_debug_out -F $my_fix -S $my_scratch $my_test_arg"
 
   if [ "$my_dry_run" == "0" ]; then
-
     my_session=`curl -E $SSL "$SESSION?$RESOURCES" -d "image=$IMAGE:$version" -d "name=${my_name}" -d "cmd=$cmd_remote" --data-urlencode "args=${my_arg[@]}"`
   fi
 
@@ -79,7 +79,7 @@ function command () {
         res=$?
 
         if [ "$debug_out" != "-1" ]; then
-          echo "${pat}exit code = $res" >> $debug_out
+          echo "${pat}exit code=$res" >> $debug_out
         fi
 
         if [ $VERBOSE == 1 ]; then
@@ -89,6 +89,12 @@ function command () {
               echo -e "${RED}error, return value = $res${NC}"
               if [ $STOP == 1 ]; then
                   echo "${RED}exiting  $(basename "$0")', error in command '$cmd'${NC}"
+                  if [ "$debug_out" != "-1" ]; then
+                      echo "${pat}${RED}exiting  $(basename "$0")', error in command '$cmd'${NC}" >> $debug_out
+                  fi
+                  if [ "$debug_out" != "-1" ]; then
+                      echo "${pat}${RED}exiting  $(basename "$0")', error in command '$cmd'${NC}" >> $debug_out
+                  fi
                   exit $res
               else
                   echo "${RED}continuing '$(basename "$0")', error in command '$cmd'${NC}"

scripts/sh/init_run_exclusive_canfar.sh

@@ -504,6 +504,50 @@ if [[ $do_job != 0 ]]; then
   fi
 fi
 
+(( do_job = $job & 128 ))
+if [[ $do_job != 0 ]]; then
+
+    echo
+
+    cat_ngmix="run_sp_tile_ngmix_Ng1u/ngmix_runner/output/ngmix-*.fits"
+    dir_ngmix_prev="run_sp_tile_ngmix_Ng1u_prev/ngmix_runner/output"
+    cat_ngmix_prev="$dir_ngmix_prev/ngmix-*.fits"
+
+    # Check whether ngmix output exists
+    if [ -e $cat_ngmix ]; then
+        message "ngmix output catalogue exists" $debug_out -1
+
+        # Check whether previous ngmix directory and output cat exist
+        exists="1"
+        if [ ! -d $dir_ngmix_prev ]; then
+            exists="0"
+        elif [ ! -e "$cat_ngmix_prev" ]; then
+            exists="1"
+        fi
+        if [ "$exists" == "0" ]; then
+            message "Moving to previous batch-save dir (does not exist yet)" $debug_out -1
+            command "mkdir -p $dir_ngmix_prev" $dry_run
+            command "mv $cat_ngmix $dir_ngmix_prev" $dry_run
+        else
+            # Compare file sizes
+            size_cat_ngmix=$(stat -c%s $cat_ngmix)
+            size_cat_ngmix_prev=$(stat -c%s $cat_ngmix_prev)
+            if [ "$size_cat_ngmix" -gt "$size_cat_ngmix_prev" ]; then
+                message "Moving to batch-save dir, overwriting smaller batch-save cat" $debug_out -1
+                command "mv $cat_ngmix $dir_ngmix_prev" $dry_run
+            else
+                message "Previous batch-save dir not smaller, removing ngmix output" $debug_out -1
+                command "rm $cat_ngmix" $dry_run
+            fi
+        fi
+    else
+        # Whether or not previous ngmix exists, job_sp_canfar will handle it
+        message "No ngmix output exists, continuing..." $debug_out -1
+    fi
+
+    echo
+fi
+
 (( do_job = $job & 256 ))
 if [[ $do_job != 0 ]]; then