diff --git a/example/cfis/config_make_cat_psfex_nosm.ini b/example/cfis/config_make_cat_psfex_nosm.ini new file mode 100644 index 00000000..1983f91c --- /dev/null +++ b/example/cfis/config_make_cat_psfex_nosm.ini @@ -0,0 +1,76 @@ +# ShapePipe post-run configuration file: create final catalogs, with +# no spread model on input + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_Mc + +# Add date and time to RUN_NAME, optional, default: True +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = make_cat_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = ./output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MAKE_CAT_RUNNER] + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:psfex_interp_runner, last:merge_sep_cats_runner + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = sexcat, galaxy_psf, ngmix + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits, .sqlite, .fits + +# Numbering convention, string that exemplifies a numbering pattern. +# Matches input single exposures (with 'p' removed) +# Needs to be given in this section, will be updated in module +# sections below +NUMBERING_SCHEME = -000-000 + +SM_DO_CLASSIFICATION = False + +SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh index d712fb8b..ada5dc4f 100755 --- a/scripts/sh/curl_canfar_local.sh +++ b/scripts/sh/curl_canfar_local.sh @@ -27,8 +27,7 @@ sp_local=0 test_only=0 debug_out="-1" scratch="-1" - -script_version=1.1 +sm=1 pat="- " @@ -47,6 +46,8 @@ usage="Usage: $(basename "$0") -j JOB -[e ID |-f file_IDs] -k KIND [OPTIONS] \tmerged header file local (MH=0) or global (MH=1); default is $mh_local\n -s, --sp_local SP\n \tsplit local run local (SP=1) or global (SP=0); default is SP=$sp_local\n + --sm SM\n + \tWith (SM=1; default) or without (SM=0) spread model input\n -N, --N_SMP N_SMOp\n \tnumber of jobs (SMP mode only), default=$N_SMP\n -F, --fix FIX\n @@ -57,8 +58,6 @@ usage="Usage: $(basename "$0") -j JOB -[e ID |-f file_IDs] -k KIND [OPTIONS] \tremote command to run on canfar, default='$cmd_remote'\n -S, --scratch\n \tprocessing scratch directory, default is None ($scratch)\n - -B, --batch\n - \tbatch size = size of subsamples if number of jobs > batch_max\n -b, --batch_max\n \tmaximum batch size = number of jobs run simultaneously, default=$batch_max\n --debug_out PATH\n @@ -98,6 +97,10 @@ while [ $# -gt 0 ]; do sp_local="$2" shift ;; + --sm) + sm="$2" + shift + ;; -e|--exclusive) ID="$2" shift @@ -180,7 +183,6 @@ if [ "$debug_out" != "-1" ]; then echo ${pat}`date` >> $debug_out fi -#. /opt/conda/etc/profile.d/conda.sh source activate shapepipe if [ "$debug_out" != "-1" ]; then echo "${pat}conda prefix = ${CONDA_PREFIX}" >> $debug_out @@ -201,10 +203,11 @@ function submit_batch() { for ID in `cat $path`; do IDt=`echo $ID | tr "." "-"` my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $debug_out $fix $scratch $test_arg + call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg done } +batch=50 if [ "$batch" -ge "$batch_max" ]; then ((batch=batch_max/2)) echo "Reducing batch size to $batch" @@ -226,7 +229,7 @@ if [ "$dry_run" == 2 ]; then for ID in `cat $file_IDs`; do IDt=`echo $ID | tr "." "-"` my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $debug_out $fix $scratch $test_arg + call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg done else @@ -234,7 +237,7 @@ if [ "$dry_run" == 2 ]; then # Submit image (dry run = 2) IDt=`echo $ID | tr "." "-"` my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $debug_out $fix $scratch $test_arg + call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg fi @@ -256,20 +259,20 @@ else echo "Split '$file_IDs' into $n_split batches of size $batch" count=1 - n_running=`stats_jobs_canfar.sh` + n_queued=`stats_jobs_canfar.sh -w all` for batch in $prefix*; do - echo "Number of running jobs = $n_running" + echo "Number of queued jobs = $n_queued" echo "Submitting batch $batch ($count/$n_split)" echo -ne "\033]0;curl patch=$patch job=$job $count/$n_split\007" submit_batch $batch ((count=count+1)) - n_running=`stats_jobs_canfar.sh` + n_queued=`stats_jobs_canfar.sh -w all` - while [ "$n_running" -gt "$n_thresh" ]; do - echo "Wait for #jobs = $n_running jobs to go < $n_thresh ..." + while [ "$n_queued" -gt "$n_thresh" ]; do + echo "Wait for #jobs = $n_queued jobs to go < $n_thresh ..." sleep $sleep - n_running=`stats_jobs_canfar.sh` + n_queued=`stats_jobs_canfar.sh -w all` done done @@ -287,7 +290,7 @@ else # Submit image IDt=`echo $ID | tr "." "-"` my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $debug_out $fix $scratch $test_arg + call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg fi @@ -296,5 +299,5 @@ fi echo "Done $(basename "$0")" if [ "$debug_out" != "-1" ]; then - echo "${pat}End $(basename "$0")" $test_arg >> $debug_out + echo "${pat}End $(basename "$0") $test_arg" >> $debug_out fi diff --git a/scripts/sh/functions.sh b/scripts/sh/functions.sh index e06df3cc..02f6544b 100644 --- a/scripts/sh/functions.sh +++ b/scripts/sh/functions.sh @@ -1,4 +1,4 @@ - # Global variables +# Global variables SSL=~/.ssl/cadcproxy.pem SESSION=https://ws-uv.canfar.net/skaha/v0/session IMAGE=images.canfar.net/unions/shapepipe @@ -26,15 +26,15 @@ function call_curl() { my_dir=$7 my_mh_local=$8 my_sp_local=$9 - my_debug_out=${10} - my_fix=${11} - my_scratch=${12} - my_test_arg=${13} + my_sm=${10} + my_debug_out=${11} + my_fix=${12} + my_scratch=${13} + my_test_arg=${14} - my_arg="-j $my_job -p $my_psf -e $my_ID -N $my_N_SMP -n $my_dry_run -d $my_dir -m $my_mh_local -s $my_sp_local --debug_out $my_debug_out -F $my_fix -S $my_scratch $my_test_arg" + my_arg="-j $my_job -p $my_psf -e $my_ID -N $my_N_SMP -n $my_dry_run -d $my_dir -m $my_mh_local -s $my_sp_local --sm $my_sm --debug_out $my_debug_out -F $my_fix -S $my_scratch $my_test_arg" if [ "$my_dry_run" == "0" ]; then - my_session=`curl -E $SSL "$SESSION?$RESOURCES" -d "image=$IMAGE:$version" -d "name=${my_name}" -d "cmd=$cmd_remote" --data-urlencode "args=${my_arg[@]}"` fi @@ -79,7 +79,7 @@ function command () { res=$? if [ "$debug_out" != "-1" ]; then - echo "${pat}exit code = $res" >> $debug_out + echo "${pat}exit code=$res" >> $debug_out fi if [ $VERBOSE == 1 ]; then @@ -89,6 +89,12 @@ function command () { echo -e "${RED}error, return value = $res${NC}" if [ $STOP == 1 ]; then echo "${RED}exiting $(basename "$0")', error in command '$cmd'${NC}" + if [ "$debug_out" != "-1" ]; then + echo "${pat}${RED}exiting $(basename "$0")', error in command '$cmd'${NC}" >> $debug_out + fi + if [ "$debug_out" != "-1" ]; then + echo "${pat}${RED}exiting $(basename "$0")', error in command '$cmd'${NC}" >> $debug_out + fi exit $res else echo "${RED}continuing '$(basename "$0")', error in command '$cmd'${NC}" diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh index 815dc585..67f62797 100755 --- a/scripts/sh/init_run_exclusive_canfar.sh +++ b/scripts/sh/init_run_exclusive_canfar.sh @@ -504,6 +504,50 @@ if [[ $do_job != 0 ]]; then fi fi +(( do_job = $job & 128 )) +if [[ $do_job != 0 ]]; then + + echo + + cat_ngmix="run_sp_tile_ngmix_Ng1u/ngmix_runner/output/ngmix-*.fits" + dir_ngmix_prev="run_sp_tile_ngmix_Ng1u_prev/ngmix_runner/output" + cat_ngmix_prev="$dir_ngmix_prev/ngmix-*.fits" + + # Check whether ngmix output exists + if [ -e $cat_ngmix ]; then + message "ngmix output catalogue exists" $debug_out -1 + + # Check whether previous ngmix directory and output cat exist + exists="1" + if [ ! -d $dir_ngmix_prev ]; then + exists="0" + elif [ ! -e "$cat_ngmix_prev" ]; then + exists="1" + fi + if [ "$exists" == "0" ]; then + message "Moving to previous batch-save dir (does not exist yet)" $debug_out -1 + command "mkdir -p $dir_ngmix_prev" $dry_run + command "mv $cat_ngmix $dir_ngmix_prev" $dry_run + else + # Compare file sizes + size_cat_ngmix=$(stat -c%s $cat_ngmix) + size_cat_ngmix_prev=$(stat -c%s $cat_ngmix_prev) + if [ "$size_cat_ngmix" -gt "$size_cat_ngmix_prev" ]; then + message "Moving to batch-save dir, overwriting smaller batch-save cat" $debug_out -1 + command "mv $cat_ngmix $dir_ngmix_prev" $dry_run + else + message "Previous batch-save dir not smaller, removing ngmix output" $debug_out -1 + command "rm $cat_ngmix" $dry_run + fi + fi + else + # Whether or not previous ngmix exists, job_sp_canfar will handle it + message "No ngmix output exists, continuing..." $debug_out -1 + fi + + echo +fi + (( do_job = $job & 256 )) if [[ $do_job != 0 ]]; then diff --git a/scripts/sh/job_sp_canfar.bash b/scripts/sh/job_sp_canfar.bash index d6a62f31..54036c93 100755 --- a/scripts/sh/job_sp_canfar.bash +++ b/scripts/sh/job_sp_canfar.bash @@ -22,6 +22,7 @@ results='cosmostat/kilbinger/results_v2' n_smp=-1 nsh_jobs=8 debug_out=-1 +sm=1 pat="--- " @@ -48,6 +49,8 @@ usage="Usage: $(basename "$0") [OPTIONS] [TILE_ID] -s, --star_cat_for_mask\n \tcatalogue for masking bright stars, allowed are 'onthefly', 'save',\n \tdefault is '${star_cat_for_mask}'\n + --sm SM\n + \tWith (SM=1; default) or without (SM=0) spread model input\n -e, --exclusive ID\n \texclusive input filer number string ID (default: None)\n -o, --output_dir\n @@ -96,6 +99,10 @@ while [ $# -gt 0 ]; do star_cat_for_mask="$2" shift ;; + --sm) + sm="$2" + shift + ;; -e|--exclusive) exclusive="$2" shift @@ -141,6 +148,7 @@ if [ "$debug_out" != "-1" ]; then echo "${pat}Starting $(basename "$0")" >> $debug_out fi +CONDA_PREFIX=/arc/home/kilbinger/.conda/envs/shapepipe PATH=$PATH:$CONDA_PREFIX/bin # For tar archives. TODO: Should be unique to each job @@ -196,8 +204,7 @@ function command () { echo "${pat}pwd = `pwd`" >> $debug_out echo "${pat}SP_RUN = $SP_RUN" >> $debug_out echo "${pat}SP_CONFIG = $SP_CONFIG" >> $debug_out - echo "${pat}pwd = `pwd`" >> $debug_out - fi + fi if [ $# == 2 ]; then if [ $VERBOSE == 1 ]; then @@ -214,7 +221,7 @@ function command () { echo "$str: running '$cmd $4 \"$5 $6\"'" fi if [ "$debug_out" != "-1" ]; then - echo "${pat}Running[2] $cmd $4 \"$5 $6\"" >> $debug_out + echo "${pat}Running $cmd $4 \"$5 $6\"" >> $debug_out fi $cmd $4 "$5 $6" @@ -228,7 +235,7 @@ function command () { fi if [ $VERBOSE == 1 ]; then - if [ "$res" == "0" ]; then + if [ $res == 0 ]; then echo -e "${GREEN}success, return value = $res${NC}" else echo -e "${RED}error, return value = $res${NC}" @@ -332,11 +339,6 @@ function update_config() { ### Start ### -command_sp "source activate shapepipe" "Activate conda shapepipe env" -if [ "$debug_out" != "-1" ]; then - echo "${pat}conda prefix = ${CONDA_PREFIX}" >> $debug_out -fi -CONDA_PREFIX=/arc/home/kilbinger/.conda/envs/shapepipe echo "Start processing" # Create input and output directories @@ -460,7 +462,6 @@ if [[ $do_job != 0 ]]; then fi ## Process tiles up to shape measurement -## MKDEBUG new 14/01/25: remove spread_model_runner (( do_job = $job & 64 )) if [[ $do_job != 0 ]]; then @@ -468,8 +469,8 @@ if [[ $do_job != 0 ]]; then letter=${psf:0:1} Letter=${letter^} command_cfg_shapepipe \ - "config_tile_${Letter}iViVi_canfar.ini" \ - "Run shapepipe (tile PsfInterp=$Letter}: up to ngmix" \ + "config_tile_${Letter}iViSmVi_canfar.ini" \ + "Run shapepipe (tile PsfInterp=$Letter}: up to ngmix+galsim)" \ $n_smp \ $exclusive @@ -540,12 +541,11 @@ if [[ $do_job != 0 ]]; then perl -ane \ 's/(N_SPLIT_MAX =) X/$1 '$nsh_jobs'/; print' \ > $SP_CONFIG_MOD/config_merge_sep_cats.ini - + ### Merge separated shapes catalogues command_sp \ "shapepipe_run -c $SP_CONFIG_MOD/config_merge_sep_cats.ini" \ "Run shapepipe (tile: merge sep cats)" \ - $exclusive \ "$VERBOSE" \ "$ID" fi @@ -553,9 +553,16 @@ fi (( do_job = $job & 512 )) if [[ $do_job != 0 ]]; then + # spread_model suffix for config file with or without SM input + if [ "$sm" == "0" ]; then + suff_sm="_nosm" + else + suff_sm="" + fi + ### Merge all relevant information into final catalogue command_cfg_shapepipe \ - "config_make_cat_$psf.ini" \ + "config_make_cat_$psf${suff_sm}.ini" \ "Run shapepipe (tile: create final cat $psf)" \ $n_smp \ $exclusive @@ -575,5 +582,5 @@ if [[ $do_job != 0 ]]; then fi if [ "$debug_out" != "-1" ]; then - echo "${pat}End $(basename "$0") ID=$exclusive" >> $debug_out + echo "${pat}End $(basename "$0") ID=$exclusive success" >> $debug_out fi diff --git a/shapepipe/modules/make_cat_package/make_cat.py b/shapepipe/modules/make_cat_package/make_cat.py index 71aaf55c..c29191e6 100644 --- a/shapepipe/modules/make_cat_package/make_cat.py +++ b/shapepipe/modules/make_cat_package/make_cat.py @@ -145,6 +145,7 @@ def save_sm_data( do_classif=True, star_thresh=0.003, gal_thresh=0.01, + n_obj=-1, ): r"""Save Spread-Model Data. @@ -155,7 +156,8 @@ def save_sm_data( final_cat_file : file_io.FITSCatalogue Final catalogue sexcat_sm_path : str - Path to spread-model catalogue to save. + Path to spread-model catalogue to save. If ``None``, spread_model is + set to 99 do_classif : bool If ``True`` objects will be classified into stars, galaxies, and other, using the classifier @@ -166,6 +168,8 @@ def save_sm_data( gal_thresh : float Threshold for galaxy selection; object is classified as galaxy if :math:`{\rm class} >` ``gal_thresh`` + nobj : int, optional + Number of objects, only used if sexcat_sm_path is ``None`` Returns ------- @@ -174,14 +178,21 @@ def save_sm_data( """ final_cat_file.open() - sexcat_sm_file = file_io.FITSCatalogue(sexcat_sm_path, SEx_catalogue=True) - sexcat_sm_file.open() + if sexcat_sm_path is not None: + sexcat_sm_file = file_io.FITSCatalogue( + sexcat_sm_path, + SEx_catalogue=True, + ) + sexcat_sm_file.open() - sm = np.copy(sexcat_sm_file.get_data()["SPREAD_MODEL"]) - sm_err = np.copy(sexcat_sm_file.get_data()["SPREADERR_MODEL"]) - cat_size = len(sm) + sm = np.copy(sexcat_sm_file.get_data()["SPREAD_MODEL"]) + sm_err = np.copy(sexcat_sm_file.get_data()["SPREADERR_MODEL"] + + sexcat_sm_file.close() - sexcat_sm_file.close() + else: + sm = np.ones(n_obj) * 99 + sm_err = np.ones(n_obj) * 99 final_cat_file.add_col("SPREAD_MODEL", sm) final_cat_file.add_col("SPREADERR_MODEL", sm_err) @@ -310,7 +321,7 @@ def _add2dict(self, key, value, index=None): else: self._output_dict[key] = value - def _save_ngmix_data(self, ngmix_cat_path, moments=False): + def _save_ngmix_data(self, ngmix_cat_path, moments=False, w_log): """Save NGMIX Data. Save the NGMIX catalogue into the final one. @@ -338,6 +349,9 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): ngmix_mom_fail = ngmix_cat_file.get_data()["moments_fail"] + n_obj = len(self._obj_id) + self._w_log.info(f"writing ngmix info for {n_obj} objects") + if moments: m = "m" else: @@ -346,8 +360,8 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): ngmix_mcal_flags = ngmix_cat_file.get_data()["mcal_flags"] ngmix_id = ngmix_cat_file.get_data()["id"] - self._add2dict("NGMIX_N_EPOCH", np.zeros(len(self._obj_id))) - self._add2dict("NGMIX_MOM_FAIL", np.zeros(len(self._obj_id))) + self._add2dict("NGMIX_N_EPOCH", np.zeros(n_obj)) + self._add2dict("NGMIX_MOM_FAIL", np.zeros(n_obj)) prefix = f"NGMIX{m}" @@ -360,7 +374,7 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): f"{prefix}_FLAGS_", f"{prefix}_T_PSFo_", ): - self._update_dict(key_str, np.zeros(len(self._obj_id))) + self._update_dict(key_str, np.zeros(n_obj) for key_str in (f"NGMIX{m}_FLUX_ERR_", f"NGMIX{m}_MAG_ERR_"): self._update_dict(key_str, np.ones(len(self._obj_id)) * -1) for key_str in ( @@ -478,7 +492,7 @@ def _save_galsim_shapes(self, galsim_cat_path): ): self._update_dict(key_str, np.zeros(len(self._obj_id))) for key_str in ("GALSIM_FLUX_ERR_", "GALSIM_MAG_ERR_", "GALSIM_RES_"): - self._update_dict(key_str, np.ones(len(self._obj_id)) * -1) + self._update_dict(key_str, np.ones(n_obj) * -1) for key_str in ( "GALSIM_GAL_ELL_", "GALSIM_GAL_ELL_ERR_", diff --git a/shapepipe/modules/make_cat_runner.py b/shapepipe/modules/make_cat_runner.py index 9feb1804..abc5a03d 100644 --- a/shapepipe/modules/make_cat_runner.py +++ b/shapepipe/modules/make_cat_runner.py @@ -39,14 +39,25 @@ def make_cat_runner( ): """Define The Make Catalogue Runner.""" # Set input file paths - ( - tile_sexcat_path, - sexcat_sm_path, - galaxy_psf_path, - shape1_cat_path, - ) = input_file_list[0:4] - if len(input_file_list) == 5: - shape2_cat_path = input_file_list[4] + if len(input_file_list) == 3: + # No spread model input + ( + tile_sexcat_path, + galaxy_psf_path, + shape1_cat_path, + ) = input_file_list + sexcat_sm_path = None + else: + # With spread model input + ( + tile_sexcat_path, + sexcat_sm_path, + galaxy_psf_path, + shape1_cat_path, + ) = input_file_list[0:4] + if len(input_file_list) == 5: + # With second shape catalogue input + shape2_cat_path = input_file_list[4] # Fetch classification options do_classif = config.getboolean( @@ -85,19 +96,20 @@ def make_cat_runner( # Save SExtractor data w_log.info("Save SExtractor data") - cat_size_sextractor = make_cat.save_sextractor_data( - final_cat_file, - tile_sexcat_path - ) + n_obj = make_cat.save_sextractor_data(final_cat_file, tile_sexcat_path) # Save spread-model data w_log.info("Save spread-model data") - cat_size_sm = make_cat.save_sm_data( + if sexcat_sm_path is None: + w_log.info("No sm cat input, setting spread model to 99") + n_obj + make_cat.save_sm_data( final_cat_file, sexcat_sm_path, do_classif, star_thresh, gal_thresh, + n_obj=n_obj ) if cat_size_sextractor != cat_size_sm: