From c68ec3239c359cbc1200f8d6537f4145d07df437 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Sat, 22 Jan 2022 14:09:16 +0100 Subject: [PATCH 1/5] added config files for matching 800deg2 survey --- .../unions_800/config_tile_match_ext_r.ini | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 example/unions_800/config_tile_match_ext_r.ini diff --git a/example/unions_800/config_tile_match_ext_r.ini b/example/unions_800/config_tile_match_ext_r.ini new file mode 100644 index 000000000..377e92bdc --- /dev/null +++ b/example/unions_800/config_tile_match_ext_r.ini @@ -0,0 +1,104 @@ +# ShapePipe configuration file: match with external catalogue + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_match_ext_r + +# Add date and time to RUN_NAME, optional, default: True +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = match_external_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 4 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MATCH_EXTERNAL_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_combined:make_catalog_runner + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = final_cat + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Distance tolerance in arcsec +TOLERANCE = 0.3 + +# Internal data + +# Column names for matching: ra, dec +COL_MATCH = X_WORLD, Y_WORLD + +# HDU number, optional, default = 2 +HDU = 1 + +# Processing mode, 'CLASSIC' or 'MULTI-EPOCH' +MODE = CLASSIC + +# External data + +# File path +EXTERNAL_CAT_PATH = /n17data/mkilbing/astro/data/CFIS/800deg2/unions.ugriz2.fits + +# Column names for matching: ra, dec +EXTERNAL_COL_MATCH = RA, Dec + +# Column(s) to copy to matched catalogue +EXTERNAL_COL_COPY = CFIS_ID, CFIS_U_MAG_AUTO, CFIS_U_MAGERR_AUTO, CFIS_R_MAG_AUTO, CFIS_R_MAGERR_AUTO, HSC_G_MAG_AUTO, HSC_G_MAGERR_AUTO, PS_i_stk_kron, PS_i_stk_kron_err, PS_z_stk_kron_err, PS_z_stk_kron_err, CFIS_R_StarGal + +# HDU number, optional, default = 1 +EXTERNAL_HDU = 1 + +# Output file pattern, optional, default = 'cat_matched' +OUTPUT_FILE_PATTERN = cat_r_matched + +# Optional, float, can be 'nan': If not None, output not only matched objects but all, +# and mark non-matched objects with MARK_NON_MATCHED +#MARK_NON_MATCHED = nan + From 13076d32622d43c105ce927a47d4d55319c51fb2 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Wed, 26 Jan 2022 14:35:31 +0100 Subject: [PATCH 2/5] Added Xavier as contributor; match external catalogue fix --- example/unions_800/cat_matched.param | 112 ++++++++++++++++++ .../unions_800/config_tile_match_ext_r.ini | 5 +- shapepipe/info.py | 1 + .../match_external_package/match_external.py | 7 +- 4 files changed, 116 insertions(+), 9 deletions(-) create mode 100644 example/unions_800/cat_matched.param diff --git a/example/unions_800/cat_matched.param b/example/unions_800/cat_matched.param new file mode 100644 index 000000000..7fe5cc6cf --- /dev/null +++ b/example/unions_800/cat_matched.param @@ -0,0 +1,112 @@ +# coordinates +XWIN_WORLD +YWIN_WORLD + +# tile ID, for plot of tile-dependent additive bias. +# Can maybe be removed. +TILE_ID + +# flags +FLAGS +IMAFLAGS_ISO +NGMIX_MCAL_FLAGS + +# PSF ellipticity +NGMIX_ELL_PSFo_NOSHEAR + +# spread class +SPREAD_CLASS + +# spread model flag and error +SPREAD_MODEL +SPREADERR_MODEL + +# Number of epochs (exposures) +N_EPOCH +NGMIX_N_EPOCH + +## Shape measurement outputs +## Ngmix: model fitting + +# galaxy ellipticity +NGMIX_ELL_1M +NGMIX_ELL_1P +NGMIX_ELL_2M +NGMIX_ELL_2P +NGMIX_ELL_NOSHEAR +#NGMIX_ELL_ERR_1M +#NGMIX_ELL_ERR_1P +#NGMIX_ELL_ERR_2M +#NGMIX_ELL_ERR_2P +NGMIX_ELL_ERR_NOSHEAR + +# flags +NGMIX_FLAGS_1M +NGMIX_FLAGS_1P +NGMIX_FLAGS_2M +NGMIX_FLAGS_2P +NGMIX_FLAGS_NOSHEAR + +# size and error +NGMIX_T_1M +NGMIX_T_1P +NGMIX_T_2M +NGMIX_T_2P +NGMIX_T_NOSHEAR +NGMIX_T_ERR_1M +NGMIX_T_ERR_1P +NGMIX_T_ERR_2M +NGMIX_T_ERR_2P +NGMIX_T_ERR_NOSHEAR +NGMIX_Tpsf_1M +NGMIX_Tpsf_1P +NGMIX_Tpsf_2M +NGMIX_Tpsf_2P +NGMIX_Tpsf_NOSHEAR + +# flux and error +NGMIX_FLUX_1M +NGMIX_FLUX_1P +NGMIX_FLUX_2M +NGMIX_FLUX_2P +NGMIX_FLUX_NOSHEAR +NGMIX_FLUX_ERR_1M +NGMIX_FLUX_ERR_1P +NGMIX_FLUX_ERR_2M +NGMIX_FLUX_ERR_2P +NGMIX_FLUX_ERR_NOSHEAR + +# Flag for multiple objects due to overlapping +# tiles +FLAG_TILING + +# magnitude, mainly for plots +MAG_AUTO + +# SNR from SExtractor, used for cuts on GALSIM shapes +SNR_WIN + +# PSF size measured on original image +NGMIX_T_PSFo_NOSHEAR + +# PSF size measured on reconvolved image +# NGMIX_Tpsf_NOSHEAR + +# ngmix moment failure flag +NGMIX_MOM_FAIL + +# 800 deg^2 external cat parameters +CFIS_ID +RA +Dec +CFIS_U_MAG_AUTO +CFIS_U_MAGERR_AUTO +CFIS_R_MAG_AUTO +CFIS_R_MAGERR_AUTO +HSC_G_MAG_AUTO +HSC_G_MAGERR_AUTO +PS_i_stk_kron +PS_i_stk_kron_err +PS_z_stk_kron +PS_z_stk_kron_err +CFIS_R_StarGal diff --git a/example/unions_800/config_tile_match_ext_r.ini b/example/unions_800/config_tile_match_ext_r.ini index 377e92bdc..0f9754e6c 100644 --- a/example/unions_800/config_tile_match_ext_r.ini +++ b/example/unions_800/config_tile_match_ext_r.ini @@ -45,7 +45,7 @@ OUTPUT_DIR = $SP_RUN/output [JOB] # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 +SMP_BATCH_SIZE = 16 # Timeout value (optional), default is None, i.e. no timeout limit applied TIMEOUT = 96:00:00 @@ -90,7 +90,7 @@ EXTERNAL_CAT_PATH = /n17data/mkilbing/astro/data/CFIS/800deg2/unions.ugriz2.fits EXTERNAL_COL_MATCH = RA, Dec # Column(s) to copy to matched catalogue -EXTERNAL_COL_COPY = CFIS_ID, CFIS_U_MAG_AUTO, CFIS_U_MAGERR_AUTO, CFIS_R_MAG_AUTO, CFIS_R_MAGERR_AUTO, HSC_G_MAG_AUTO, HSC_G_MAGERR_AUTO, PS_i_stk_kron, PS_i_stk_kron_err, PS_z_stk_kron_err, PS_z_stk_kron_err, CFIS_R_StarGal +EXTERNAL_COL_COPY = CFIS_ID, RA, Dec, CFIS_U_MAG_AUTO, CFIS_U_MAGERR_AUTO, CFIS_R_MAG_AUTO, CFIS_R_MAGERR_AUTO, HSC_G_MAG_AUTO, HSC_G_MAGERR_AUTO, PS_i_stk_kron, PS_i_stk_kron_err, PS_z_stk_kron, PS_z_stk_kron_err, CFIS_R_StarGal # HDU number, optional, default = 1 EXTERNAL_HDU = 1 @@ -101,4 +101,3 @@ OUTPUT_FILE_PATTERN = cat_r_matched # Optional, float, can be 'nan': If not None, output not only matched objects but all, # and mark non-matched objects with MARK_NON_MATCHED #MARK_NON_MATCHED = nan - diff --git a/shapepipe/info.py b/shapepipe/info.py index 198b73791..aeb1e1a36 100644 --- a/shapepipe/info.py +++ b/shapepipe/info.py @@ -76,6 +76,7 @@ def shapepipe_logo(colour=False): Morgan Schmitz Andre Zamorano Vitorelli Francois Lanusse + Xavier Jimenez Version: {} diff --git a/shapepipe/modules/match_external_package/match_external.py b/shapepipe/modules/match_external_package/match_external.py index 1cf4aca22..c490a38dd 100644 --- a/shapepipe/modules/match_external_package/match_external.py +++ b/shapepipe/modules/match_external_package/match_external.py @@ -181,8 +181,6 @@ def process(self): ra, dec = get_ra_dec(data, self._col_match[0], self._col_match[1]) coord = SkyCoord(ra=ra, dec=dec, unit='deg') - # Todo: cut duplicates - # Match objects in external cat to internal cat. indices=indices to # external object for each object in internal cat e.g. # external_coord[indices[0]] is the match for coord[0]. @@ -241,13 +239,12 @@ def process(self): # Write FITS file out_cat = file_io.FITSCatalogue( self._output_path, - SEx_catalogue=True, + SEx_catalogue=False, open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, ) out_cat.save_as_fits( data=matched, ext_name='MATCHED', - sex_cat_path=self._input_file_list[0], ) # Write all extensions if in multi-epoch mode @@ -268,5 +265,3 @@ def process(self): data=matched_me, ext_name=ext_names[hdu_me], ) - - # TODO: Compute stats From e4be5f3c0aa79770e38f40483775a7ee845a525e Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Fri, 4 Feb 2022 14:47:18 +0100 Subject: [PATCH 3/5] match 800 --- example/unions_800/cat_matched.param | 11 ----------- example/unions_800/config_tile_match_ext_r.ini | 8 ++++++-- scripts/python/merge_final_cat.py | 2 +- .../modules/match_external_package/match_external.py | 8 ++++++++ shapepipe/modules/match_external_runner.py | 9 +++++++++ 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/example/unions_800/cat_matched.param b/example/unions_800/cat_matched.param index 7fe5cc6cf..acbbe18d4 100644 --- a/example/unions_800/cat_matched.param +++ b/example/unions_800/cat_matched.param @@ -99,14 +99,3 @@ NGMIX_MOM_FAIL CFIS_ID RA Dec -CFIS_U_MAG_AUTO -CFIS_U_MAGERR_AUTO -CFIS_R_MAG_AUTO -CFIS_R_MAGERR_AUTO -HSC_G_MAG_AUTO -HSC_G_MAGERR_AUTO -PS_i_stk_kron -PS_i_stk_kron_err -PS_z_stk_kron -PS_z_stk_kron_err -CFIS_R_StarGal diff --git a/example/unions_800/config_tile_match_ext_r.ini b/example/unions_800/config_tile_match_ext_r.ini index 0f9754e6c..7506064ac 100644 --- a/example/unions_800/config_tile_match_ext_r.ini +++ b/example/unions_800/config_tile_match_ext_r.ini @@ -45,7 +45,7 @@ OUTPUT_DIR = $SP_RUN/output [JOB] # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 +SMP_BATCH_SIZE = 8 # Timeout value (optional), default is None, i.e. no timeout limit applied TIMEOUT = 96:00:00 @@ -90,7 +90,8 @@ EXTERNAL_CAT_PATH = /n17data/mkilbing/astro/data/CFIS/800deg2/unions.ugriz2.fits EXTERNAL_COL_MATCH = RA, Dec # Column(s) to copy to matched catalogue -EXTERNAL_COL_COPY = CFIS_ID, RA, Dec, CFIS_U_MAG_AUTO, CFIS_U_MAGERR_AUTO, CFIS_R_MAG_AUTO, CFIS_R_MAGERR_AUTO, HSC_G_MAG_AUTO, HSC_G_MAGERR_AUTO, PS_i_stk_kron, PS_i_stk_kron_err, PS_z_stk_kron, PS_z_stk_kron_err, CFIS_R_StarGal +#EXTERNAL_COL_COPY = CFIS_ID, RA, Dec, CFIS_U_MAG_AUTO, CFIS_U_MAGERR_AUTO, CFIS_R_MAG_AUTO, CFIS_R_MAGERR_AUTO, HSC_G_MAG_AUTO, HSC_G_MAGERR_AUTO, PS_i_stk_kron, PS_i_stk_kron_err, PS_z_stk_kron, PS_z_stk_kron_err, CFIS_R_StarGal +EXTERNAL_COL_COPY = CFIS_ID, RA, Dec # HDU number, optional, default = 1 EXTERNAL_HDU = 1 @@ -101,3 +102,6 @@ OUTPUT_FILE_PATTERN = cat_r_matched # Optional, float, can be 'nan': If not None, output not only matched objects but all, # and mark non-matched objects with MARK_NON_MATCHED #MARK_NON_MATCHED = nan + +# Optional, bool, add distance to output catalogue +OUTPUT_DISTANCE = True diff --git a/scripts/python/merge_final_cat.py b/scripts/python/merge_final_cat.py index 837ae0986..31acb24ef 100755 --- a/scripts/python/merge_final_cat.py +++ b/scripts/python/merge_final_cat.py @@ -319,7 +319,7 @@ def main(argv=None): continue try: - d_tmp = get_data(idx, 1, param.param_list) + d_tmp = get_data(idx, param.hdu_num, param.param_list) dd = np.zeros(d_tmp.shape, dtype=d.dtype) for key in d_tmp.dtype.names: diff --git a/shapepipe/modules/match_external_package/match_external.py b/shapepipe/modules/match_external_package/match_external.py index 3c3c740c8..1a6146376 100644 --- a/shapepipe/modules/match_external_package/match_external.py +++ b/shapepipe/modules/match_external_package/match_external.py @@ -119,6 +119,8 @@ class MatchCats(object): mark_non_matched : float, optional If not None, output not only matched but all objects, and mark non-matched objects with this value + output_distance : bool, optional, default=False + Output distance between matches if True """ @@ -136,6 +138,7 @@ def __init__( external_col_copy, external_hdu_no=1, mark_non_matched=None, + output_distance=False, ): self._input_file_list = input_file_list @@ -154,6 +157,7 @@ def __init__( self._external_hdu_no = external_hdu_no self._mark_non_matched = mark_non_matched + self._output_distance = output_distance def process(self): """Process. @@ -235,6 +239,10 @@ def process(self): if not indices_close[idx]: matched[col][idx] = self._mark_non_matched + # Output distance if desired + if self._output_distance: + matched['distance'] = d2d[id_data] + # Write FITS file out_cat = file_io.FITSCatalogue( self._output_path, diff --git a/shapepipe/modules/match_external_runner.py b/shapepipe/modules/match_external_runner.py index e2b05d6da..4915596e8 100644 --- a/shapepipe/modules/match_external_runner.py +++ b/shapepipe/modules/match_external_runner.py @@ -81,6 +81,14 @@ def match_external_runner( else: mark_non_matched = None + if config.has_option(module_config_sec, 'OUTPUT_DISTANCE'): + output_distance = config.getbool( + module_config_sec, + 'OUTPUT_DISTANCE', + ) + else: + output_distance = None + # Set output file path file_ext = 'fits' output_path = ( @@ -102,6 +110,7 @@ def match_external_runner( external_col_copy, external_hdu_no=external_hdu_no, mark_non_matched=mark_non_matched, + output_distance=output_distance, ) # Process inputs From d3a9aeb1bd27430d904da3c2d85f6986b139752a Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Wed, 9 Feb 2022 07:58:20 +0100 Subject: [PATCH 4/5] match_ext: output distance --- shapepipe/modules/match_external_package/match_external.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shapepipe/modules/match_external_package/match_external.py b/shapepipe/modules/match_external_package/match_external.py index 1a6146376..e1adad8ef 100644 --- a/shapepipe/modules/match_external_package/match_external.py +++ b/shapepipe/modules/match_external_package/match_external.py @@ -241,7 +241,8 @@ def process(self): # Output distance if desired if self._output_distance: - matched['distance'] = d2d[id_data] + # Output distance in arcsec + matched['distance'] = d2d[id_data].to('arcsec').value # Write FITS file out_cat = file_io.FITSCatalogue( From cb8069219c6e3c59b46c86f9c60ddbde760f4a3c Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Wed, 9 Feb 2022 08:03:25 +0100 Subject: [PATCH 5/5] match_ext output_dist def value --- example/unions_800/cat_matched.param | 1 + shapepipe/modules/match_external_runner.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/example/unions_800/cat_matched.param b/example/unions_800/cat_matched.param index acbbe18d4..0df134071 100644 --- a/example/unions_800/cat_matched.param +++ b/example/unions_800/cat_matched.param @@ -99,3 +99,4 @@ NGMIX_MOM_FAIL CFIS_ID RA Dec +distance diff --git a/shapepipe/modules/match_external_runner.py b/shapepipe/modules/match_external_runner.py index 4915596e8..c1dfd5594 100644 --- a/shapepipe/modules/match_external_runner.py +++ b/shapepipe/modules/match_external_runner.py @@ -82,12 +82,12 @@ def match_external_runner( mark_non_matched = None if config.has_option(module_config_sec, 'OUTPUT_DISTANCE'): - output_distance = config.getbool( + output_distance = config.getboolean( module_config_sec, 'OUTPUT_DISTANCE', ) else: - output_distance = None + output_distance = False # Set output file path file_ext = 'fits'