From e3a3e5ca00e97f3cc3e5e2c40a548238172cdbc0 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Mon, 19 Jul 2021 14:45:25 +0200 Subject: [PATCH 01/26] Update MCCD version --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 16f39f138..f2b63d63a 100644 --- a/environment.yml +++ b/environment.yml @@ -13,7 +13,7 @@ dependencies: - libtool==2.4.6 - matplotlib==3.3.4 - pip: - - mccd==0.0.4 + - mccd==1.1.1 - modopt==1.5.1 - numba==0.53.1 - pandas==1.2.4 From 9d8af3756711b4c76fd15de365f852ac728ba57c Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Mon, 19 Jul 2021 14:48:36 +0200 Subject: [PATCH 02/26] Updating MCCD config files for the new version --- example/cfis/config_MCCD.ini | 22 +++++---- example/test_mccd/config_MCCD.ini | 81 ++++++++++++++++++++----------- 2 files changed, 67 insertions(+), 36 deletions(-) diff --git a/example/cfis/config_MCCD.ini b/example/cfis/config_MCCD.ini index e0b0fd6a0..ac84fb664 100644 --- a/example/cfis/config_MCCD.ini +++ b/example/cfis/config_MCCD.ini @@ -11,23 +11,27 @@ OUTLIER_STD_MAX = 100. USE_SNR_WEIGHTS = False [INSTANCE] -N_COMP_LOC = 12 -D_COMP_GLOB = 3 -KSIG_LOC = 1.0 -KSIG_GLOB = 1.0 +N_COMP_LOC = 4 +D_COMP_GLOB = 6 +KSIG_LOC = 0.00 +KSIG_GLOB = 0.00 FILTER_PATH = None +D_HYB_LOC = 2 +MIN_D_COMP_GLOB = None +RMSE_THRESH = 1.25 +CCD_STAR_THRESH = 0.15 [FIT] LOC_MODEL = hybrid -PSF_SIZE = 6.15 +PSF_SIZE = 6.2 PSF_SIZE_TYPE = R2 N_EIGENVECTS = 5 N_ITER_RCA = 1 N_ITER_GLOB = 2 N_ITER_LOC = 2 -NB_SUBITER_S_LOC = 100 -NB_SUBITER_A_LOC = 500 -NB_SUBITER_S_GLOB = 30 +NB_SUBITER_S_LOC = 300 +NB_SUBITER_A_LOC = 400 +NB_SUBITER_S_GLOB = 100 NB_SUBITER_A_GLOB = 200 [VALIDATION] @@ -39,7 +43,7 @@ VAL_REGEX_FILE_PATTERN = star_split_ratio_20-*-*.fits VAL_SEPARATOR = - APPLY_DEGRADATION = True MCCD_DEBUG = False -GLOBAL_POL_INTERP = True +GLOBAL_POL_INTERP = False # Parameter description: # diff --git a/example/test_mccd/config_MCCD.ini b/example/test_mccd/config_MCCD.ini index 69c9e9c96..c9ec14107 100644 --- a/example/test_mccd/config_MCCD.ini +++ b/example/test_mccd/config_MCCD.ini @@ -1,45 +1,49 @@ # Configuration file for the MCCD method [INPUTS] -INPUT_DIR = /Users/tliaudat/Documents/PhD/github/cosmostat_official/mccd/tests/hidden_tests/sex_inputs/ -PREPROCESSED_OUTPUT_DIR = /Users/tliaudat/Documents/PhD/github/cosmostat_official/mccd/tests/hidden_tests/preprocessed_inputs/ -OUTPUT_DIR = /Users/tliaudat/Documents/PhD/github/cosmostat_official/mccd/tests/hidden_tests/outputs/ +INPUT_DIR = . +PREPROCESSED_OUTPUT_DIR = ./output +OUTPUT_DIR = ./output INPUT_REGEX_FILE_PATTERN = star_split_ratio_80-*-*.fits INPUT_SEPARATOR = - -MIN_N_STARS = 20 +MIN_N_STARS = 22 OUTLIER_STD_MAX = 100. USE_SNR_WEIGHTS = False [INSTANCE] -N_COMP_LOC = 12 -D_COMP_GLOB = 3 -KSIG_LOC = 1.0 -KSIG_GLOB = 1.0 +N_COMP_LOC = 4 +D_COMP_GLOB = 6 +KSIG_LOC = 0.00 +KSIG_GLOB = 0.00 FILTER_PATH = None +D_HYB_LOC = 2 +MIN_D_COMP_GLOB = None +RMSE_THRESH = 1.25 +CCD_STAR_THRESH = 0.15 [FIT] LOC_MODEL = hybrid -PSF_SIZE = 6.15 +PSF_SIZE = 6.2 PSF_SIZE_TYPE = R2 N_EIGENVECTS = 5 N_ITER_RCA = 1 N_ITER_GLOB = 2 N_ITER_LOC = 2 -NB_SUBITER_S_LOC = 100 -NB_SUBITER_A_LOC = 500 -NB_SUBITER_S_GLOB = 30 +NB_SUBITER_S_LOC = 300 +NB_SUBITER_A_LOC = 400 +NB_SUBITER_S_GLOB = 100 NB_SUBITER_A_GLOB = 200 [VALIDATION] -VAL_DATA_INPUT_DIR = /Users/tliaudat/Documents/PhD/github/cosmostat_official/mccd/tests/hidden_tests/sex_inputs/ -VAL_PREPROCESSED_OUTPUT_DIR = /Users/tliaudat/Documents/PhD/github/cosmostat_official/mccd/tests/hidden_tests/preprocessed_inputs/ -VAL_MODEL_INPUT_DIR = /Users/tliaudat/Documents/PhD/github/cosmostat_official/mccd/tests/hidden_tests/outputs/ -VAL_OUTPUT_DIR = /Users/tliaudat/Documents/PhD/github/cosmostat_official/mccd/tests/hidden_tests/outputs/ +VAL_DATA_INPUT_DIR = . +VAL_PREPROCESSED_OUTPUT_DIR = ./output +VAL_MODEL_INPUT_DIR = ./output +VAL_OUTPUT_DIR = ./output VAL_REGEX_FILE_PATTERN = star_split_ratio_20-*-*.fits VAL_SEPARATOR = - APPLY_DEGRADATION = True MCCD_DEBUG = False -GLOBAL_POL_INTERP = True +GLOBAL_POL_INTERP = False # Parameter description: # @@ -47,10 +51,6 @@ GLOBAL_POL_INTERP = True # [INPUTS] # INPUT_DIR : (Required) Must be a valid directory containing the input # MCCD files. -# PREPROCESSED_OUTPUT_DIR : (Required) Must be a valid directory to write the -# preprocessed input files. -# OUTPUT_DIR : (Required) Must be a valid directory to write the output files. -# The constructed models will be saved. # INPUT_REGEX_FILE_PATTERN : File pattern of the input files to use. It should # follow regex (regular expression) standards. # INPUT_SEPARATOR : Separator of the different fields in the filename, @@ -62,6 +62,10 @@ GLOBAL_POL_INTERP = True # USE_SNR_WEIGHTS : Boolean to determine if the SNR weighting strategy will # be used. # For now, it needs the SNR estimations from SExtractor. +# PREPROCESSED_OUTPUT_DIR : (Required) Must be a valid directory to write the +# preprocessed input files. +# OUTPUT_DIR : (Required) Must be a valid directory to write the output files. +# The constructed models will be saved. # # # [INSTANCE] @@ -73,6 +77,31 @@ GLOBAL_POL_INTERP = True # KSIG_GLOB : Denoising parameter of the global model. # ie 1 is a normal denoising, 3 is a hard denoising. # FILTER_PATH : Path for predefined filters. +# D_HYB_LOC : Degree of the polynomial component for the local part in case +# the LOC_MODEL used is 'hybrid'. +# MIN_D_COMP_GLOB : The minimum degree of the polynomial for the global +# component. For example, if the paramter is set to 1, the +# polynomials of degree 0 and 1 will be excluded from the +# global polynomial variations. ``None`` means that we are +# not excluding any degree. +# RMSE_THRESH : Parameter concerning the CCD outlier rejection. Once the PSF +# model is calculated we perform an outlier check on the training +# stars. We divide each star in two parts with a given circle. +# The inner part corresponds to the most of the PSF/star energy +# while the outer part corresponds to the observation background. +# The outer part is used to calculate the noise level and the inner +# part to calculate the model residual +# (star observation - PSF model reconstruction). If the RMSE error +# of the residual divided by the noise level is over the RMSE_THRESH, +# the star will be considered an outlier. A perfect reconstruction +# would have RMSE_THRESH equal to 1. +# CCD_STAR_THRESH : Parameter concerning the CCD outlier rejection. If the +# percentage of outlier stars in a single CCD is bigger than +# CCD_STAR_THRESH, the CCD is considered to be an outlier. +# In this case, the CCD is rejected from the PSF model. +# A value lower than 0 means that no outlier rejection +# will be done. +# # # # [FIT] @@ -97,16 +126,14 @@ GLOBAL_POL_INTERP = True # # # [VALIDATION] +# MODEL_INPUT_DIR : (Required) Must be a valid directory which contains the +# saved trained models. # VAL_DATA_INPUT_DIR : (Required) Must be a valid directory which contains the # validation input data (test dataset). -# VAL_PREPROCESSED_OUTPUT_DIR : (Required) Must be a valid directory where the -# preprocessed input file will be saved. -# VAL_MODEL_INPUT_DIR : (Required) Must be a valid directory which contains the -# saved trained models. -# VAL_OUTPUT_DIR : (Required) Must be a valid directory where to save the -# validation outputs, test PSFs and interpolated PSFs. # VAL_REGEX_FILE_PATTERN : Same as INPUT_REGEX_FILE_PATTERN but for validation. # VAL_SEPARATOR : Same as INPUT_SEPARATOR but for validation. +# VAL_OUTPUT_DIR : (Required) Must be a valid directory where to save the +# validation outputs, test PSFs and interpolated PSFs. # APPLY_DEGRADATION : Whether the PSF models should be degraded # (sampling/shifts/flux) to match stars; use True if you # plan on making pixel-based comparisons (residuals etc.). From 17b1395f1d8c5cc8d7da7568ca832a52c1fcf411 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Mon, 26 Jul 2021 11:54:32 +0200 Subject: [PATCH 03/26] Adding rejection of objects with zero size that are not flagged --- .../modules/MCCD_package/mccd_plot_utilities.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/shapepipe/modules/MCCD_package/mccd_plot_utilities.py b/shapepipe/modules/MCCD_package/mccd_plot_utilities.py index a50920b90..0ffc29a10 100644 --- a/shapepipe/modules/MCCD_package/mccd_plot_utilities.py +++ b/shapepipe/modules/MCCD_package/mccd_plot_utilities.py @@ -146,6 +146,20 @@ def plot_meanshapes(starcat_path, output_path, nb_pixel, w_log, all_X = starcat[2].data['X'] all_Y = starcat[2].data['Y'] + # Remove stars/PSFs where the measured size is zero + # Sometimes the HSM shape measurement gives objects with measured + # size equals to zero without an error Flag. + bad_stars = (abs(all_star_shapes[2, :]) < 0.1) + bad_psfs = (abs(all_psf_shapes[2, :]) < 0.1) + size_mask = np.abs(bad_stars) * np.abs(bad_psfs) + # Remove outlier stars/PSFs + all_star_shapes = all_star_shapes[:, ~size_mask] + all_psf_shapes = all_psf_shapes[:, ~size_mask] + all_CCDs = all_CCDs[~size_mask] + all_X = all_X[~size_mask] + all_Y = all_Y[~size_mask] + flagmask = flagmask[~size_mask] + if remove_outliers: shape_std_max = 5. # Outlier rejection based on the size From e73573ef6ed2eaf87292951e3e0795928bfbd861 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Tue, 27 Jul 2021 09:48:59 +0200 Subject: [PATCH 04/26] Printing MCCD configuration parameters so that the info is saved in logs. --- shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py b/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py index 1dfc343f9..c6377d68b 100644 --- a/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py +++ b/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py @@ -201,6 +201,12 @@ def mccd_fit_pipeline(trainstar_path, file_number_string, mccd_parser, mccd_fit_kw = mccd_parser.get_fit_kw() use_SNR_weight = mccd_parser.get_extra_kw('use_SNR_weight') + # Print the model configuration so that it is saved in log files + print('MCCD configuration parameters:') + print('[INPUTS]\n', 'use_SNR_weight: ', use_SNR_weight) + print('[INSTANCE]\n', mccd_inst_kw) + print('[FIT]\n', mccd_fit_kw) + # Open fits file starcat = fits.open(trainstar_path, memmap=False) From e25bfe19a53aa34cc1ffa6de95cb9a87aaf1409a Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Tue, 27 Jul 2021 10:25:49 +0200 Subject: [PATCH 05/26] Propagate w_log to auxiliary mccd_fit_pipeline function --- shapepipe/modules/mccd_fit_runner.py | 9 +++++++-- shapepipe/modules/mccd_fit_val_runner.py | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/shapepipe/modules/mccd_fit_runner.py b/shapepipe/modules/mccd_fit_runner.py index cdca57948..69b6400e3 100644 --- a/shapepipe/modules/mccd_fit_runner.py +++ b/shapepipe/modules/mccd_fit_runner.py @@ -35,8 +35,13 @@ def mccd_fit_runner(input_file_list, run_dirs, file_number_string, if mccd_mode == 'FIT': aux_mccd.mccd_fit_pipeline( - trainstar_path, file_number_string, mccd_parser, output_dir, - verbose, saving_name) + trainstar_path=trainstar_path, + file_number_string=file_number_string, + mccd_parser=mccd_parser, + output_dir=output_dir, + verbose=verbose, + saving_name=saving_name, + w_log=w_log) else: raise ValueError('''mccd_fit_runner should be called when the diff --git a/shapepipe/modules/mccd_fit_val_runner.py b/shapepipe/modules/mccd_fit_val_runner.py index 0391edea1..8e128afc8 100644 --- a/shapepipe/modules/mccd_fit_val_runner.py +++ b/shapepipe/modules/mccd_fit_val_runner.py @@ -45,7 +45,8 @@ def mccd_fit_val_runner(input_file_list, run_dirs, file_number_string, mccd_parser=mccd_parser, output_dir=output_dir, verbose=verbose, - saving_name=fit_saving_name) + saving_name=fit_saving_name, + w_log=w_log) # Fitted model is found in the output directory mccd_model_path = output_dir + fit_saving_name + file_number_string \ From e2b191ddeee9d54118a6171c86c858f75e1c4936 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Tue, 27 Jul 2021 10:26:16 +0200 Subject: [PATCH 06/26] Print MCCD config files into the log and not the stdout --- .../MCCD_package/shapepipe_auxiliary_mccd.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py b/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py index c6377d68b..d584e04b3 100644 --- a/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py +++ b/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py @@ -15,6 +15,7 @@ from astropy.io import fits import galsim from shapepipe.pipeline import file_io as sc +import pprint NOT_ENOUGH_STARS = 'Not enough stars to train the model.' @@ -194,7 +195,8 @@ def print_fun(x): def mccd_fit_pipeline(trainstar_path, file_number_string, mccd_parser, - output_dir, verbose, saving_name='fitted_model'): + output_dir, verbose, saving_name='fitted_model', + w_log=None): r"""Fit the MCCD model to the observations.""" # Extract the MCCD parameters from the parser mccd_inst_kw = mccd_parser.get_instance_kw() @@ -202,10 +204,17 @@ def mccd_fit_pipeline(trainstar_path, file_number_string, mccd_parser, use_SNR_weight = mccd_parser.get_extra_kw('use_SNR_weight') # Print the model configuration so that it is saved in log files - print('MCCD configuration parameters:') - print('[INPUTS]\n', 'use_SNR_weight: ', use_SNR_weight) - print('[INSTANCE]\n', mccd_inst_kw) - print('[FIT]\n', mccd_fit_kw) + w_log.info('MCCD configuration parameters:') + w_log.info('[INPUTS]') + inputs_dict_str = pprint.pformat({'use_SNR_weight': use_SNR_weight}) + w_log.info(inputs_dict_str) + w_log.info('[INSTANCE]') + inst_dict_str = pprint.pformat(mccd_inst_kw) + w_log.info(inst_dict_str) + w_log.info('[FIT]') + fit_dict_str = pprint.pformat(mccd_fit_kw) + w_log.info(fit_dict_str) + w_log.info('End of MCCD configuration parameters.') # Open fits file starcat = fits.open(trainstar_path, memmap=False) From 93b69c106efec6d3ae31fb1fc1aed297dd0032f1 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 12:05:49 +0200 Subject: [PATCH 07/26] Adding style changes to shapepipe_auxiliary_mccd --- .../MCCD_package/shapepipe_auxiliary_mccd.py | 112 ++++++++++++------ 1 file changed, 77 insertions(+), 35 deletions(-) diff --git a/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py b/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py index d584e04b3..8292f07f9 100644 --- a/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py +++ b/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py @@ -20,18 +20,20 @@ NOT_ENOUGH_STARS = 'Not enough stars to train the model.' -def mccd_preprocessing_pipeline(input_file_list, - output_path, - input_file_position=0, - min_n_stars=20, - separator='-', - CCD_id_filter_list=None, - outlier_std_max=100., - save_masks=True, - save_name='train_star_selection', - save_extension='.fits', - verbose=True, - print_fun=None): +def mccd_preprocessing_pipeline( + input_file_list, + output_path, + input_file_position=0, + min_n_stars=20, + separator='-', + CCD_id_filter_list=None, + outlier_std_max=100., + save_masks=True, + save_name='train_star_selection', + save_extension='.fits', + verbose=True, + print_fun=None +): r"""Preprocess input catalog. Parameters @@ -97,7 +99,9 @@ def print_fun(x): print_fun('Processing dataset..') mccd_inputs = mccd.mccd_utils.MccdInputs(separator=separator) catalog_ids = mccd_inputs.proprocess_pipeline_data( - input_file_list, element_position=input_file_position) + input_file_list, + element_position=input_file_position + ) # Loop over the catalogs for it in range(catalog_ids.shape[0]): @@ -108,8 +112,16 @@ def print_fun(x): star_list, pos_list, mask_list, ccd_list, SNR_list, RA_list, \ DEC_list, _ = mccd_inputs.outlier_rejection( - star_list, pos_list, mask_list, ccd_list, SNR_list, RA_list, - DEC_list, shape_std_max=outlier_std_max, print_fun=print_fun) + star_list, + pos_list, + mask_list, + ccd_list, + SNR_list, + RA_list, + DEC_list, + shape_std_max=outlier_std_max, + print_fun=print_fun + ) mccd_star_list = [] mccd_pos_list = [] @@ -182,7 +194,8 @@ def print_fun(x): 'GLOB_POSITION_IMG_LIST': mccd_poss, 'MASK_LIST': mccd_masks, 'CCD_ID_LIST': mccd_ccds, 'SNR_WIN_LIST': mccd_SNRs, - 'RA_LIST': mccd_RAs, 'DEC_LIST': mccd_DECs} + 'RA_LIST': mccd_RAs, 'DEC_LIST': mccd_DECs + } saving_path = output_path + save_name + separator \ + catalog_id + save_extension @@ -194,9 +207,15 @@ def print_fun(x): return mccd_inputs -def mccd_fit_pipeline(trainstar_path, file_number_string, mccd_parser, - output_dir, verbose, saving_name='fitted_model', - w_log=None): +def mccd_fit_pipeline( + trainstar_path, + file_number_string, + mccd_parser, + output_dir, + verbose, + saving_name='fitted_model', + w_log=None +): r"""Fit the MCCD model to the observations.""" # Extract the MCCD parameters from the parser mccd_inst_kw = mccd_parser.get_instance_kw() @@ -228,14 +247,21 @@ def mccd_fit_pipeline(trainstar_path, file_number_string, mccd_parser, sex_thresh=-1e5, use_SNR_weight=use_SNR_weight, verbose=verbose, - saving_name=saving_name) + saving_name=saving_name + ) starcat.close() -def mccd_validation_pipeline(teststar_path, mccd_model_path, mccd_parser, - output_dir, file_number_string, w_log, - val_saving_name): +def mccd_validation_pipeline( + teststar_path, + mccd_model_path, + mccd_parser, + output_dir, + file_number_string, + w_log, + val_saving_name +): r"""Validate the MCCD trained model against a set of observations.""" w_log.info('Validating catalog %s..' % file_number_string) @@ -251,7 +277,8 @@ def mccd_validation_pipeline(teststar_path, mccd_model_path, mccd_parser, mccd_model_path=mccd_model_path, testcat=testcat[1], **mccd_val_kw, - sex_thresh=-1e5) + sex_thresh=-1e5 + ) testcat.close() @@ -268,8 +295,14 @@ def mccd_validation_pipeline(teststar_path, mccd_model_path, mccd_parser, found.''' % file_number_string) -def mccd_interpolation_pipeline(mccd_model_path, galcat_path, pos_params, - ccd_id, saving_path, get_shapes): +def mccd_interpolation_pipeline( + mccd_model_path, + galcat_path, + pos_params, + ccd_id, + saving_path, + get_shapes +): r"""Interpolate MCCD model.""" # Import MCCD model mccd_model = mccd.mccd_quickload(mccd_model_path) @@ -295,21 +328,29 @@ def mccd_interpolation_pipeline(mccd_model_path, galcat_path, pos_params, moms.observed_shape.g2, moms.moments_sigma, int(bool(moms.error_message))] - for moms in PSF_moms]) + for moms in PSF_moms] + ) - shapepipe_write_output(saving_path=saving_path, - example_fits_path=galcat_path, - get_shapes=get_shapes, - interp_PSFs=interp_PSFs, - PSF_shapes=PSF_shapes) + shapepipe_write_output( + saving_path=saving_path, + example_fits_path=galcat_path, + get_shapes=get_shapes, + interp_PSFs=interp_PSFs, + PSF_shapes=PSF_shapes + ) return None else: return NOT_ENOUGH_STARS -def shapepipe_write_output(saving_path, example_fits_path, get_shapes, - interp_PSFs, PSF_shapes=None): +def shapepipe_write_output( + saving_path, + example_fits_path, + get_shapes, + interp_PSFs, + PSF_shapes=None +): r"""Save interpolated PSFs dictionary to fits file. The saved files are compatible with the previous shapepipe's standard. @@ -323,7 +364,8 @@ def shapepipe_write_output(saving_path, example_fits_path, get_shapes, 'E1_PSF_HSM': PSF_shapes[:, 0], 'E2_PSF_HSM': PSF_shapes[:, 1], 'SIGMA_PSF_HSM': PSF_shapes[:, 2], - 'FLAG_PSF_HSM': PSF_shapes[:, 3].astype(int)} + 'FLAG_PSF_HSM': PSF_shapes[:, 3].astype(int) + } else: data = {'VIGNET': interp_PSFs} From 8fc77c307c461a5bfe237de6c5f11116e1cc08c4 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 12:33:04 +0200 Subject: [PATCH 08/26] Style changes to mccd_plot_utilities Not perfect as there is some rho-stats borrowed code. At least is closer to the style guidelines --- .../MCCD_package/mccd_plot_utilities.py | 438 +++++++++++++----- 1 file changed, 317 insertions(+), 121 deletions(-) diff --git a/shapepipe/modules/MCCD_package/mccd_plot_utilities.py b/shapepipe/modules/MCCD_package/mccd_plot_utilities.py index 0ffc29a10..b5903d480 100644 --- a/shapepipe/modules/MCCD_package/mccd_plot_utilities.py +++ b/shapepipe/modules/MCCD_package/mccd_plot_utilities.py @@ -84,8 +84,14 @@ def MegaCamFlip_2(xbins, ybins, ccd_nb, nb_pixel): return xbins, ybins -def MeanShapesPlot(ccd_maps, filename, title='', colorbar_ampl=1., wind=None, - cmap='bwr'): +def MeanShapesPlot( + ccd_maps, + filename, + title='', + colorbar_ampl=1., + wind=None, + cmap='bwr' +): r"""Plot meanshapes from ccd maps.""" # colorbar amplitude if wind is None: @@ -102,8 +108,13 @@ def MeanShapesPlot(ccd_maps, filename, title='', colorbar_ampl=1., wind=None, axes.flat[j].axis('off') for ccd_nb, ccd_map in enumerate(ccd_maps): ax = axes.flat[MegaCamPos(ccd_nb)] - im = ax.imshow(ccd_map.T, cmap=cmap, interpolation='Nearest', - vmin=vmin, vmax=vmax) + im = ax.imshow( + ccd_map.T, + cmap=cmap, + interpolation='Nearest', + vmin=vmin, + vmax=vmax + ) ax.set_xticks([]) ax.set_yticks([]) ax.set_title('rmse=%.3e' % (np.sqrt(np.nanmean(ccd_map ** 2))), size=8) @@ -115,9 +126,15 @@ def MeanShapesPlot(ccd_maps, filename, title='', colorbar_ampl=1., wind=None, plt.close() -def plot_meanshapes(starcat_path, output_path, nb_pixel, w_log, - remove_outliers=False, plot_meanshapes=True, - plot_histograms=True): +def plot_meanshapes( + starcat_path, + output_path, + nb_pixel, + w_log, + remove_outliers=False, + plot_meanshapes=True, + plot_histograms=True +): r"""Plot meanshapes and histograms.""" # READ FULL STARCAT starcat = fits.open(starcat_path, memmap=False) @@ -166,7 +183,6 @@ def plot_meanshapes(starcat_path, output_path, nb_pixel, w_log, R2_thresh = shape_std_max * np.std(all_star_shapes[2, :]) + np.mean( all_star_shapes[2, :]) bad_stars = (abs(all_star_shapes[2, :]) > R2_thresh) - bad_stars_idx = np.nonzero(bad_stars)[0] w_log.info('Nb of outlier stars: %d' % (np.sum(bad_stars))) # Remove outlier PSFs all_star_shapes = all_star_shapes[:, ~bad_stars] @@ -224,55 +240,80 @@ def plot_meanshapes(starcat_path, output_path, nb_pixel, w_log, np.abs(np.nanmin(ccd_maps[:, :, 0]))) vmin = -vmax wind = [vmin, vmax] + title = "e_1 (stars), std=%.5e\nvmax=%.4e" % ( + np.nanstd(ccd_maps[:, 0, 0]), + np.nanmax(abs(ccd_maps[:, 0, 0])) + ) MeanShapesPlot( - ccd_maps[:, 0, 0], output_path + 'e1s', - "e_1 (stars), std=%.5e\nvmax=%.4e" % ( - np.nanstd(ccd_maps[:, 0, 0]), - np.nanmax(abs(ccd_maps[:, 0, 0]))), - wind=wind) - + ccd_maps[:, 0, 0], + output_path + 'e1s', + title, + wind=wind + ) + + title = "e_1 (model), std=%.5e\nvmax=%.4e" % ( + np.nanstd(ccd_maps[:, 1, 0]), + np.nanmax(abs(ccd_maps[:, 1, 0])) + ) MeanShapesPlot( - ccd_maps[:, 1, 0], output_path + 'e1m', - 'e_1 (model), std=%.5e\nvmax=%.4e' % - (np.nanstd(ccd_maps[:, 1, 0]), - np.nanmax(abs(ccd_maps[:, 1, 0]))), - wind=wind) + ccd_maps[:, 1, 0], + output_path + 'e1m', + title, + wind=wind + ) if auto_colorbar: wind = None e1_res = ccd_maps[:, 0, 0] - ccd_maps[:, 1, 0] e1_res = e1_res[~np.isnan(e1_res)] rmse_e1 = np.sqrt(np.mean(e1_res ** 2)) - w_log.info('Bins: e1 residual RMSE: %.6f\n' % rmse_e1) + w_log.info("Bins: e1 residual RMSE: %.6f\n" % rmse_e1) vmax = np.nanmax(abs(ccd_maps[:, 0, 0] - ccd_maps[:, 1, 0])) vmin = -vmax wind = [vmin, vmax] + title = "e_1 res, rmse=%.5e\nvmax=%.4e , std=%.5e" % ( + rmse_e1, + vmax, + np.nanstd(ccd_maps[:, 0, 0] - ccd_maps[:, 1, 0]) + ) MeanShapesPlot( ccd_maps[:, 0, 0] - ccd_maps[:, 1, 0], output_path + 'e1res', - 'e_1 res, rmse=%.5e\nvmax=%.4e , std=%.5e' % ( - rmse_e1, vmax, - np.nanstd(ccd_maps[:, 0, 0] - ccd_maps[:, 1, 0])), + title, wind=wind, - colorbar_ampl=colorbar_ampl) + colorbar_ampl=colorbar_ampl + ) # e_2 vmax = max(np.nanmax(ccd_maps[:, :, 1]), np.abs(np.nanmin(ccd_maps[:, :, 1]))) vmin = -vmax wind = [vmin, vmax] - MeanShapesPlot(ccd_maps[:, 0, 1], output_path + 'e2s', - 'e_2 (stars), std=%.5e\nvmax=%.4e' % - (np.nanstd(ccd_maps[:, 0, 1]), - np.nanmax(abs(ccd_maps[:, 0, 1]))), wind=wind) - MeanShapesPlot(ccd_maps[:, 1, 1], output_path + 'e2m', - 'e_2 (model), std=%.5e\nvmax=%.4e' % - (np.nanstd(ccd_maps[:, 1, 1]), - np.nanmax(abs(ccd_maps[:, 1, 1]))), wind=wind) + title = "e_2 (stars), std=%.5e\nvmax=%.4e" % ( + np.nanstd(ccd_maps[:, 0, 1]), + np.nanmax(abs(ccd_maps[:, 0, 1])) + ) + MeanShapesPlot( + ccd_maps[:, 0, 1], + output_path + 'e2s', + title, + wind=wind + ) + title = "e_2 (model), std=%.5e\nvmax=%.4e" % ( + np.nanstd(ccd_maps[:, 1, 1]), + np.nanmax(abs(ccd_maps[:, 1, 1])) + ) + MeanShapesPlot( + ccd_maps[:, 1, 1], + output_path + 'e2m', + title, + wind=wind + ) if auto_colorbar: wind = None colorbar_ampl = 1. + e2_res = ccd_maps[:, 0, 1] - ccd_maps[:, 1, 1] e2_res = e2_res[~np.isnan(e2_res)] rmse_e2 = np.sqrt(np.mean(e2_res ** 2)) @@ -280,29 +321,45 @@ def plot_meanshapes(starcat_path, output_path, nb_pixel, w_log, vmax = np.nanmax(abs(ccd_maps[:, 0, 1] - ccd_maps[:, 1, 1])) vmin = -vmax wind = [vmin, vmax] + title = 'e_2 res, rmse=%.5e\nvmax=%.4e , std=%.5e' % ( + rmse_e2, + vmax, + np.nanstd(ccd_maps[:, 0, 1] - ccd_maps[:, 1, 1]) + ) MeanShapesPlot( - ccd_maps[:, 0, 1] - ccd_maps[:, 1, 1], output_path + 'e2res', - 'e_2 res, rmse=%.5e\nvmax=%.4e , std=%.5e' % ( - rmse_e2, vmax, - np.nanstd(ccd_maps[:, 0, 1] - ccd_maps[:, 1, 1])), - wind=wind, colorbar_ampl=colorbar_ampl) + ccd_maps[:, 0, 1] - ccd_maps[:, 1, 1], + output_path + 'e2res', + title, + wind=wind, + colorbar_ampl=colorbar_ampl + ) # R^2 wind = [0, np.nanmax(ccd_maps[:, :, 2])] colorbar_ampl = 1 + title = 'R_2 (stars), std=%.5e\nvmax=%.4e' % ( + np.nanstd(ccd_maps[:, 0, 2]), + np.nanmax(abs(ccd_maps[:, 0, 2])) + ) MeanShapesPlot( - ccd_maps[:, 0, 2], output_path + 'R2s', - 'R_2 (stars), std=%.5e\nvmax=%.4e' % ( - np.nanstd(ccd_maps[:, 0, 2]), - np.nanmax(abs(ccd_maps[:, 0, 2]))), - wind=wind, cmap='Reds') - + ccd_maps[:, 0, 2], + output_path + 'R2s', + title, + wind=wind, + cmap='Reds' + ) + + title = 'R_2 (model), std=%.5e\nvmax=%.4e' % ( + np.nanstd(ccd_maps[:, 1, 2]), + np.nanmax(abs(ccd_maps[:, 1, 2])) + ) MeanShapesPlot( - ccd_maps[:, 1, 2], output_path + 'R2m', - 'R_2 (model), std=%.5e\nvmax=%.4e' % ( - np.nanstd(ccd_maps[:, 1, 2]), - np.nanmax(abs(ccd_maps[:, 1, 2]))), - wind=wind, cmap='Reds') + ccd_maps[:, 1, 2], + output_path + 'R2m', + title, + wind=wind, + cmap='Reds' + ) if auto_colorbar: wind = [0, np.nanmax(np.abs( @@ -316,39 +373,60 @@ def plot_meanshapes(starcat_path, output_path, nb_pixel, w_log, abs((ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / ccd_maps[:, 0, 2])) wind = [0, vmax] if remove_outliers: - plot_title = "Outliers removed\n∆(R_2)/R_2 res, " \ - "rmse=%.5e\nvmax=%.4e , std=%.5e" % ( - rmse_r2, vmax, - np.nanstd( - (ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / - ccd_maps[:, 0, 2])) + title = "Outliers removed\n∆(R_2)/R_2 res, " \ + "rmse=%.5e\nvmax=%.4e , std=%.5e" % ( + rmse_r2, + vmax, + np.nanstd((ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / + ccd_maps[:, 0, 2]) + ) else: - plot_title = "∆(R_2)/R_2 res, rmse=%.5e\nvmax=%.4e , std=%.5e" % ( - rmse_r2, vmax, + title = "∆(R_2)/R_2 res, rmse=%.5e\nvmax=%.4e , std=%.5e" % ( + rmse_r2, + vmax, np.nanstd((ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / - ccd_maps[:, 0, 2])) + ccd_maps[:, 0, 2]) + ) MeanShapesPlot( np.abs((ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / ccd_maps[:, 0, 2]), output_path + 'R2res', - plot_title, wind=wind, colorbar_ampl=colorbar_ampl, cmap='Reds') + title, + wind=wind, + colorbar_ampl=colorbar_ampl, + cmap='Reds' + ) # nstars wind = (0, np.max(ccd_maps[:, 0, 3])) + title = 'Number of stars\nTotal=%d' % (np.nansum(ccd_maps[:, 0, 3])) MeanShapesPlot( - ccd_maps[:, 0, 3], output_path + 'nstar', - 'Number of stars\nTotal=%d' % (np.nansum(ccd_maps[:, 0, 3])), - wind=wind, cmap='magma') + ccd_maps[:, 0, 3], + output_path + 'nstar', + title, + wind=wind, + cmap='magma' + ) # Histograms if plot_histograms: hist_bins = 50 plt.figure(figsize=(12, 6), dpi=300) - plt.hist(all_star_shapes[0, :], bins=hist_bins, range=[-0.2, 0.2], - label='stars', alpha=0.5) - plt.hist(all_psf_shapes[0, :], bins=hist_bins, range=[-0.2, 0.2], - label='PSFs', alpha=0.5) + plt.hist( + all_star_shapes[0, :], + bins=hist_bins, + range=[-0.2, 0.2], + label='stars', + alpha=0.5 + ) + plt.hist( + all_psf_shapes[0, :], + bins=hist_bins, + range=[-0.2, 0.2], + label='PSFs', + alpha=0.5 + ) plt.legend(loc='best', fontsize=16) plt.title('e1', fontsize=24) plt.savefig(output_path + 'e1_hist.png') @@ -356,19 +434,33 @@ def plot_meanshapes(starcat_path, output_path, nb_pixel, w_log, plt.figure(figsize=(12, 6), dpi=300) data_hist = all_star_shapes[0, :] - all_psf_shapes[0, :] - plt.hist(data_hist, bins=hist_bins, - range=[np.min(data_hist), np.max(data_hist)], - label='err(star - psf)', alpha=0.5) + plt.hist( + data_hist, + bins=hist_bins, + range=[np.min(data_hist), np.max(data_hist)], + label='err(star - psf)', + alpha=0.5 + ) plt.legend(loc='best', fontsize=16) plt.title('e1 err', fontsize=24) plt.savefig(output_path + 'err_e1_hist.png') plt.close() plt.figure(figsize=(12, 6), dpi=300) - plt.hist(all_star_shapes[1, :], bins=hist_bins, range=[-0.2, 0.2], - label='stars', alpha=0.5) - plt.hist(all_psf_shapes[1, :], bins=hist_bins, range=[-0.2, 0.2], - label='PSFs', alpha=0.5) + plt.hist( + all_star_shapes[1, :], + bins=hist_bins, + range=[-0.2, 0.2], + label='stars', + alpha=0.5 + ) + plt.hist( + all_psf_shapes[1, :], + bins=hist_bins, + range=[-0.2, 0.2], + label='PSFs', + alpha=0.5 + ) plt.legend(loc='best', fontsize=16) plt.title('e2', fontsize=24) plt.savefig(output_path + 'e2_hist.png') @@ -376,9 +468,13 @@ def plot_meanshapes(starcat_path, output_path, nb_pixel, w_log, plt.figure(figsize=(12, 6), dpi=300) data_hist = all_star_shapes[1, :] - all_psf_shapes[1, :] - plt.hist(data_hist, bins=hist_bins, - range=[np.min(data_hist), np.max(data_hist)], - label='err(star - psf)', alpha=0.5) + plt.hist( + data_hist, + bins=hist_bins, + range=[np.min(data_hist), np.max(data_hist)], + label='err(star - psf)', + alpha=0.5 + ) plt.legend(loc='best', fontsize=16) plt.title('e2 err', fontsize=24) plt.savefig(output_path + 'err_e2_hist.png') @@ -387,10 +483,20 @@ def plot_meanshapes(starcat_path, output_path, nb_pixel, w_log, plt.figure(figsize=(12, 6), dpi=300) mean_R2 = np.mean(all_star_shapes[2, :]) wind = [mean_R2 - 4, mean_R2 + 4] - plt.hist(all_star_shapes[2, :], bins=hist_bins, range=wind, label='stars', - alpha=0.5) - plt.hist(all_psf_shapes[2, :], bins=hist_bins, range=wind, label='PSFs', - alpha=0.5) + plt.hist( + all_star_shapes[2, :], + bins=hist_bins, + range=wind, + label='stars', + alpha=0.5 + ) + plt.hist( + all_psf_shapes[2, :], + bins=hist_bins, + range=wind, + label='PSFs', + alpha=0.5 + ) plt.legend(loc='best', fontsize=16) plt.title('R2', fontsize=24) plt.savefig(output_path + 'R2_hist.png') @@ -399,9 +505,13 @@ def plot_meanshapes(starcat_path, output_path, nb_pixel, w_log, plt.figure(figsize=(12, 6), dpi=300) data_hist = (all_star_shapes[2, :] - all_psf_shapes[2, :]) / all_star_shapes[2, :] - plt.hist(data_hist, bins=hist_bins, - range=[np.min(data_hist), np.max(data_hist)], - label='err(star - psf)/star', alpha=0.5) + plt.hist( + data_hist, + bins=hist_bins, + range=[np.min(data_hist), np.max(data_hist)], + label='err(star - psf)/star', + alpha=0.5 + ) plt.legend(loc='best', fontsize=16) plt.title('R2 err', fontsize=24) plt.savefig(output_path + 'err_R2_hist.png') @@ -413,12 +523,21 @@ def plot_meanshapes(starcat_path, output_path, nb_pixel, w_log, # Rho stats functions - -def NegDash(x_in, y_in, yerr_in, plot_name='', vertical_lines=True, - xlabel='', ylabel='', rho_nb='', - ylim=None, semilogx=False, semilogy=False, - **kwargs): - """ This function is for making plots with vertical errorbars, +def NegDash( + x_in, + y_in, + yerr_in, + plot_name='', + vertical_lines=True, + xlabel='', + ylabel='', + rho_nb='', + ylim=None, + semilogx=False, + semilogy=False, + **kwargs +): + r""" This function is for making plots with vertical errorbars, where negative values are shown in absolute value as dashed lines. The resulting plot can either be saved by specifying a file name as `plot_name', or be kept as a pyplot instance (for instance to combine @@ -501,8 +620,13 @@ class new_BaseCorrelationFunctionSysTest(BaseCorrelationFunctionSysTest): """ - def makeCatalog(self, data, config=None, use_as_k=None, - use_chip_coords=False): + def makeCatalog( + self, + data, + config=None, + use_as_k=None, + use_chip_coords=False + ): if data is None or isinstance(data, treecorr.Catalog): return data catalog_kwargs = {} @@ -557,8 +681,15 @@ class Rho1SysTest(new_BaseCorrelationFunctionSysTest): objects_list = ['star PSF'] required_quantities = [('ra', 'dec', 'g1', 'g2', 'psf_g1', 'psf_g2', 'w')] - def __call__(self, data, data2=None, random=None, random2=None, - config=None, **kwargs): + def __call__( + self, + data, + data2=None, + random=None, + random2=None, + config=None, + **kwargs + ): new_data = data.copy() new_data['g1'] = new_data['g1'] - new_data['psf_g1'] new_data['g2'] = new_data['g2'] - new_data['psf_g2'] @@ -594,8 +725,15 @@ class DESRho2SysTest(new_BaseCorrelationFunctionSysTest): objects_list = ['star PSF'] required_quantities = [('ra', 'dec', 'g1', 'g2', 'psf_g1', 'psf_g2', 'w')] - def __call__(self, data, data2=None, random=None, random2=None, - config=None, **kwargs): + def __call__( + self, + data, + data2=None, + random=None, + random2=None, + config=None, + **kwargs + ): new_data = np.rec.fromarrays([data['ra'], data['dec'], data['g1'], data['g2'], data['w']], names=['ra', 'dec', 'g1', 'g2', 'w']) @@ -639,8 +777,15 @@ class DESRho3SysTest(new_BaseCorrelationFunctionSysTest): required_quantities = [('ra', 'dec', 'sigma', 'g1', 'g2', 'psf_sigma', 'w')] - def __call__(self, data, data2=None, random=None, random2=None, - config=None, **kwargs): + def __call__( + self, + data, + data2=None, + random=None, + random2=None, + config=None, + **kwargs + ): new_data = np.rec.fromarrays( [data['ra'], data['dec'], data['g1'] * (data['sigma'] - data[ @@ -648,7 +793,8 @@ def __call__(self, data, data2=None, random=None, random2=None, data['g2'] * (data['sigma'] - data[ 'psf_sigma']) / data['sigma'], data['w']], - names=['ra', 'dec', 'g1', 'g2', 'w']) + names=['ra', 'dec', 'g1', 'g2', 'w'] + ) if data2 is not None: new_data2 = np.rec.fromarrays( [data2['ra'], data2['dec'], @@ -657,7 +803,8 @@ def __call__(self, data, data2=None, random=None, random2=None, data2['g2'] * (data2['sigma'] - data2['psf_sigma']) / data2['sigma'], data2['w']], - names=['ra', 'dec', 'g1', 'g2', 'w']) + names=['ra', 'dec', 'g1', 'g2', 'w'] + ) else: new_data2 = data2 @@ -669,7 +816,8 @@ def __call__(self, data, data2=None, random=None, random2=None, random['g2'] * (random['sigma'] - random['psf_sigma']) / random['sigma'], random['w']], - names=['ra', 'dec', 'g1', 'g2', 'w']) + names=['ra', 'dec', 'g1', 'g2', 'w'] + ) else: new_random = random @@ -681,12 +829,21 @@ def __call__(self, data, data2=None, random=None, random2=None, random2['g2'] * (random2['sigma'] - random2['psf_sigma']) / random2['sigma'], random2['w']], - names=['ra', 'dec', 'g1', 'g2', 'w']) + names=['ra', 'dec', 'g1', 'g2', 'w'] + ) else: new_random2 = random2 - return self.getCF('gg', new_data, new_data2, new_random, new_random2, - config=config, **kwargs) + + return self.getCF( + 'gg', + new_data, + new_data2, + new_random, + new_random2, + config=config, + **kwargs + ) class DESRho4SysTest(new_BaseCorrelationFunctionSysTest): @@ -700,12 +857,20 @@ class DESRho4SysTest(new_BaseCorrelationFunctionSysTest): required_quantities = [('ra', 'dec', 'g1', 'g2', 'sigma', 'psf_g1', 'psf_g2', 'psf_sigma', 'w')] - def __call__(self, data, data2=None, random=None, random2=None, - config=None, **kwargs): + def __call__( + self, + data, + data2=None, + random=None, + random2=None, + config=None, + **kwargs + ): new_data = np.rec.fromarrays( [data['ra'], data['dec'], data['g1'] - data['psf_g1'], data['g2'] - data['psf_g2'], data['w']], - names=['ra', 'dec', 'g1', 'g2', 'w']) + names=['ra', 'dec', 'g1', 'g2', 'w'] + ) if data2 is None: data2 = data new_data2 = np.rec.fromarrays( @@ -715,14 +880,16 @@ def __call__(self, data, data2=None, random=None, random2=None, data2['g2'] * (data2['sigma'] - data2['psf_sigma']) / data2[ 'sigma'], data2['w']], - names=['ra', 'dec', 'g1', 'g2', 'w']) + names=['ra', 'dec', 'g1', 'g2', 'w'] + ) if random is not None: new_random = np.rec.fromarrays( [random['ra'], random['dec'], random['g1'] - random['psf_g1'], random['g2'] - random['psf_g2'], random['w']], - names=['ra', 'dec', 'g1', 'g2', 'w']) + names=['ra', 'dec', 'g1', 'g2', 'w'] + ) else: new_random = random if random2 is None: @@ -735,11 +902,19 @@ def __call__(self, data, data2=None, random=None, random2=None, random2['g2'] * (random2['sigma'] - random2['psf_sigma']) / random2['sigma'], random2['w']], - names=['ra', 'dec', 'g1', 'g2', 'w']) + names=['ra', 'dec', 'g1', 'g2', 'w'] + ) else: new_random2 = random2 - return self.getCF('gg', new_data, new_data2, new_random, new_random2, - config=config, **kwargs) + return self.getCF( + 'gg', + new_data, + new_data2, + new_random, + new_random2, + config=config, + **kwargs + ) class DESRho5SysTest(new_BaseCorrelationFunctionSysTest): @@ -755,8 +930,15 @@ class DESRho5SysTest(new_BaseCorrelationFunctionSysTest): required_quantities = [('ra', 'dec', 'sigma', 'g1', 'g2', 'psf_sigma', 'w')] - def __call__(self, data, data2=None, random=None, random2=None, - config=None, **kwargs): + def __call__( + self, + data, + data2=None, + random=None, + random2=None, + config=None, + **kwargs + ): new_data = np.rec.fromarrays([data['ra'], data['dec'], data['g1'], data['g2'], data['w']], names=['ra', 'dec', 'g1', 'g2', 'w']) @@ -786,16 +968,28 @@ def __call__(self, data, data2=None, random=None, random2=None, random2['g2'] * (random2['sigma'] - random2['psf_sigma']) / random2['sigma'], random2['w']], - names=['ra', 'dec', 'g1', 'g2', 'w']) + names=['ra', 'dec', 'g1', 'g2', 'w'] + ) else: new_random2 = random2 - return self.getCF('gg', new_data, new_data2, new_random, new_random2, - config=config, **kwargs) - - -def rho_stats(starcat_path, output_path, rho_def='HSC', - print_fun=lambda x: print(x)): + return self.getCF( + 'gg', + new_data, + new_data2, + new_random, + new_random2, + config=config, + **kwargs + ) + + +def rho_stats( + starcat_path, + output_path, + rho_def='HSC', + print_fun=lambda x: print(x) +): """Compute and plot the five rho statistics. Syntax: @@ -829,12 +1023,14 @@ def rho_stats(starcat_path, output_path, rho_def='HSC', starcat[hdu_no].data['E2_PSF_HSM'], starcat[hdu_no].data['SIGMA_PSF_HSM'] ** 2], names=['w', 'ra', 'dec', 'g1', 'g2', 'sigma', 'psf_g1', 'psf_g2', - 'psf_sigma']) + 'psf_sigma'] + ) # TreeCorr config: TreeCorrConfig = {'ra_units': 'degrees', 'dec_units': 'degrees', 'max_sep': '3e2', 'min_sep': 5e-1, 'sep_units': 'arcmin', - 'nbins': 32} + 'nbins': 32 + } # Ininitialize all 5 rho stats if rho_def == 'HSC': From 86ca3ec88e78cc5986dff896eb5e7ed123406ba4 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 12:54:01 +0200 Subject: [PATCH 09/26] Adding style changes to mccd_interpolation_script --- .../MCCD_package/mccd_interpolation_script.py | 179 +++++++++++------- 1 file changed, 113 insertions(+), 66 deletions(-) diff --git a/shapepipe/modules/MCCD_package/mccd_interpolation_script.py b/shapepipe/modules/MCCD_package/mccd_interpolation_script.py index 2c3e2816f..7a510071c 100644 --- a/shapepipe/modules/MCCD_package/mccd_interpolation_script.py +++ b/shapepipe/modules/MCCD_package/mccd_interpolation_script.py @@ -59,8 +59,10 @@ def interp_MCCD(mccd_model_path, positions, ccd): # Positions to global coordinates loc2glob = mccd.mccd_utils.Loc2Glob() - glob_pos = np.array([loc2glob.loc2glob_img_coord(_ccd, _pos[0], _pos[1]) - for _ccd, _pos in zip(ccd_list, positions)]) + glob_pos = np.array( + [loc2glob.loc2glob_img_coord(_ccd, _pos[0], _pos[1]) + for _ccd, _pos in zip(ccd_list, positions)] + ) # Interpolate the model PSFs = mccd_instance.estimate_psf(glob_pos, ccd) @@ -102,8 +104,16 @@ class MCCDinterpolator(object): """ - def __init__(self, dotpsf_path, galcat_path, output_path, img_number, - w_log, pos_params=None, get_shapes=True): + def __init__( + self, + dotpsf_path, + galcat_path, + output_path, + img_number, + w_log, + pos_params=None, + get_shapes=True + ): # Path to PSFEx output file self._dotpsf_path = dotpsf_path # Path to catalog containing galaxy positions @@ -127,9 +137,10 @@ def __init__(self, dotpsf_path, galcat_path, output_path, img_number, # CosmoStat's ShapePipe) if pos_params: if not len(pos_params) == 2: - raise ValueError('{} position parameters were passed on; ' - 'there should be exactly two.' - ''.format(len(pos_params))) + raise ValueError( + f'{len(pos_params)} position parameters were passed on;' + + f' there should be exactly two.' + ) self._pos_params = pos_params else: self._pos_params = None @@ -143,8 +154,10 @@ def _get_position_parameters(self): """ dotpsf = sc.FITSCatalog(self._dotpsf_path) dotpsf.open() - self._pos_params = [dotpsf.get_header()['POLNAME1'], - dotpsf.get_header()['POLNAME2']] + self._pos_params = [ + dotpsf.get_header()['POLNAME1'], + dotpsf.get_header()['POLNAME2'] + ] dotpsf.close() def _get_galaxy_positions(self): @@ -159,15 +172,19 @@ def _get_galaxy_positions(self): self.gal_pos = np.array( [[x, y] for x, y in zip( galcat.get_data()[self._pos_params[0]], - galcat.get_data()[self._pos_params[1]])]) + galcat.get_data()[self._pos_params[1]] + )] + ) except KeyError as detail: # extract erroneous position parameter from original exception err_pos_param = detail.args[0][4:-15] - pos_param_err = ('Required position parameter ' + err_pos_param + - 'was not found in galaxy catalog. Leave ' - 'pos_params (or EXTRA_CODE_OPTION) blank to ' - 'read them from .psf file.') + pos_param_err = ( + 'Required position parameter ' + err_pos_param + + 'was not found in galaxy catalog. Leave ' + 'pos_params (or EXTRA_CODE_OPTION) blank to ' + 'read them from .psf file.' + ) raise KeyError(pos_param_err) galcat.close() @@ -177,28 +194,36 @@ def _get_psfshapes(self): if import_fail: raise ImportError('Galsim is required to get shapes information') - psf_moms = [hsm.FindAdaptiveMom(Image(psf), strict=False) - for psf in self.interp_PSFs] + psf_moms = [ + hsm.FindAdaptiveMom(Image(psf), strict=False) + for psf in self.interp_PSFs + ] - self.psf_shapes = np.array([[moms.observed_shape.g1, - moms.observed_shape.g2, - moms.moments_sigma, - int(bool(moms.error_message))] for moms in - psf_moms]) + self.psf_shapes = np.array( + [[moms.observed_shape.g1, + moms.observed_shape.g2, + moms.moments_sigma, + int(bool(moms.error_message))] + for moms in psf_moms] + ) def _write_output(self): """ Save computed PSFs to fits file. """ - output = sc.FITSCatalog(self._output_path + self._img_number + '.fits', - open_mode=sc.BaseCatalog.OpenMode.ReadWrite, - SEx_catalog=True) + output = sc.FITSCatalog( + self._output_path + self._img_number + '.fits', + open_mode=sc.BaseCatalog.OpenMode.ReadWrite, + SEx_catalog=True + ) if self._compute_shape: - data = {'VIGNET': self.interp_PSFs, - 'E1_PSF_HSM': self.psf_shapes[:, 0], - 'E2_PSF_HSM': self.psf_shapes[:, 1], - 'SIGMA_PSF_HSM': self.psf_shapes[:, 2], - 'FLAG_PSF_HSM': self.psf_shapes[:, 3].astype(int)} + data = { + 'VIGNET': self.interp_PSFs, + 'E1_PSF_HSM': self.psf_shapes[:, 0], + 'E2_PSF_HSM': self.psf_shapes[:, 1], + 'SIGMA_PSF_HSM': self.psf_shapes[:, 2], + 'FLAG_PSF_HSM': self.psf_shapes[:, 3].astype(int) + } else: data = {'VIGNET': self.interp_PSFs} output.save_as_fits(data, sex_cat_path=self._galcat_path) @@ -216,15 +241,20 @@ def _get_starshapes(self, star_vign): masks = np.zeros_like(star_vign) masks[np.where(star_vign == -1e30)] = 1 - star_moms = [ - hsm.FindAdaptiveMom(Image(star), badpix=Image(mask), strict=False) - for star, mask in zip(star_vign, masks)] - - self.star_shapes = np.array([[moms.observed_shape.g1, - moms.observed_shape.g2, - moms.moments_sigma, - int(bool(moms.error_message))] for moms - in star_moms]) + star_moms = [hsm.FindAdaptiveMom( + Image(star), + badpix=Image(mask), + strict=False) + for star, mask in zip(star_vign, masks) + ] + + self.star_shapes = np.array( + [[moms.observed_shape.g1, + moms.observed_shape.g2, + moms.moments_sigma, + int(bool(moms.error_message))] + for moms in star_moms] + ) def _get_psfexcatdict(self, psfex_cat_path): """ Get data from PSFEx .cat file. @@ -266,14 +296,16 @@ def _write_output_validation(self, star_dict, psfex_cat_dict): open_mode=sc.BaseCatalog.OpenMode.ReadWrite, SEx_catalog=True) - data = {'E1_PSF_HSM': self.psf_shapes[:, 0], - 'E2_PSF_HSM': self.psf_shapes[:, 1], - 'SIGMA_PSF_HSM': self.psf_shapes[:, 2], - 'FLAG_PSF_HSM': self.psf_shapes[:, 3].astype(int), - 'E1_STAR_HSM': self.star_shapes[:, 0], - 'E2_STAR_HSM': self.star_shapes[:, 1], - 'SIGMA_STAR_HSM': self.star_shapes[:, 2], - 'FLAG_STAR_HSM': self.star_shapes[:, 3].astype(int)} + data = { + 'E1_PSF_HSM': self.psf_shapes[:, 0], + 'E2_PSF_HSM': self.psf_shapes[:, 1], + 'SIGMA_PSF_HSM': self.psf_shapes[:, 2], + 'FLAG_PSF_HSM': self.psf_shapes[:, 3].astype(int), + 'E1_STAR_HSM': self.star_shapes[:, 0], + 'E2_STAR_HSM': self.star_shapes[:, 1], + 'SIGMA_STAR_HSM': self.star_shapes[:, 2], + 'FLAG_STAR_HSM': self.star_shapes[:, 3].astype(int) + } data = {**data, **star_dict} data['ACCEPTED'] = np.ones_like(data['NUMBER'], dtype='int16') @@ -345,38 +377,53 @@ def _interpolate_me(self): # self._dot_psf_pattern + '-' + exp_name + '-' + str(ccd) +\ # '.psf' mccd_model_path = self._dot_psf_dir + '/' +\ - self._dot_psf_pattern + '-' + exp_name +\ - '.npy' + self._dot_psf_pattern + '-' + exp_name + '.npy' + ind_obj = np.where(cat.get_data(hdu_index)['CCD_N'] == ccd)[0] obj_id = all_id[ind_obj] gal_pos = np.array( self._f_wcs_file[exp_name][ccd]['WCS'].all_world2pix( self.gal_pos[:, 0][ind_obj], - self.gal_pos[:, 1][ind_obj], 0)).T + self.gal_pos[:, 1][ind_obj], + 0 + ) + ).T self.interp_PSFs = interp_MCCD(mccd_model_path, gal_pos, ccd) # self.interp_PSFs = interpsfex( # dot_psf_path, gal_pos, self._star_thresh, self._chi2_thresh) - if isinstance(self.interp_PSFs, - str) and self.interp_PSFs == NOT_ENOUGH_STARS: - self._w_log.info('Not enough stars find in the ccd' - ' {} of the exposure {}. Object inside' - ' this ccd will lose an' - 'epoch.'.format(ccd, exp_name)) + if ( + isinstance(self.interp_PSFs, str) + and (self.interp_PSFs == NOT_ENOUGH_STARS) + ): + self._w_log.info( + f'Not enough stars find in the ccd {ccd} of the' + + f' exposure {exp_name}. Object inside this ' + + f'ccd will lose an epoch.' + ) continue - if isinstance(self.interp_PSFs, - str) and self.interp_PSFs == BAD_CHI2: - self._w_log.info('Bad chi2 for the psf model in the ccd' - ' {} of the exposure {}. Object inside' - ' this ccd will lose an' - 'epoch.'.format(ccd, exp_name)) + + if ( + isinstance(self.interp_PSFs, str) + and (self.interp_PSFs == BAD_CHI2) + ): + self._w_log.info( + f'Bad chi2 for the psf model in the ccd {ccd} of the' + + f' exposure {exp_name}. Object inside this ccd' + + f' will lose an epoch.' + ) + continue - if isinstance(self.interp_PSFs, - str) and self.interp_PSFs == FILE_NOT_FOUND: - self._w_log.info('Psf model file {} not found. ' - 'Object inside this ccd will lose' - 'an epoch'.format(self._dotpsf_path)) + + if ( + isinstance(self.interp_PSFs, str) + and (self.interp_PSFs == FILE_NOT_FOUND) + ): + self._w_log.info( + f'Psf model file {self._dotpsf_path} not found.' + + f' Object inside this ccd will lose an epoch.' + ) continue if array_psf is None: From bc4776ff05adc7a8aa9b81d24148fb9626ac4441 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 12:59:10 +0200 Subject: [PATCH 10/26] Adding style changes to mccd_fit_runner --- shapepipe/modules/mccd_fit_runner.py | 30 +++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/shapepipe/modules/mccd_fit_runner.py b/shapepipe/modules/mccd_fit_runner.py index 69b6400e3..4c5e18d76 100644 --- a/shapepipe/modules/mccd_fit_runner.py +++ b/shapepipe/modules/mccd_fit_runner.py @@ -13,12 +13,22 @@ import mccd -@module_runner(input_module=['mccd_preprocessing_runner'], version='1.0', - file_pattern=['train_star_selection'], - file_ext=['.fits'], numbering_scheme='-0000000', - depends=['numpy', 'mccd', 'galsim'], run_method='parallel') -def mccd_fit_runner(input_file_list, run_dirs, file_number_string, - config, w_log): +@module_runner( + input_module=['mccd_preprocessing_runner'], + version='1.0', + file_pattern=['train_star_selection'], + file_ext=['.fits'], + numbering_scheme='-0000000', + depends=['numpy', 'mccd', 'galsim'], + run_method='parallel' +) +def mccd_fit_runner( + input_file_list, + run_dirs, + file_number_string, + config, + w_log +): # Recover the MCCD config file and its params config_file_path = config.getexpanded('MCCD', 'CONFIG_PATH') mccd_mode = config.get('MCCD', 'MODE') @@ -41,10 +51,12 @@ def mccd_fit_runner(input_file_list, run_dirs, file_number_string, output_dir=output_dir, verbose=verbose, saving_name=saving_name, - w_log=w_log) + w_log=w_log + ) else: - raise ValueError('''mccd_fit_runner should be called when the - MODE is "FIT".''') + raise ValueError( + "mccd_fit_runner should be called when the MODE is 'FIT'." + ) return None, None From d173e882072aa5270dfc1cd15d0955458ebe3334 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 13:01:05 +0200 Subject: [PATCH 11/26] Adding style change to mccd_fit_val_runner --- shapepipe/modules/mccd_fit_val_runner.py | 34 +++++++++++++++++------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/shapepipe/modules/mccd_fit_val_runner.py b/shapepipe/modules/mccd_fit_val_runner.py index 8e128afc8..f7b1466f7 100644 --- a/shapepipe/modules/mccd_fit_val_runner.py +++ b/shapepipe/modules/mccd_fit_val_runner.py @@ -13,12 +13,22 @@ import mccd -@module_runner(input_module=['mccd_preprocessing_runner'], version='1.0', - file_pattern=['train_star_selection', 'test_star_selection'], - file_ext=['.fits', '.fits'], numbering_scheme='-0000000', - depends=['numpy', 'mccd', 'galsim'], run_method='parallel') -def mccd_fit_val_runner(input_file_list, run_dirs, file_number_string, - config, w_log): +@module_runner( + input_module=['mccd_preprocessing_runner'], + version='1.0', + file_pattern=['train_star_selection', 'test_star_selection'], + file_ext=['.fits', '.fits'], + numbering_scheme='-0000000', + depends=['numpy', 'mccd', 'galsim'], + run_method='parallel' +) +def mccd_fit_val_runner( + input_file_list, + run_dirs, + file_number_string, + config, + w_log +): # Recover the MCCD config file and its params config_file_path = config.getexpanded('MCCD', 'CONFIG_PATH') mccd_mode = config.get('MCCD', 'MODE') @@ -46,7 +56,8 @@ def mccd_fit_val_runner(input_file_list, run_dirs, file_number_string, output_dir=output_dir, verbose=verbose, saving_name=fit_saving_name, - w_log=w_log) + w_log=w_log + ) # Fitted model is found in the output directory mccd_model_path = output_dir + fit_saving_name + file_number_string \ @@ -59,10 +70,13 @@ def mccd_fit_val_runner(input_file_list, run_dirs, file_number_string, output_dir=output_dir, file_number_string=file_number_string, w_log=w_log, - val_saving_name=val_saving_name) + val_saving_name=val_saving_name + ) else: - raise ValueError('''mccd_fit_val_runner should be called when the - MODE is "FIT_VALIDATION".''') + raise ValueError( + "mccd_fit_val_runner should be called when the MODE" + + " is 'FIT_VALIDATION'." + ) return None, None From faac965814a03f8e0a35d11e8711d1ea167d2457 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 13:05:47 +0200 Subject: [PATCH 12/26] Adding style changes to mccd_interp_runner --- shapepipe/modules/mccd_interp_runner.py | 98 ++++++++++++++++--------- 1 file changed, 63 insertions(+), 35 deletions(-) diff --git a/shapepipe/modules/mccd_interp_runner.py b/shapepipe/modules/mccd_interp_runner.py index 6b5d7ad96..c6157dea1 100644 --- a/shapepipe/modules/mccd_interp_runner.py +++ b/shapepipe/modules/mccd_interp_runner.py @@ -15,14 +15,21 @@ import os -@module_runner(input_module=['setools_runner'], - version='1.0', - file_pattern=['galaxy_selection'], - file_ext=['.npy'], - depends=['numpy', 'astropy', 'galsim', 'sqlitedict', 'mccd'], - run_method='parallel') -def mccd_interp_runner(input_file_list, run_dirs, file_number_string, - config, w_log): +@module_runner( + input_module=['setools_runner'], + version='1.0', + file_pattern=['galaxy_selection'], + file_ext=['.npy'], + depends=['numpy', 'astropy', 'galsim', 'sqlitedict', 'mccd'], + run_method='parallel' +) +def mccd_interp_runner( + input_file_list, + run_dirs, + file_number_string, + config, + w_log +): mode = config.getexpanded('MCCD_INTERP_RUNNER', 'MODE') pos_params = config.getlist('MCCD_INTERP_RUNNER', 'POSITION_PARAMS') @@ -31,12 +38,18 @@ def mccd_interp_runner(input_file_list, run_dirs, file_number_string, output_dir = run_dirs['output'] if mode == 'CLASSIC': - psf_model_dir = config.getexpanded('MCCD_INTERP_RUNNER', - 'PSF_MODEL_DIR') - psf_model_pattern = config.get('MCCD_INTERP_RUNNER', - 'PSF_MODEL_PATTERN') - psf_separator = config.get('MCCD_INTERP_RUNNER', - 'PSF_MODEL_SEPARATOR') + psf_model_dir = config.getexpanded( + 'MCCD_INTERP_RUNNER', + 'PSF_MODEL_DIR' + ) + psf_model_pattern = config.get( + 'MCCD_INTERP_RUNNER', + 'PSF_MODEL_PATTERN' + ) + psf_separator = config.get( + 'MCCD_INTERP_RUNNER', + 'PSF_MODEL_SEPARATOR' + ) # psfcat_path, galcat_path = input_file_list galcat_path = input_file_list[0] @@ -52,8 +65,10 @@ def mccd_interp_runner(input_file_list, run_dirs, file_number_string, if not os.path.exists(psf_model_path): error_msg = "The corresponding PSF model was not not found." - w_log.info("Error message: {}.\n On catalog with id: {}.".format( - error_msg, file_number_string)) + w_log.info( + f"Error message: {error_msg}.\n On catalog with" + + f" id: {file_number_string}." + ) return None, None @@ -64,37 +79,50 @@ def mccd_interp_runner(input_file_list, run_dirs, file_number_string, pos_params=pos_params, ccd_id=int(ccd_id), saving_path=saving_path, - get_shapes=get_shapes) + get_shapes=get_shapes + ) if output_msg is not None: - w_log.info("Error message: {}.\n On catalog with id: {}.".format( - output_msg, file_number_string)) + w_log.info( + f"Error message: {output_msg}.\n On catalog with" + + f" id: {file_number_string}." + ) elif mode == 'MULTI-EPOCH': - psf_model_dir = config.getexpanded('MCCD_INTERP_RUNNER', - 'PSF_MODEL_DIR') - psf_model_pattern = config.get('MCCD_INTERP_RUNNER', - 'PSF_MODEL_PATTERN') - f_wcs_path = config.getexpanded('MCCD_INTERP_RUNNER', - 'ME_LOG_WCS') + psf_model_dir = config.getexpanded( + 'MCCD_INTERP_RUNNER', + 'PSF_MODEL_DIR' + ) + psf_model_pattern = config.get( + 'MCCD_INTERP_RUNNER', + 'PSF_MODEL_PATTERN' + ) + f_wcs_path = config.getexpanded( + 'MCCD_INTERP_RUNNER', + 'ME_LOG_WCS' + ) galcat_path = input_file_list[0] - inst = mccd_interp.MCCDinterpolator(None, - galcat_path, - output_dir, - file_number_string, - w_log, - pos_params, - get_shapes) + inst = mccd_interp.MCCDinterpolator( + None, + galcat_path, + output_dir, + file_number_string, + w_log, + pos_params, + get_shapes + ) inst.process_me(psf_model_dir, psf_model_pattern, f_wcs_path) elif mode == 'VALIDATION': - ValueError('''MODE has to be in MULTI-EPOCH or CLASSIC. - For validation use MCCD validation runner''') + ValueError( + "MODE has to be in MULTI-EPOCH or CLASSIC. For validation" + + " use MCCD validation runner." + ) else: - ValueError('MODE has to be in : [CLASSIC, MULTI-EPOCH]') + ValueError("MODE has to be in : [CLASSIC, MULTI-EPOCH]") return None, None From 18e97862daea99225788c437c190a294419412e6 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 13:14:02 +0200 Subject: [PATCH 13/26] Adding style changes to mccd_merge_starcat_runner --- .../modules/mccd_merge_starcat_runner.py | 192 +++++++++++------- 1 file changed, 114 insertions(+), 78 deletions(-) diff --git a/shapepipe/modules/mccd_merge_starcat_runner.py b/shapepipe/modules/mccd_merge_starcat_runner.py index 36318e3ff..469da74ec 100644 --- a/shapepipe/modules/mccd_merge_starcat_runner.py +++ b/shapepipe/modules/mccd_merge_starcat_runner.py @@ -15,13 +15,22 @@ from shapepipe.modules.module_decorator import module_runner -@module_runner(input_module=['mccd_fit_val_runner'], version='1.0', - file_pattern=['validation_psf'], - file_ext=['.fits'], numbering_scheme='-0000000', - depends=['numpy', 'astropy'], - run_method='serial') -def mccd_merge_starcat_runner(input_file_list, run_dirs, file_number_string, - config, w_log): +@module_runner( + input_module=['mccd_fit_val_runner'], + version='1.0', + file_pattern=['validation_psf'], + file_ext=['.fits'], + numbering_scheme='-0000000', + depends=['numpy', 'astropy'], + run_method='serial' +) +def mccd_merge_starcat_runner( + input_file_list, + run_dirs, + file_number_string, + config, + w_log +): w_log.info('Merging validation results..') hdu_table = 1 @@ -69,16 +78,19 @@ def mccd_merge_starcat_runner(input_file_list, run_dirs, file_number_string, # Pixel mse calculation pix_val = np.sum((stars - psfs) ** 2) pix_sum = np.sum((stars - psfs)) - masked_diffs = np.array([(_star - _psf)[my_mask] for _star, _psf in - zip(stars, psfs)]) + masked_diffs = np.array( + [(_star - _psf)[my_mask] for _star, _psf in zip(stars, psfs)] + ) masked_pix_val = np.sum(masked_diffs ** 2) masked_pix_sum = np.sum(masked_diffs) # Star noise variance (using masked stars) star_noise_var_val = np.array( - [np.var(_star[~my_mask]) for _star in stars]) - res_var_val = np.array([np.var(_star - _psf) for _star, _psf in - zip(stars, psfs)]) + [np.var(_star[~my_mask]) for _star in stars] + ) + res_var_val = np.array( + [np.var(_star - _psf) for _star, _psf in zip(stars, psfs)] + ) # Variance of the model # (Residual variance - Star variance (using masked stars)) @@ -90,16 +102,21 @@ def mccd_merge_starcat_runner(input_file_list, run_dirs, file_number_string, stars_norm_vals = np.sqrt(np.sum(stars ** 2, axis=(1, 2))) psfs_norm_vals = np.sqrt(np.sum(psfs ** 2, axis=(1, 2))) # Select non zero stars & psfs - non_zero_elems = np.logical_and((psfs_norm_vals != 0), - (stars_norm_vals != 0)) + non_zero_elems = np.logical_and( + (psfs_norm_vals != 0), + (stars_norm_vals != 0) + ) # Calculate the filtered mse calculation - pix_filt_val = np.sum((stars[non_zero_elems] - - psfs[non_zero_elems]) ** 2) + pix_filt_val = np.sum( + (stars[non_zero_elems] - psfs[non_zero_elems]) ** 2 + ) # Calculate the normalized (& filtered) mse calculation stars_norm_vals = stars_norm_vals[non_zero_elems].reshape(-1, 1, 1) psfs_norm_vals = psfs_norm_vals[non_zero_elems].reshape(-1, 1, 1) - pix_norm_val = np.sum((stars[non_zero_elems] / stars_norm_vals - - psfs[non_zero_elems] / psfs_norm_vals) ** 2) + pix_norm_val = np.sum( + (stars[non_zero_elems] / stars_norm_vals - + psfs[non_zero_elems] / psfs_norm_vals) ** 2 + ) # Calculate sizes filt_size = stars[non_zero_elems].size regular_size = stars.size @@ -132,12 +149,16 @@ def mccd_merge_starcat_runner(input_file_list, run_dirs, file_number_string, ra += list(starcat_j[hdu_table].data['RA_LIST'][:]) dec += list(starcat_j[hdu_table].data['DEC_LIST'][:]) except Exception: - ra += list(np.zeros( - starcat_j[hdu_table].data['GLOB_POSITION_IMG_LIST'][:, 0].shape, - dtype=int)) - dec += list(np.zeros( - starcat_j[hdu_table].data['GLOB_POSITION_IMG_LIST'][:, 0].shape, - dtype=int)) + ra += list( + np.zeros(starcat_j[hdu_table].data[ + 'GLOB_POSITION_IMG_LIST'][:, 0].shape, + dtype=int) + ) + dec += list( + np.zeros(starcat_j[hdu_table].data[ + 'GLOB_POSITION_IMG_LIST'][:, 0].shape, + dtype=int) + ) # shapes (convert sigmas to R^2) g1_psf += list(starcat_j[hdu_table].data['PSF_MOM_LIST'][:, 0]) @@ -160,56 +181,63 @@ def mccd_merge_starcat_runner(input_file_list, run_dirs, file_number_string, # Stats about the merging def rmse_calc(values, sizes): return np.sqrt( - np.nansum(np.array(values)) / np.nansum(np.array(sizes))) + np.nansum(np.array(values)) / np.nansum(np.array(sizes)) + ) def rmse_calc_2(values, sizes): return np.sqrt( - np.nansum(np.array(values) ** 2) / np.nansum(np.array(sizes))) + np.nansum(np.array(values) ** 2) / np.nansum(np.array(sizes)) + ) def mean_calc(values, sizes): - return np.nansum(np.array(values)) / np.nansum( - np.array(sizes)) + return np.nansum( + np.array(values)) / np.nansum(np.array(sizes) + ) def std_calc(values): return np.nanstd(np.array(values)) # Regular pixel RMSE tot_pixel_rmse = rmse_calc(pixel_mse, size_mse) - # print('Regular Total pixel RMSE = %.5e\n' % tot_pixel_rmse) - w_log.info('MCCD_merge_starcat: Regular Total pixel RMSE = %.5e\n' % ( - tot_pixel_rmse)) + w_log.info( + 'MCCD_merge_starcat: Regular Total pixel RMSE = %.5e\n' % ( + tot_pixel_rmse) + ) # Regular Total pixel mean tot_pixel_mean = mean_calc(pixel_sum, size_mse) - # print('Regular Total pixel mean = %.5e\n' % tot_pixel_mean) - w_log.info('MCCD_merge_starcat: Regular Total pixel mean = %.5e\n' % ( - tot_pixel_mean)) + w_log.info( + 'MCCD_merge_starcat: Regular Total pixel mean = %.5e\n' % ( + tot_pixel_mean) + ) # Regular Total MASKED pixel RMSE tot_masked_pixel_rmse = rmse_calc(masked_pixel_mse, masked_size) - # print('Regular Total MASKED pixel RMSE = %.5e\n' % tot_masked_pixel_rmse) w_log.info( 'MCCD_merge_starcat: Regular Total MASKED pixel RMSE = %.5e\n' % ( - tot_masked_pixel_rmse)) + tot_masked_pixel_rmse) + ) # Regular Total MASKED pixel mean tot_masked_pixel_mean = mean_calc(masked_pixel_sum, masked_size) - # print('Regular Total MASKED pixel mean = %.5e\n' % tot_masked_pixel_mean) w_log.info( 'MCCD_merge_starcat: Regular Total MASKED pixel mean = %.5e\n' % ( - tot_masked_pixel_mean)) + tot_masked_pixel_mean) + ) # Normalized pixel RMSE tot_pix_norm_rmse = rmse_calc(pix_norm_mse, size_norm_mse) - # print('Normalized Total pixel RMSE = %.5e\n' % tot_pix_norm_rmse) - w_log.info('MCCD_merge_starcat: Normalized Total pixel RMSE = %.5e\n' % ( - tot_pix_norm_rmse)) + w_log.info( + 'MCCD_merge_starcat: Normalized Total pixel RMSE = %.5e\n' % ( + tot_pix_norm_rmse) + ) # Normalized filtered pixel RMSE tot_pix_filt_rmse = rmse_calc(pix_filt_mse, size_filt_mse) - # print('Filtered Total pixel RMSE = %.5e\n' % tot_pix_filt_rmse) - w_log.info('MCCD_merge_starcat: Filtered Total pixel RMSE = %.5e\n' % ( - tot_pix_filt_rmse)) + w_log.info( + 'MCCD_merge_starcat: Filtered Total pixel RMSE = %.5e\n' % ( + tot_pix_filt_rmse) + ) concat_model_var = np.concatenate(np.array(model_var)) concat_star_noise_var = np.concatenate(np.array(star_noise_var)) @@ -218,23 +246,21 @@ def std_calc(values): mean_model_var = mean_calc(concat_model_var, model_var_size) std_model_var = std_calc(concat_model_var) rmse_model_var = rmse_calc_2(concat_model_var, model_var_size) - # print("MCCD-RCA variance:\n Mean Variance= %.5e\n " - # "Std Variance= %.5e\n RMSE Variance= %.5e\n" % ( - # mean_model_var, std_model_var, rmse_model_var)) - w_log.info("MCCD-RCA variance:\n Mean Variance= %.5e\n " - "Std Variance= %.5e\n RMSE Variance= %.5e\n" % - (mean_model_var, std_model_var, rmse_model_var)) + w_log.info( + "MCCD-RCA variance:\n Mean Variance= %.5e\n " + "Std Variance= %.5e\n RMSE Variance= %.5e\n" % + (mean_model_var, std_model_var, rmse_model_var) + ) # Star Noise Variance mean_star_var = mean_calc(concat_star_noise_var, star_noise_size) std_star_var = std_calc(concat_star_noise_var) rmse_star_var = rmse_calc_2(concat_star_noise_var, star_noise_size) - # print("Masked stars variance:\n Mean Variance= %.5e\n" - # "Std Variance= %.5e\n RMSE Variance= %.5e\n" % ( - # mean_star_var, std_star_var, rmse_star_var)) - w_log.info("Masked stars variance:\n Mean Variance= %.5e\n" - "Std Variance= %.5e\n RMSE Variance= %.5e\n" % - (mean_star_var, std_star_var, rmse_star_var)) + w_log.info( + "Masked stars variance:\n Mean Variance= %.5e\n" + "Std Variance= %.5e\n RMSE Variance= %.5e\n" % + (mean_star_var, std_star_var, rmse_star_var) + ) # Mask and transform to numpy arrays flagmask = np.abs(np.array(flag_star) - 1) * np.abs(np.array(flag_psf) - 1) @@ -246,38 +272,48 @@ def std_calc(values): star_r2 = np.array(size)[flagmask.astype(bool)] rmse, mean, std_dev = stats_calculator(star_e1, psf_e1) - # print("Moment residual e1:\n Mean= %.5e\n " - # "Std Dev= %.5e\n RMSE= %.5e\n" % (mean, std_dev, rmse)) - w_log.info("Moment residual e1:\n Mean= %.5e\n" - "Std Dev= %.5e\n RMSE= %.5e\n" % (mean, std_dev, rmse)) + w_log.info( + "Moment residual e1:\n Mean= %.5e\n" + "Std Dev= %.5e\n RMSE= %.5e\n" % (mean, std_dev, rmse) + ) rmse, mean, std_dev = stats_calculator(star_e2, psf_e2) - # print("Moment residual e2:\n Mean= %.5e\n" - # "Std Dev= %.5e\n RMSE= %.5e\n" % (mean, std_dev, rmse)) - w_log.info("Moment residual e2:\n Mean= %.5e\n" - "Std Dev= %.5e\n RMSE= %.5e\n" % (mean, std_dev, rmse)) + w_log.info( + "Moment residual e2:\n Mean= %.5e\n" + "Std Dev= %.5e\n RMSE= %.5e\n" % (mean, std_dev, rmse) + ) rmse, mean, std_dev = stats_calculator(star_r2, psf_r2) - # print("Moment residual R2:\n Mean= %.5e\n" - # "Std Dev= %.5e\n RMSE= %.5e\n" % (mean, std_dev, rmse)) - w_log.info("Moment residual R2:\n Mean= %.5e\n" - "Std Dev= %.5e\n RMSE= %.5e\n" % (mean, std_dev, rmse)) + w_log.info( + "Moment residual R2:\n Mean= %.5e\n" + "Std Dev= %.5e\n RMSE= %.5e\n" % (mean, std_dev, rmse) + ) print('MCCD: Number of stars: %d' % (star_e1.shape[0])) w_log.info('MCCD: Number of stars: %d' % (star_e1.shape[0])) - output = sc.FITSCatalog(output_dir + '/full_starcat-0000000.fits', - open_mode=sc.BaseCatalog.OpenMode.ReadWrite, - SEx_catalog=True) + output = sc.FITSCatalog( + output_dir + '/full_starcat-0000000.fits', + open_mode=sc.BaseCatalog.OpenMode.ReadWrite, + SEx_catalog=True + ) # convert back to sigma for consistency - data = {'X': x, 'Y': y, 'RA': ra, 'DEC': dec, - 'E1_PSF_HSM': g1_psf, 'E2_PSF_HSM': g2_psf, - 'SIGMA_PSF_HSM': np.sqrt(size_psf), - 'E1_STAR_HSM': g1, 'E2_STAR_HSM': g2, - 'SIGMA_STAR_HSM': np.sqrt(size), - 'FLAG_PSF_HSM': flag_psf, 'FLAG_STAR_HSM': flag_star, - 'CCD_NB': ccd_nb} + data = { + 'X': x, + 'Y': y, + 'RA': ra, + 'DEC': dec, + 'E1_PSF_HSM': g1_psf, + 'E2_PSF_HSM': g2_psf, + 'SIGMA_PSF_HSM': np.sqrt(size_psf), + 'E1_STAR_HSM': g1, + 'E2_STAR_HSM': g2, + 'SIGMA_STAR_HSM': np.sqrt(size), + 'FLAG_PSF_HSM': flag_psf, + 'FLAG_STAR_HSM': flag_star, + 'CCD_NB': ccd_nb + } print('Writing full catalog...') output.save_as_fits(data, sex_cat_path=input_file_list[0][0]) From 699b75b0e9f0b10a6c1a92d21e7b020ce10e32ca Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 13:19:42 +0200 Subject: [PATCH 14/26] Adding style changes to mccd_plots_runner --- shapepipe/modules/mccd_plots_runner.py | 76 ++++++++++++++++---------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/shapepipe/modules/mccd_plots_runner.py b/shapepipe/modules/mccd_plots_runner.py index de072d0ac..4b871a72e 100644 --- a/shapepipe/modules/mccd_plots_runner.py +++ b/shapepipe/modules/mccd_plots_runner.py @@ -45,14 +45,22 @@ has_mpl = False -@module_runner(input_module=['mccd_merge_starcat_runner'], version='1.0', - file_pattern=['full_starcat'], - file_ext=['.fits'], numbering_scheme='-0000000', - depends=['numpy', 'mccd', 'astropy', 'matplotlib', 'stile', - 'treecorr'], - run_method='serial') -def mccd_plots_runner(input_file_list, run_dirs, file_number_string, - config, w_log): +@module_runner( + input_module=['mccd_merge_starcat_runner'], + version='1.0', + file_pattern=['full_starcat'], + file_ext=['.fits'], + numbering_scheme='-0000000', + depends=['numpy', 'mccd', 'astropy', 'matplotlib', 'stile', 'treecorr'], + run_method='serial' +) +def mccd_plots_runner( + input_file_list, + run_dirs, + file_number_string, + config, + w_log +): # Get parameters for meanshapes plots x_nb_bins = config.getint('MCCD_PLOTS_RUNNER', 'X_GRID') y_nb_bins = config.getint('MCCD_PLOTS_RUNNER', 'Y_GRID') @@ -70,37 +78,49 @@ def mccd_plots_runner(input_file_list, run_dirs, file_number_string, if plot_meanshapes or plot_histograms: if has_mpl: - mccd_plots.plot_meanshapes(starcat_path, output_path, nb_pixel, - w_log, remove_outliers, - plot_meanshapes, - plot_histograms) + mccd_plots.plot_meanshapes( + starcat_path, + output_path, + nb_pixel, + w_log, + remove_outliers, + plot_meanshapes, + plot_histograms + ) else: - msg = "[!] In order to plot the Meanshapes the package " \ - "_matplotlib_ has to be correctly imported." \ - "This was not the case, so the task is" \ - "aborted. For the next time make sure the package is" \ - "installed." + msg = ( + "[!] In order to plot the Meanshapes the package " + + "_matplotlib_ has to be correctly imported. This was not" + + " the case, so the task is aborted. For the next time make" + + " sure the package is installed." + ) warnings.warn(msg) w_log.info(msg) if plot_rho_stats: if has_stile is False or has_treecorr is False: - msg = "[!] In order to calculate the Rho stats the packages " \ - "_stile_ and _treecorr_ have to be correctly imported." \ - "This was not the case, so the rho stat calculation is" \ - "aborted. For the next time make sure both of the" \ - "packages are installed." + msg = ( + "[!] In order to calculate the Rho stats the packages " + + "_stile_ and _treecorr_ have to be correctly imported." + + " This was not the case, so the rho stat calculation is" + + "aborted. For the next time make sure both of the" + + "packages are installed." + ) warnings.warn(msg) w_log.info(msg) elif rho_stat_plot_style != 'HSC' and rho_stat_plot_style != 'DEC': - msg = "The rho stat definition should be HSC or DEC." \ - "An unknown definition was used. Rho stat calculation" \ - "aborted." + msg = ( + "The rho stat definition should be HSC or DEC. An unknown" + + " definition was used. Rho stat calculation aborted." + ) warnings.warn(msg) w_log.info(msg) else: - mccd_plots.rho_stats(starcat_path, output_path, - rho_def=rho_stat_plot_style, - print_fun=lambda x: w_log.info(x)) + mccd_plots.rho_stats( + starcat_path, + output_path, + rho_def=rho_stat_plot_style, + print_fun=lambda x: w_log.info(x) + ) return None, None From 75395f68d6047ec0eaad4c753e5bd00d7f154d43 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 13:23:09 +0200 Subject: [PATCH 15/26] Adding style changes to mccd_preprocessing runner --- .../modules/mccd_preprocessing_runner.py | 63 +++++++++++-------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/shapepipe/modules/mccd_preprocessing_runner.py b/shapepipe/modules/mccd_preprocessing_runner.py index 86fa50cd5..2a9de4867 100644 --- a/shapepipe/modules/mccd_preprocessing_runner.py +++ b/shapepipe/modules/mccd_preprocessing_runner.py @@ -9,17 +9,26 @@ """ from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.MCCD_package import shapepipe_auxiliary_mccd as aux_mccd +from shapepipe.modules.MCCD_package import shapepipe_auxiliary_mccd\ + as aux_mccd import mccd -@module_runner(input_module=['setools_runner'], version='1.0', - file_pattern=['star_split_ratio_80', 'star_split_ratio_20'], - file_ext=['.fits', '.fits'], - depends=['numpy', 'mccd', 'galsim', 'astropy'], - run_method='serial') -def mccd_preprocessing_runner(input_file_list, run_dirs, file_number_string, - config, w_log): +@module_runner( + input_module=['setools_runner'], + version='1.0', + file_pattern=['star_split_ratio_80', 'star_split_ratio_20'], + file_ext=['.fits', '.fits'], + depends=['numpy', 'mccd', 'galsim', 'astropy'], + run_method='serial' +) +def mccd_preprocessing_runner( + input_file_list, + run_dirs, + file_number_string, + config, + w_log +): # Recover the MCCD config file and its params config_file_path = config.getexpanded('MCCD', 'CONFIG_PATH') mccd_mode = config.get('MCCD', 'MODE') @@ -56,28 +65,32 @@ def mccd_preprocessing_runner(input_file_list, run_dirs, file_number_string, min_n_stars_list = [1] else: - raise ValueError('''MODE should be in ["FIT", "FIT_VALIDATION", - "VALIDATION"].''') + raise ValueError( + "MODE should be in ['FIT', FIT_VALIDATION', 'VALIDATION']." + ) # Use the outfile from the pipeline and ignore the output directory from # the MCCD config file # Output paths for both newly generates datasets output_mccd_path = run_dirs['output'] + '/' - [aux_mccd.mccd_preprocessing_pipeline( - input_file_list=input_file_list, - output_path=output_mccd_path, - input_file_position=_input_pos, - min_n_stars=_min_stars, - separator=separator, - CCD_id_filter_list=None, - outlier_std_max=outlier_std_max, - save_masks=False, - save_name=_save_name, - save_extension='.fits', - verbose=verbose, - print_fun=w_log.info) - for _input_pos, _save_name, _min_stars in - zip(input_file_pos_list, save_name_list, min_n_stars_list)] + [ + aux_mccd.mccd_preprocessing_pipeline( + input_file_list=input_file_list, + output_path=output_mccd_path, + input_file_position=_input_pos, + min_n_stars=_min_stars, + separator=separator, + CCD_id_filter_list=None, + outlier_std_max=outlier_std_max, + save_masks=False, + save_name=_save_name, + save_extension='.fits', + verbose=verbose, + print_fun=w_log.info + ) + for _input_pos, _save_name, _min_stars in + zip(input_file_pos_list, save_name_list, min_n_stars_list) + ] return None, None From 907d931be07f41a06f7f92ceb46ddede2cf546e6 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 13:24:49 +0200 Subject: [PATCH 16/26] Adding style changes to mccd_val_runner --- shapepipe/modules/mccd_val_runner.py | 33 +++++++++++++++++++--------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/shapepipe/modules/mccd_val_runner.py b/shapepipe/modules/mccd_val_runner.py index e1a8828e5..ae4cc84af 100644 --- a/shapepipe/modules/mccd_val_runner.py +++ b/shapepipe/modules/mccd_val_runner.py @@ -9,16 +9,27 @@ """ from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.MCCD_package import shapepipe_auxiliary_mccd as aux_mccd +from shapepipe.modules.MCCD_package import shapepipe_auxiliary_mccd as\ + aux_mccd import mccd -@module_runner(input_module=['mccd_preprocessing_runner'], version='1.0', - file_pattern=['fitted_model', 'test_star_selection'], - file_ext=['.npy', '.fits'], numbering_scheme='-0000000', - depends=['numpy', 'mccd', 'galsim'], run_method='parallel') -def mccd_val_runner(input_file_list, run_dirs, file_number_string, - config, w_log): +@module_runner( + input_module=['mccd_preprocessing_runner'], + version='1.0', + file_pattern=['fitted_model', 'test_star_selection'], + file_ext=['.npy', '.fits'], + numbering_scheme='-0000000', + depends=['numpy', 'mccd', 'galsim'], + run_method='parallel' +) +def mccd_val_runner( + input_file_list, + run_dirs, + file_number_string, + config, + w_log +): # Recover the MCCD config file and its params config_file_path = config.getexpanded('MCCD', 'CONFIG_PATH') @@ -47,10 +58,12 @@ def mccd_val_runner(input_file_list, run_dirs, file_number_string, output_dir=output_dir, file_number_string=file_number_string, w_log=w_log, - val_saving_name=val_saving_name) + val_saving_name=val_saving_name + ) else: - raise ValueError('''mccd_val_runner should be called when the - MODE is "VALIDATION".''') + raise ValueError( + "mccd_val_runner should be called when the MODE is 'VALIDATION'." + ) return None, None From 7142e2a60d55cea1823a07e00d05d8de076ce95f Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 13:26:31 +0200 Subject: [PATCH 17/26] Small fixes to styling --- shapepipe/modules/mccd_fit_runner.py | 3 ++- shapepipe/modules/mccd_interp_runner.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/shapepipe/modules/mccd_fit_runner.py b/shapepipe/modules/mccd_fit_runner.py index 4c5e18d76..38435c110 100644 --- a/shapepipe/modules/mccd_fit_runner.py +++ b/shapepipe/modules/mccd_fit_runner.py @@ -9,7 +9,8 @@ """ from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.MCCD_package import shapepipe_auxiliary_mccd as aux_mccd +from shapepipe.modules.MCCD_package import shapepipe_auxiliary_mccd\ + as aux_mccd import mccd diff --git a/shapepipe/modules/mccd_interp_runner.py b/shapepipe/modules/mccd_interp_runner.py index c6157dea1..79ba00428 100644 --- a/shapepipe/modules/mccd_interp_runner.py +++ b/shapepipe/modules/mccd_interp_runner.py @@ -11,7 +11,8 @@ from shapepipe.modules.module_decorator import module_runner from shapepipe.modules.MCCD_package import mccd_interpolation_script\ as mccd_interp -from shapepipe.modules.MCCD_package import shapepipe_auxiliary_mccd as aux_mccd +from shapepipe.modules.MCCD_package import shapepipe_auxiliary_mccd\ + as aux_mccd import os From 5dc9e8a439846ae52dc717e7af2a4273100de4e3 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 15:47:12 +0200 Subject: [PATCH 18/26] Updating mccd_package name to lowercase --- shapepipe/modules/MCCD_package/__init__.py | 2 +- shapepipe/modules/mccd_fit_runner.py | 2 +- shapepipe/modules/mccd_fit_val_runner.py | 2 +- shapepipe/modules/mccd_interp_runner.py | 4 ++-- shapepipe/modules/mccd_plots_runner.py | 2 +- shapepipe/modules/mccd_preprocessing_runner.py | 2 +- shapepipe/modules/mccd_val_runner.py | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/shapepipe/modules/MCCD_package/__init__.py b/shapepipe/modules/MCCD_package/__init__.py index 180f073f7..afcdeb80a 100644 --- a/shapepipe/modules/MCCD_package/__init__.py +++ b/shapepipe/modules/MCCD_package/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -"""MCCD auxiliary functions package. +"""MCCD AUXILIARY FUNCTIONS PACKAGE. This module contains auxiliary functions for the MCCD package. diff --git a/shapepipe/modules/mccd_fit_runner.py b/shapepipe/modules/mccd_fit_runner.py index 38435c110..0d0364b71 100644 --- a/shapepipe/modules/mccd_fit_runner.py +++ b/shapepipe/modules/mccd_fit_runner.py @@ -9,7 +9,7 @@ """ from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.MCCD_package import shapepipe_auxiliary_mccd\ +from shapepipe.modules.mccd_package import shapepipe_auxiliary_mccd\ as aux_mccd import mccd diff --git a/shapepipe/modules/mccd_fit_val_runner.py b/shapepipe/modules/mccd_fit_val_runner.py index f7b1466f7..070d34957 100644 --- a/shapepipe/modules/mccd_fit_val_runner.py +++ b/shapepipe/modules/mccd_fit_val_runner.py @@ -9,7 +9,7 @@ """ from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.MCCD_package import shapepipe_auxiliary_mccd as aux_mccd +from shapepipe.modules.mccd_package import shapepipe_auxiliary_mccd as aux_mccd import mccd diff --git a/shapepipe/modules/mccd_interp_runner.py b/shapepipe/modules/mccd_interp_runner.py index 79ba00428..affb37b56 100644 --- a/shapepipe/modules/mccd_interp_runner.py +++ b/shapepipe/modules/mccd_interp_runner.py @@ -9,9 +9,9 @@ """ from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.MCCD_package import mccd_interpolation_script\ +from shapepipe.modules.mccd_package import mccd_interpolation_script\ as mccd_interp -from shapepipe.modules.MCCD_package import shapepipe_auxiliary_mccd\ +from shapepipe.modules.mccd_package import shapepipe_auxiliary_mccd\ as aux_mccd import os diff --git a/shapepipe/modules/mccd_plots_runner.py b/shapepipe/modules/mccd_plots_runner.py index 4b871a72e..73c6e4527 100644 --- a/shapepipe/modules/mccd_plots_runner.py +++ b/shapepipe/modules/mccd_plots_runner.py @@ -13,7 +13,7 @@ """ from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.MCCD_package import mccd_plot_utilities as mccd_plots +from shapepipe.modules.mccd_package import mccd_plot_utilities as mccd_plots import warnings diff --git a/shapepipe/modules/mccd_preprocessing_runner.py b/shapepipe/modules/mccd_preprocessing_runner.py index 2a9de4867..7ac28abfc 100644 --- a/shapepipe/modules/mccd_preprocessing_runner.py +++ b/shapepipe/modules/mccd_preprocessing_runner.py @@ -9,7 +9,7 @@ """ from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.MCCD_package import shapepipe_auxiliary_mccd\ +from shapepipe.modules.mccd_package import shapepipe_auxiliary_mccd\ as aux_mccd import mccd diff --git a/shapepipe/modules/mccd_val_runner.py b/shapepipe/modules/mccd_val_runner.py index ae4cc84af..151f80112 100644 --- a/shapepipe/modules/mccd_val_runner.py +++ b/shapepipe/modules/mccd_val_runner.py @@ -9,7 +9,7 @@ """ from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.MCCD_package import shapepipe_auxiliary_mccd as\ +from shapepipe.modules.mccd_package import shapepipe_auxiliary_mccd as\ aux_mccd import mccd From 061ae5eba8cc2aaed53f223fb59f81d2f506c1c3 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 16:06:32 +0200 Subject: [PATCH 19/26] Modifs for multiple runner configuration. Handling module_config_sec --- shapepipe/modules/mccd_fit_runner.py | 7 ++++--- shapepipe/modules/mccd_fit_val_runner.py | 7 ++++--- shapepipe/modules/mccd_interp_runner.py | 19 ++++++++++--------- .../modules/mccd_merge_starcat_runner.py | 1 + shapepipe/modules/mccd_plots_runner.py | 15 ++++++++------- .../modules/mccd_preprocessing_runner.py | 7 ++++--- shapepipe/modules/mccd_val_runner.py | 7 ++++--- 7 files changed, 35 insertions(+), 28 deletions(-) diff --git a/shapepipe/modules/mccd_fit_runner.py b/shapepipe/modules/mccd_fit_runner.py index 0d0364b71..403294a61 100644 --- a/shapepipe/modules/mccd_fit_runner.py +++ b/shapepipe/modules/mccd_fit_runner.py @@ -28,12 +28,13 @@ def mccd_fit_runner( run_dirs, file_number_string, config, + module_config_sec, w_log ): # Recover the MCCD config file and its params - config_file_path = config.getexpanded('MCCD', 'CONFIG_PATH') - mccd_mode = config.get('MCCD', 'MODE') - verbose = config.getboolean('MCCD', 'VERBOSE') + config_file_path = config.getexpanded(module_config_sec, 'CONFIG_PATH') + mccd_mode = config.get(module_config_sec, 'MODE') + verbose = config.getboolean(module_config_sec, 'VERBOSE') # Parse MCCD config file mccd_parser = mccd.auxiliary_fun.MCCDParamsParser(config_file_path) diff --git a/shapepipe/modules/mccd_fit_val_runner.py b/shapepipe/modules/mccd_fit_val_runner.py index 070d34957..189b0fd06 100644 --- a/shapepipe/modules/mccd_fit_val_runner.py +++ b/shapepipe/modules/mccd_fit_val_runner.py @@ -27,12 +27,13 @@ def mccd_fit_val_runner( run_dirs, file_number_string, config, + module_config_sec, w_log ): # Recover the MCCD config file and its params - config_file_path = config.getexpanded('MCCD', 'CONFIG_PATH') - mccd_mode = config.get('MCCD', 'MODE') - verbose = config.getboolean('MCCD', 'VERBOSE') + config_file_path = config.getexpanded(module_config_sec, 'CONFIG_PATH') + mccd_mode = config.get(module_config_sec, 'MODE') + verbose = config.getboolean(module_config_sec, 'VERBOSE') # Parse MCCD config file mccd_parser = mccd.auxiliary_fun.MCCDParamsParser(config_file_path) diff --git a/shapepipe/modules/mccd_interp_runner.py b/shapepipe/modules/mccd_interp_runner.py index affb37b56..f4b42398e 100644 --- a/shapepipe/modules/mccd_interp_runner.py +++ b/shapepipe/modules/mccd_interp_runner.py @@ -29,26 +29,27 @@ def mccd_interp_runner( run_dirs, file_number_string, config, + module_config_sec, w_log ): - mode = config.getexpanded('MCCD_INTERP_RUNNER', 'MODE') - pos_params = config.getlist('MCCD_INTERP_RUNNER', 'POSITION_PARAMS') - get_shapes = config.getboolean('MCCD_INTERP_RUNNER', 'GET_SHAPES') + mode = config.getexpanded(module_config_sec, 'MODE') + pos_params = config.getlist(module_config_sec, 'POSITION_PARAMS') + get_shapes = config.getboolean(module_config_sec, 'GET_SHAPES') mccd_model_extension = '.npy' output_dir = run_dirs['output'] if mode == 'CLASSIC': psf_model_dir = config.getexpanded( - 'MCCD_INTERP_RUNNER', + module_config_sec, 'PSF_MODEL_DIR' ) psf_model_pattern = config.get( - 'MCCD_INTERP_RUNNER', + module_config_sec, 'PSF_MODEL_PATTERN' ) psf_separator = config.get( - 'MCCD_INTERP_RUNNER', + module_config_sec, 'PSF_MODEL_SEPARATOR' ) # psfcat_path, galcat_path = input_file_list @@ -91,15 +92,15 @@ def mccd_interp_runner( elif mode == 'MULTI-EPOCH': psf_model_dir = config.getexpanded( - 'MCCD_INTERP_RUNNER', + module_config_sec, 'PSF_MODEL_DIR' ) psf_model_pattern = config.get( - 'MCCD_INTERP_RUNNER', + module_config_sec, 'PSF_MODEL_PATTERN' ) f_wcs_path = config.getexpanded( - 'MCCD_INTERP_RUNNER', + module_config_sec, 'ME_LOG_WCS' ) diff --git a/shapepipe/modules/mccd_merge_starcat_runner.py b/shapepipe/modules/mccd_merge_starcat_runner.py index 469da74ec..cf78f938b 100644 --- a/shapepipe/modules/mccd_merge_starcat_runner.py +++ b/shapepipe/modules/mccd_merge_starcat_runner.py @@ -29,6 +29,7 @@ def mccd_merge_starcat_runner( run_dirs, file_number_string, config, + module_config_sec, w_log ): w_log.info('Merging validation results..') diff --git a/shapepipe/modules/mccd_plots_runner.py b/shapepipe/modules/mccd_plots_runner.py index 73c6e4527..59401982f 100644 --- a/shapepipe/modules/mccd_plots_runner.py +++ b/shapepipe/modules/mccd_plots_runner.py @@ -59,18 +59,19 @@ def mccd_plots_runner( run_dirs, file_number_string, config, + module_config_sec, w_log ): # Get parameters for meanshapes plots - x_nb_bins = config.getint('MCCD_PLOTS_RUNNER', 'X_GRID') - y_nb_bins = config.getint('MCCD_PLOTS_RUNNER', 'Y_GRID') - remove_outliers = config.getboolean('MCCD_PLOTS_RUNNER', 'REMOVE_OUTLIERS') - plot_meanshapes = config.getboolean('MCCD_PLOTS_RUNNER', 'PLOT_MEANSHAPES') - plot_histograms = config.getboolean('MCCD_PLOTS_RUNNER', 'PLOT_HISTOGRAMS') + x_nb_bins = config.getint(module_config_sec, 'X_GRID') + y_nb_bins = config.getint(module_config_sec, 'Y_GRID') + remove_outliers = config.getboolean(module_config_sec, 'REMOVE_OUTLIERS') + plot_meanshapes = config.getboolean(module_config_sec, 'PLOT_MEANSHAPES') + plot_histograms = config.getboolean(module_config_sec, 'PLOT_HISTOGRAMS') # Get parameters for Rho stats plots - plot_rho_stats = config.getboolean('MCCD_PLOTS_RUNNER', 'PLOT_RHO_STATS') - rho_stat_plot_style = config.get('MCCD_PLOTS_RUNNER', 'RHO_STATS_STYLE') + plot_rho_stats = config.getboolean(module_config_sec, 'PLOT_RHO_STATS') + rho_stat_plot_style = config.get(module_config_sec, 'RHO_STATS_STYLE') nb_pixel = x_nb_bins, y_nb_bins starcat_path = input_file_list[0][0] diff --git a/shapepipe/modules/mccd_preprocessing_runner.py b/shapepipe/modules/mccd_preprocessing_runner.py index 7ac28abfc..627b44885 100644 --- a/shapepipe/modules/mccd_preprocessing_runner.py +++ b/shapepipe/modules/mccd_preprocessing_runner.py @@ -27,12 +27,13 @@ def mccd_preprocessing_runner( run_dirs, file_number_string, config, + module_config_sec, w_log ): # Recover the MCCD config file and its params - config_file_path = config.getexpanded('MCCD', 'CONFIG_PATH') - mccd_mode = config.get('MCCD', 'MODE') - verbose = config.getboolean('MCCD', 'VERBOSE') + config_file_path = config.getexpanded(module_config_sec, 'CONFIG_PATH') + mccd_mode = config.get(module_config_sec, 'MODE') + verbose = config.getboolean(module_config_sec, 'VERBOSE') # Parse MCCD config file mccd_parser = mccd.auxiliary_fun.MCCDParamsParser(config_file_path) diff --git a/shapepipe/modules/mccd_val_runner.py b/shapepipe/modules/mccd_val_runner.py index 151f80112..a9ab9c0fb 100644 --- a/shapepipe/modules/mccd_val_runner.py +++ b/shapepipe/modules/mccd_val_runner.py @@ -28,13 +28,14 @@ def mccd_val_runner( run_dirs, file_number_string, config, + module_config_sec, w_log ): # Recover the MCCD config file and its params - config_file_path = config.getexpanded('MCCD', 'CONFIG_PATH') - mccd_mode = config.get('MCCD', 'MODE') - verbose = config.getboolean('MCCD', 'VERBOSE') + config_file_path = config.getexpanded(module_config_sec, 'CONFIG_PATH') + mccd_mode = config.get(module_config_sec, 'MODE') + verbose = config.getboolean(module_config_sec, 'VERBOSE') # Parse MCCD config file mccd_parser = mccd.auxiliary_fun.MCCDParamsParser(config_file_path) From ae87b7791b8daf167973c2fc73ecf70dcc6c2d9e Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 16:30:47 +0200 Subject: [PATCH 20/26] Changed string to f-string convention --- .../MCCD_package/mccd_plot_utilities.py | 111 +++++++++--------- 1 file changed, 55 insertions(+), 56 deletions(-) diff --git a/shapepipe/modules/MCCD_package/mccd_plot_utilities.py b/shapepipe/modules/MCCD_package/mccd_plot_utilities.py index b5903d480..f03f1b956 100644 --- a/shapepipe/modules/MCCD_package/mccd_plot_utilities.py +++ b/shapepipe/modules/MCCD_package/mccd_plot_utilities.py @@ -117,12 +117,12 @@ def MeanShapesPlot( ) ax.set_xticks([]) ax.set_yticks([]) - ax.set_title('rmse=%.3e' % (np.sqrt(np.nanmean(ccd_map ** 2))), size=8) + ax.set_title(f'rmse={np.sqrt(np.nanmean(ccd_map ** 2)):.3e}', size=8) plt.suptitle(title, size=20) # TODO: fix title fig.subplots_adjust(right=0.8) cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7]) fig.colorbar(im, cax=cbar_ax) - plt.savefig('{}.png'.format(filename)) + plt.savefig(f'{filename}.png') plt.close() @@ -183,7 +183,7 @@ def plot_meanshapes( R2_thresh = shape_std_max * np.std(all_star_shapes[2, :]) + np.mean( all_star_shapes[2, :]) bad_stars = (abs(all_star_shapes[2, :]) > R2_thresh) - w_log.info('Nb of outlier stars: %d' % (np.sum(bad_stars))) + w_log.info(f'Nb of outlier stars: {np.sum(bad_stars):d}') # Remove outlier PSFs all_star_shapes = all_star_shapes[:, ~bad_stars] all_psf_shapes = all_psf_shapes[:, ~bad_stars] @@ -192,12 +192,18 @@ def plot_meanshapes( all_Y = all_Y[~bad_stars] flagmask = flagmask[~bad_stars] - w_log.info('TOTAL e1 residual RMSE: %.6e\n' % ( - np.sqrt(np.mean((all_star_shapes[0, :] - all_psf_shapes[0, :]) ** 2)))) - w_log.info('TOTAL e2 residual RMSE: %.6e\n' % ( - np.sqrt(np.mean((all_star_shapes[1, :] - all_psf_shapes[1, :]) ** 2)))) - w_log.info('TOTAL R2 residual RMSE: %.6e\n' % ( - np.sqrt(np.mean((all_star_shapes[2, :] - all_psf_shapes[2, :]) ** 2)))) + e1_res_rmse = np.sqrt( + np.mean((all_star_shapes[0, :] - all_psf_shapes[0, :]) ** 2) + ) + e2_res_rmse = np.sqrt( + np.mean((all_star_shapes[1, :] - all_psf_shapes[1, :]) ** 2) + ) + R2_res_rmse = np.sqrt( + np.mean((all_star_shapes[2, :] - all_psf_shapes[2, :]) ** 2) + ) + w_log.info(f"TOTAL e1 residual RMSE: {e1_res_rmse:.6e}\n") + w_log.info(f"TOTAL e2 residual RMSE: {e2_res_rmse:.6e}\n") + w_log.info(f"TOTAL R2 residual RMSE: {R2_res_rmse:.6e}\n") # CCDs x star/model x (e1,e2,R2,nstars) x xpos x ypos ccd_maps = np.ones((40, 2, 4) + nb_pixel) * np.nan @@ -240,9 +246,9 @@ def plot_meanshapes( np.abs(np.nanmin(ccd_maps[:, :, 0]))) vmin = -vmax wind = [vmin, vmax] - title = "e_1 (stars), std=%.5e\nvmax=%.4e" % ( - np.nanstd(ccd_maps[:, 0, 0]), - np.nanmax(abs(ccd_maps[:, 0, 0])) + title = ( + f"e_1 (stars), std={np.nanstd(ccd_maps[:, 0, 0]):.5e}\n" + + f"vmax={np.nanmax(abs(ccd_maps[:, 0, 0])):.4e}" ) MeanShapesPlot( ccd_maps[:, 0, 0], @@ -251,9 +257,9 @@ def plot_meanshapes( wind=wind ) - title = "e_1 (model), std=%.5e\nvmax=%.4e" % ( - np.nanstd(ccd_maps[:, 1, 0]), - np.nanmax(abs(ccd_maps[:, 1, 0])) + title = ( + f"e_1 (model), std={np.nanstd(ccd_maps[:, 1, 0]):.5e}\n" + + f"vmax={np.nanmax(abs(ccd_maps[:, 1, 0])):.4e}" ) MeanShapesPlot( ccd_maps[:, 1, 0], @@ -267,14 +273,13 @@ def plot_meanshapes( e1_res = ccd_maps[:, 0, 0] - ccd_maps[:, 1, 0] e1_res = e1_res[~np.isnan(e1_res)] rmse_e1 = np.sqrt(np.mean(e1_res ** 2)) - w_log.info("Bins: e1 residual RMSE: %.6f\n" % rmse_e1) + w_log.info(f"Bins: e1 residual RMSE: {rmse_e1:.6f}\n") vmax = np.nanmax(abs(ccd_maps[:, 0, 0] - ccd_maps[:, 1, 0])) vmin = -vmax wind = [vmin, vmax] - title = "e_1 res, rmse=%.5e\nvmax=%.4e , std=%.5e" % ( - rmse_e1, - vmax, - np.nanstd(ccd_maps[:, 0, 0] - ccd_maps[:, 1, 0]) + title = ( + f"e_1 res, rmse={rmse_e1:.5e}\nvmax={vmax:.4e} , " + + f"std={np.nanstd(ccd_maps[:, 0, 0] - ccd_maps[:, 1, 0]):.5e}" ) MeanShapesPlot( ccd_maps[:, 0, 0] - ccd_maps[:, 1, 0], @@ -285,13 +290,15 @@ def plot_meanshapes( ) # e_2 - vmax = max(np.nanmax(ccd_maps[:, :, 1]), - np.abs(np.nanmin(ccd_maps[:, :, 1]))) + vmax = max( + np.nanmax(ccd_maps[:, :, 1]), + np.abs(np.nanmin(ccd_maps[:, :, 1])) + ) vmin = -vmax wind = [vmin, vmax] - title = "e_2 (stars), std=%.5e\nvmax=%.4e" % ( - np.nanstd(ccd_maps[:, 0, 1]), - np.nanmax(abs(ccd_maps[:, 0, 1])) + title = ( + f"e_2 (stars), std={np.nanstd(ccd_maps[:, 0, 1]):.5e}\n" + + f"vmax={np.nanmax(abs(ccd_maps[:, 0, 1])):.4e}" ) MeanShapesPlot( ccd_maps[:, 0, 1], @@ -299,9 +306,9 @@ def plot_meanshapes( title, wind=wind ) - title = "e_2 (model), std=%.5e\nvmax=%.4e" % ( - np.nanstd(ccd_maps[:, 1, 1]), - np.nanmax(abs(ccd_maps[:, 1, 1])) + title = ( + f"e_2 (model), std={np.nanstd(ccd_maps[:, 1, 1]):.5e}\n" + + f"vmax={np.nanmax(abs(ccd_maps[:, 1, 1])):.4e}" ) MeanShapesPlot( ccd_maps[:, 1, 1], @@ -317,14 +324,13 @@ def plot_meanshapes( e2_res = ccd_maps[:, 0, 1] - ccd_maps[:, 1, 1] e2_res = e2_res[~np.isnan(e2_res)] rmse_e2 = np.sqrt(np.mean(e2_res ** 2)) - w_log.info('Bins: e2 residual RMSE: %.6f\n' % rmse_e2) + w_log.info(f"Bins: e2 residual RMSE: {rmse_e2:.6f}\n") vmax = np.nanmax(abs(ccd_maps[:, 0, 1] - ccd_maps[:, 1, 1])) vmin = -vmax wind = [vmin, vmax] - title = 'e_2 res, rmse=%.5e\nvmax=%.4e , std=%.5e' % ( - rmse_e2, - vmax, - np.nanstd(ccd_maps[:, 0, 1] - ccd_maps[:, 1, 1]) + title = ( + f"e_2 res, rmse={rmse_e2:.5e}\nvmax={vmax:.4e} , " + + f"std={np.nanstd(ccd_maps[:, 0, 1] - ccd_maps[:, 1, 1]):.5e}" ) MeanShapesPlot( ccd_maps[:, 0, 1] - ccd_maps[:, 1, 1], @@ -337,9 +343,9 @@ def plot_meanshapes( # R^2 wind = [0, np.nanmax(ccd_maps[:, :, 2])] colorbar_ampl = 1 - title = 'R_2 (stars), std=%.5e\nvmax=%.4e' % ( - np.nanstd(ccd_maps[:, 0, 2]), - np.nanmax(abs(ccd_maps[:, 0, 2])) + title = ( + f"R_2 (stars), std={np.nanstd(ccd_maps[:, 0, 2]):.5e}\n" + + f"vmax={np.nanmax(abs(ccd_maps[:, 0, 2])):.4e}" ) MeanShapesPlot( ccd_maps[:, 0, 2], @@ -348,10 +354,9 @@ def plot_meanshapes( wind=wind, cmap='Reds' ) - - title = 'R_2 (model), std=%.5e\nvmax=%.4e' % ( - np.nanstd(ccd_maps[:, 1, 2]), - np.nanmax(abs(ccd_maps[:, 1, 2])) + title = ( + f"R_2 (model), std={np.nanstd(ccd_maps[:, 1, 2]):.5e}\n" + + f"vmax={np.nanmax(abs(ccd_maps[:, 1, 2])):.4e}" ) MeanShapesPlot( ccd_maps[:, 1, 2], @@ -368,25 +373,19 @@ def plot_meanshapes( R2_res = (ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / ccd_maps[:, 0, 2] R2_res = R2_res[~np.isnan(R2_res)] rmse_r2 = np.sqrt(np.mean(R2_res ** 2)) - w_log.info('Bins: R2 residual RMSE: %.6f\n' % rmse_r2) + w_log.info(f"Bins: R2 residual RMSE: {rmse_r2:.6f}\n") vmax = np.nanmax( abs((ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / ccd_maps[:, 0, 2])) wind = [0, vmax] + std_title = np.nanstd( + (ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / ccd_maps[:, 0, 2] + ) + title = ( + f"∆(R_2)/R_2 res, rmse={rmse_r2:.5e}\nvmax={vmax:.4e} , " + + f"std={std_title:.5e}" + ) if remove_outliers: - title = "Outliers removed\n∆(R_2)/R_2 res, " \ - "rmse=%.5e\nvmax=%.4e , std=%.5e" % ( - rmse_r2, - vmax, - np.nanstd((ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / - ccd_maps[:, 0, 2]) - ) - else: - title = "∆(R_2)/R_2 res, rmse=%.5e\nvmax=%.4e , std=%.5e" % ( - rmse_r2, - vmax, - np.nanstd((ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / - ccd_maps[:, 0, 2]) - ) + title = "Outliers removed\n" + title MeanShapesPlot( np.abs((ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / @@ -400,7 +399,7 @@ def plot_meanshapes( # nstars wind = (0, np.max(ccd_maps[:, 0, 3])) - title = 'Number of stars\nTotal=%d' % (np.nansum(ccd_maps[:, 0, 3])) + title = f"Number of stars\nTotal={np.nansum(ccd_maps[:, 0, 3]):d}" MeanShapesPlot( ccd_maps[:, 0, 3], output_path + 'nstar', From f60bb680c3e371d9288347575f94e12c1796acc4 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 16:41:13 +0200 Subject: [PATCH 21/26] Final styleguide changes to shapepipe_auxiliary_mccd --- .../MCCD_package/shapepipe_auxiliary_mccd.py | 85 +++++++++++-------- 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py b/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py index 8292f07f9..d0bd5fa52 100644 --- a/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py +++ b/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py @@ -148,14 +148,18 @@ def print_fun(x): mccd_RA_list.append(RA_list[j]) mccd_DEC_list.append(DEC_list[j]) else: - msg = '''Not enough stars in catalog_id %s - ,ccd %d. Total stars = %d''' % ( - catalog_id, ccd_list[j], n_stars) + msg = ( + f"Not enough stars in catalog_id {catalog_id} " + + f",ccd {ccd_list[j]:d}. " + + f"Total stars = {n_stars:d}." + ) print_fun(msg) except Exception: - msg = '''Warning! Problem detected in - catalog_id %s ,ccd %d''' % (catalog_id, ccd_list[j]) + msg = ( + f"Warning! Problem detected in catalog_id " + + f"{catalog_id} ,ccd {ccd_list[j]:d}" + ) print_fun(msg) if mccd_pos_list: @@ -190,19 +194,22 @@ def print_fun(x): mccd_star_nb += mccd_stars.shape[0] # Save the fits file - train_dic = {'VIGNET_LIST': mccd_stars, - 'GLOB_POSITION_IMG_LIST': mccd_poss, - 'MASK_LIST': mccd_masks, 'CCD_ID_LIST': mccd_ccds, - 'SNR_WIN_LIST': mccd_SNRs, - 'RA_LIST': mccd_RAs, 'DEC_LIST': mccd_DECs - } + train_dic = { + 'VIGNET_LIST': mccd_stars, + 'GLOB_POSITION_IMG_LIST': mccd_poss, + 'MASK_LIST': mccd_masks, + 'CCD_ID_LIST': mccd_ccds, + 'SNR_WIN_LIST': mccd_SNRs, + 'RA_LIST': mccd_RAs, + 'DEC_LIST': mccd_DECs + } saving_path = output_path + save_name + separator \ + catalog_id + save_extension mccd.mccd_utils.save_to_fits(train_dic, saving_path) print_fun('Finished the training dataset processing.') - print_fun('Total stars processed = %d' % mccd_star_nb) + print_fun(f"Total stars processed = {mccd_star_nb:d}") return mccd_inputs @@ -263,7 +270,7 @@ def mccd_validation_pipeline( val_saving_name ): r"""Validate the MCCD trained model against a set of observations.""" - w_log.info('Validating catalog %s..' % file_number_string) + w_log.info(f"Validating catalog {file_number_string}..") # Get MCCD parameters save_extension = '.fits' @@ -288,11 +295,13 @@ def mccd_validation_pipeline( # Save validation dictionary to fits file mccd.mccd_utils.save_to_fits(val_dict, val_saving_path) - w_log.info('Validation catalog < %s > saved.' % val_saving_path) + w_log.info(f"Validation catalog < {val_saving_path} > saved.") else: - w_log.info('''Fitted model corresponding to catalog %s was not - found.''' % file_number_string) + w_log.info( + f"Fitted model corresponding to catalog" + + f" {file_number_string} was not found." + ) def mccd_interpolation_pipeline( @@ -321,15 +330,20 @@ def mccd_interpolation_pipeline( if interp_PSFs is not None: if get_shapes: - PSF_moms = [galsim.hsm.FindAdaptiveMom( - galsim.Image(psf), strict=False) for psf in interp_PSFs] - - PSF_shapes = np.array([[moms.observed_shape.g1, - moms.observed_shape.g2, - moms.moments_sigma, - int(bool(moms.error_message))] - for moms in PSF_moms] - ) + PSF_moms = [ + galsim.hsm.FindAdaptiveMom(galsim.Image(psf), strict=False) + for psf in interp_PSFs + ] + + PSF_shapes = np.array( + [ + [moms.observed_shape.g1, + moms.observed_shape.g2, + moms.moments_sigma, + int(bool(moms.error_message))] + for moms in PSF_moms + ] + ) shapepipe_write_output( saving_path=saving_path, @@ -355,17 +369,20 @@ def shapepipe_write_output( The saved files are compatible with the previous shapepipe's standard. """ - output = sc.FITSCatalog(saving_path, - open_mode=sc.BaseCatalog.OpenMode.ReadWrite, - SEx_catalog=True) + output = sc.FITSCatalog( + saving_path, + open_mode=sc.BaseCatalog.OpenMode.ReadWrite, + SEx_catalog=True + ) if get_shapes: - data = {'VIGNET': interp_PSFs, - 'E1_PSF_HSM': PSF_shapes[:, 0], - 'E2_PSF_HSM': PSF_shapes[:, 1], - 'SIGMA_PSF_HSM': PSF_shapes[:, 2], - 'FLAG_PSF_HSM': PSF_shapes[:, 3].astype(int) - } + data = { + 'VIGNET': interp_PSFs, + 'E1_PSF_HSM': PSF_shapes[:, 0], + 'E2_PSF_HSM': PSF_shapes[:, 1], + 'SIGMA_PSF_HSM': PSF_shapes[:, 2], + 'FLAG_PSF_HSM': PSF_shapes[:, 3].astype(int) + } else: data = {'VIGNET': interp_PSFs} From bceae67ec4574494d7fa1e1f751a7b8c4772b961 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 16:51:25 +0200 Subject: [PATCH 22/26] Updated to f-string convention mccd_merge_starcat_runner --- .../modules/mccd_merge_starcat_runner.py | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/shapepipe/modules/mccd_merge_starcat_runner.py b/shapepipe/modules/mccd_merge_starcat_runner.py index cf78f938b..5819776ec 100644 --- a/shapepipe/modules/mccd_merge_starcat_runner.py +++ b/shapepipe/modules/mccd_merge_starcat_runner.py @@ -201,43 +201,43 @@ def std_calc(values): # Regular pixel RMSE tot_pixel_rmse = rmse_calc(pixel_mse, size_mse) w_log.info( - 'MCCD_merge_starcat: Regular Total pixel RMSE = %.5e\n' % ( - tot_pixel_rmse) + f"MCCD_merge_starcat: Regular Total pixel RMSE =" + + f" {tot_pixel_rmse:.5e}\n" ) # Regular Total pixel mean tot_pixel_mean = mean_calc(pixel_sum, size_mse) w_log.info( - 'MCCD_merge_starcat: Regular Total pixel mean = %.5e\n' % ( - tot_pixel_mean) + f"MCCD_merge_starcat: Regular Total pixel mean =" + + f" {tot_pixel_mean:.5e}\n" ) # Regular Total MASKED pixel RMSE tot_masked_pixel_rmse = rmse_calc(masked_pixel_mse, masked_size) w_log.info( - 'MCCD_merge_starcat: Regular Total MASKED pixel RMSE = %.5e\n' % ( - tot_masked_pixel_rmse) + f"MCCD_merge_starcat: Regular Total MASKED pixel RMSE =" + + f" {tot_masked_pixel_rmse:.5e}\n" ) # Regular Total MASKED pixel mean tot_masked_pixel_mean = mean_calc(masked_pixel_sum, masked_size) w_log.info( - 'MCCD_merge_starcat: Regular Total MASKED pixel mean = %.5e\n' % ( - tot_masked_pixel_mean) + f"MCCD_merge_starcat: Regular Total MASKED pixel mean =" + + f" {tot_masked_pixel_mean:.5e}\n" ) # Normalized pixel RMSE tot_pix_norm_rmse = rmse_calc(pix_norm_mse, size_norm_mse) w_log.info( - 'MCCD_merge_starcat: Normalized Total pixel RMSE = %.5e\n' % ( - tot_pix_norm_rmse) + "MCCD_merge_starcat: Normalized Total pixel RMSE =" + + f" {tot_pix_norm_rmse:.5e}\n" ) # Normalized filtered pixel RMSE tot_pix_filt_rmse = rmse_calc(pix_filt_mse, size_filt_mse) w_log.info( - 'MCCD_merge_starcat: Filtered Total pixel RMSE = %.5e\n' % ( - tot_pix_filt_rmse) + "MCCD_merge_starcat: Filtered Total pixel RMSE =" + + f" {tot_pix_filt_rmse:.5e}\n" ) concat_model_var = np.concatenate(np.array(model_var)) @@ -248,9 +248,9 @@ def std_calc(values): std_model_var = std_calc(concat_model_var) rmse_model_var = rmse_calc_2(concat_model_var, model_var_size) w_log.info( - "MCCD-RCA variance:\n Mean Variance= %.5e\n " - "Std Variance= %.5e\n RMSE Variance= %.5e\n" % - (mean_model_var, std_model_var, rmse_model_var) + f"MCCD-RCA variance:\n Mean Variance= {mean_model_var:.5e}\n" + + f"Std Variance= {std_model_var:.5e}\n" + + f"RMSE Variance= {rmse_model_var:.5e}\n" ) # Star Noise Variance @@ -258,9 +258,9 @@ def std_calc(values): std_star_var = std_calc(concat_star_noise_var) rmse_star_var = rmse_calc_2(concat_star_noise_var, star_noise_size) w_log.info( - "Masked stars variance:\n Mean Variance= %.5e\n" - "Std Variance= %.5e\n RMSE Variance= %.5e\n" % - (mean_star_var, std_star_var, rmse_star_var) + f"Masked stars variance:\n Mean Variance= {mean_star_var:.5e}\n" + + f"Std Variance= {std_star_var:.5e}\n" + + f"RMSE Variance= {rmse_star_var:.5e}\n" ) # Mask and transform to numpy arrays @@ -274,24 +274,24 @@ def std_calc(values): rmse, mean, std_dev = stats_calculator(star_e1, psf_e1) w_log.info( - "Moment residual e1:\n Mean= %.5e\n" - "Std Dev= %.5e\n RMSE= %.5e\n" % (mean, std_dev, rmse) + f"Moment residual e1:\n Mean= {mean:.5e}\nStd Dev= {std_dev:.5e}\n" + + f" RMSE= {rmse:.5e}\n" ) rmse, mean, std_dev = stats_calculator(star_e2, psf_e2) w_log.info( - "Moment residual e2:\n Mean= %.5e\n" - "Std Dev= %.5e\n RMSE= %.5e\n" % (mean, std_dev, rmse) + f"Moment residual e2:\n Mean= {mean:.5e}\nStd Dev= {std_dev:.5e}\n" + + f"RMSE= {rmse:.5e}\n" ) rmse, mean, std_dev = stats_calculator(star_r2, psf_r2) w_log.info( - "Moment residual R2:\n Mean= %.5e\n" - "Std Dev= %.5e\n RMSE= %.5e\n" % (mean, std_dev, rmse) + f"Moment residual R2:\n Mean= {mean:.5e}\nStd Dev= {std_dev:.5e}\n" + + f"RMSE= {rmse:.5e}\n" ) - print('MCCD: Number of stars: %d' % (star_e1.shape[0])) - w_log.info('MCCD: Number of stars: %d' % (star_e1.shape[0])) + print(f"MCCD: Number of stars: {star_e1.shape[0]:d}") + w_log.info(f"MCCD: Number of stars: {star_e1.shape[0]:d}") output = sc.FITSCatalog( output_dir + '/full_starcat-0000000.fits', From 359cd1bcc8b7afc6e0ba595a9527ea5a6ba5c018 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 16:52:29 +0200 Subject: [PATCH 23/26] Fix indentation --- shapepipe/modules/mccd_preprocessing_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shapepipe/modules/mccd_preprocessing_runner.py b/shapepipe/modules/mccd_preprocessing_runner.py index 627b44885..025099f32 100644 --- a/shapepipe/modules/mccd_preprocessing_runner.py +++ b/shapepipe/modules/mccd_preprocessing_runner.py @@ -89,7 +89,7 @@ def mccd_preprocessing_runner( save_extension='.fits', verbose=verbose, print_fun=w_log.info - ) + ) for _input_pos, _save_name, _min_stars in zip(input_file_pos_list, save_name_list, min_n_stars_list) ] From 1d50e97179a1a6216550520063752c6e51f5ad72 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 16:56:18 +0200 Subject: [PATCH 24/26] Improved indentation with suggestions --- .../MCCD_package/mccd_interpolation_script.py | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/shapepipe/modules/MCCD_package/mccd_interpolation_script.py b/shapepipe/modules/MCCD_package/mccd_interpolation_script.py index 7a510071c..b173d3679 100644 --- a/shapepipe/modules/MCCD_package/mccd_interpolation_script.py +++ b/shapepipe/modules/MCCD_package/mccd_interpolation_script.py @@ -59,10 +59,10 @@ def interp_MCCD(mccd_model_path, positions, ccd): # Positions to global coordinates loc2glob = mccd.mccd_utils.Loc2Glob() - glob_pos = np.array( - [loc2glob.loc2glob_img_coord(_ccd, _pos[0], _pos[1]) - for _ccd, _pos in zip(ccd_list, positions)] - ) + glob_pos = np.array([ + loc2glob.loc2glob_img_coord(_ccd, _pos[0], _pos[1]) + for _ccd, _pos in zip(ccd_list, positions) + ]) # Interpolate the model PSFs = mccd_instance.estimate_psf(glob_pos, ccd) @@ -199,13 +199,15 @@ def _get_psfshapes(self): for psf in self.interp_PSFs ] - self.psf_shapes = np.array( - [[moms.observed_shape.g1, - moms.observed_shape.g2, - moms.moments_sigma, - int(bool(moms.error_message))] - for moms in psf_moms] - ) + self.psf_shapes = np.array([ + [ + moms.observed_shape.g1, + moms.observed_shape.g2, + moms.moments_sigma, + int(bool(moms.error_message)) + ] + for moms in psf_moms + ]) def _write_output(self): """ Save computed PSFs to fits file. @@ -248,13 +250,15 @@ def _get_starshapes(self, star_vign): for star, mask in zip(star_vign, masks) ] - self.star_shapes = np.array( - [[moms.observed_shape.g1, - moms.observed_shape.g2, - moms.moments_sigma, - int(bool(moms.error_message))] - for moms in star_moms] - ) + self.star_shapes = np.array([ + [ + moms.observed_shape.g1, + moms.observed_shape.g2, + moms.moments_sigma, + int(bool(moms.error_message)) + ] + for moms in star_moms + ]) def _get_psfexcatdict(self, psfex_cat_path): """ Get data from PSFEx .cat file. From 075e4a018956212ad2b66d2a0dfbaeb3851f5771 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 17:01:57 +0200 Subject: [PATCH 25/26] change mccd_package directory name --- shapepipe/modules/{MCCD_package => test_mccd_package}/__init__.py | 0 .../mccd_interpolation_script.py | 0 .../{MCCD_package => test_mccd_package}/mccd_plot_utilities.py | 0 .../shapepipe_auxiliary_mccd.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename shapepipe/modules/{MCCD_package => test_mccd_package}/__init__.py (100%) rename shapepipe/modules/{MCCD_package => test_mccd_package}/mccd_interpolation_script.py (100%) rename shapepipe/modules/{MCCD_package => test_mccd_package}/mccd_plot_utilities.py (100%) rename shapepipe/modules/{MCCD_package => test_mccd_package}/shapepipe_auxiliary_mccd.py (100%) diff --git a/shapepipe/modules/MCCD_package/__init__.py b/shapepipe/modules/test_mccd_package/__init__.py similarity index 100% rename from shapepipe/modules/MCCD_package/__init__.py rename to shapepipe/modules/test_mccd_package/__init__.py diff --git a/shapepipe/modules/MCCD_package/mccd_interpolation_script.py b/shapepipe/modules/test_mccd_package/mccd_interpolation_script.py similarity index 100% rename from shapepipe/modules/MCCD_package/mccd_interpolation_script.py rename to shapepipe/modules/test_mccd_package/mccd_interpolation_script.py diff --git a/shapepipe/modules/MCCD_package/mccd_plot_utilities.py b/shapepipe/modules/test_mccd_package/mccd_plot_utilities.py similarity index 100% rename from shapepipe/modules/MCCD_package/mccd_plot_utilities.py rename to shapepipe/modules/test_mccd_package/mccd_plot_utilities.py diff --git a/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py b/shapepipe/modules/test_mccd_package/shapepipe_auxiliary_mccd.py similarity index 100% rename from shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py rename to shapepipe/modules/test_mccd_package/shapepipe_auxiliary_mccd.py From 0e6eb0fe14cc2de3cd48bf962e2a72d8032d6140 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Jul 2021 17:02:37 +0200 Subject: [PATCH 26/26] Change to the good name of the directory mccd_package --- shapepipe/modules/{test_mccd_package => mccd_package}/__init__.py | 0 .../mccd_interpolation_script.py | 0 .../{test_mccd_package => mccd_package}/mccd_plot_utilities.py | 0 .../shapepipe_auxiliary_mccd.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename shapepipe/modules/{test_mccd_package => mccd_package}/__init__.py (100%) rename shapepipe/modules/{test_mccd_package => mccd_package}/mccd_interpolation_script.py (100%) rename shapepipe/modules/{test_mccd_package => mccd_package}/mccd_plot_utilities.py (100%) rename shapepipe/modules/{test_mccd_package => mccd_package}/shapepipe_auxiliary_mccd.py (100%) diff --git a/shapepipe/modules/test_mccd_package/__init__.py b/shapepipe/modules/mccd_package/__init__.py similarity index 100% rename from shapepipe/modules/test_mccd_package/__init__.py rename to shapepipe/modules/mccd_package/__init__.py diff --git a/shapepipe/modules/test_mccd_package/mccd_interpolation_script.py b/shapepipe/modules/mccd_package/mccd_interpolation_script.py similarity index 100% rename from shapepipe/modules/test_mccd_package/mccd_interpolation_script.py rename to shapepipe/modules/mccd_package/mccd_interpolation_script.py diff --git a/shapepipe/modules/test_mccd_package/mccd_plot_utilities.py b/shapepipe/modules/mccd_package/mccd_plot_utilities.py similarity index 100% rename from shapepipe/modules/test_mccd_package/mccd_plot_utilities.py rename to shapepipe/modules/mccd_package/mccd_plot_utilities.py diff --git a/shapepipe/modules/test_mccd_package/shapepipe_auxiliary_mccd.py b/shapepipe/modules/mccd_package/shapepipe_auxiliary_mccd.py similarity index 100% rename from shapepipe/modules/test_mccd_package/shapepipe_auxiliary_mccd.py rename to shapepipe/modules/mccd_package/shapepipe_auxiliary_mccd.py