From 89c1543e73f141d693595b9fa4c5932b8903a0e5 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Mon, 30 Nov 2020 17:10:56 +0100 Subject: [PATCH 1/5] ad config MMCD file --- example/test_mccd/config_MCCD.ini | 116 ++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 example/test_mccd/config_MCCD.ini diff --git a/example/test_mccd/config_MCCD.ini b/example/test_mccd/config_MCCD.ini new file mode 100644 index 000000000..69c9e9c96 --- /dev/null +++ b/example/test_mccd/config_MCCD.ini @@ -0,0 +1,116 @@ +# Configuration file for the MCCD method + +[INPUTS] +INPUT_DIR = /Users/tliaudat/Documents/PhD/github/cosmostat_official/mccd/tests/hidden_tests/sex_inputs/ +PREPROCESSED_OUTPUT_DIR = /Users/tliaudat/Documents/PhD/github/cosmostat_official/mccd/tests/hidden_tests/preprocessed_inputs/ +OUTPUT_DIR = /Users/tliaudat/Documents/PhD/github/cosmostat_official/mccd/tests/hidden_tests/outputs/ +INPUT_REGEX_FILE_PATTERN = star_split_ratio_80-*-*.fits +INPUT_SEPARATOR = - +MIN_N_STARS = 20 +OUTLIER_STD_MAX = 100. +USE_SNR_WEIGHTS = False + +[INSTANCE] +N_COMP_LOC = 12 +D_COMP_GLOB = 3 +KSIG_LOC = 1.0 +KSIG_GLOB = 1.0 +FILTER_PATH = None + +[FIT] +LOC_MODEL = hybrid +PSF_SIZE = 6.15 +PSF_SIZE_TYPE = R2 +N_EIGENVECTS = 5 +N_ITER_RCA = 1 +N_ITER_GLOB = 2 +N_ITER_LOC = 2 +NB_SUBITER_S_LOC = 100 +NB_SUBITER_A_LOC = 500 +NB_SUBITER_S_GLOB = 30 +NB_SUBITER_A_GLOB = 200 + +[VALIDATION] +VAL_DATA_INPUT_DIR = /Users/tliaudat/Documents/PhD/github/cosmostat_official/mccd/tests/hidden_tests/sex_inputs/ +VAL_PREPROCESSED_OUTPUT_DIR = /Users/tliaudat/Documents/PhD/github/cosmostat_official/mccd/tests/hidden_tests/preprocessed_inputs/ +VAL_MODEL_INPUT_DIR = /Users/tliaudat/Documents/PhD/github/cosmostat_official/mccd/tests/hidden_tests/outputs/ +VAL_OUTPUT_DIR = /Users/tliaudat/Documents/PhD/github/cosmostat_official/mccd/tests/hidden_tests/outputs/ +VAL_REGEX_FILE_PATTERN = star_split_ratio_20-*-*.fits +VAL_SEPARATOR = - +APPLY_DEGRADATION = True +MCCD_DEBUG = False +GLOBAL_POL_INTERP = True + +# Parameter description: +# +# +# [INPUTS] +# INPUT_DIR : (Required) Must be a valid directory containing the input +# MCCD files. +# PREPROCESSED_OUTPUT_DIR : (Required) Must be a valid directory to write the +# preprocessed input files. +# OUTPUT_DIR : (Required) Must be a valid directory to write the output files. +# The constructed models will be saved. +# INPUT_REGEX_FILE_PATTERN : File pattern of the input files to use. It should +# follow regex (regular expression) standards. +# INPUT_SEPARATOR : Separator of the different fields in the filename, +# ie sexcat[SEP]catalog_id[SEP]CCD_id.fits +# MIN_N_STARS : Minimum number of stars to keep a CCD for the training. +# OUTLIER_STD_MAX : Maximum standard deviation used for the outlier rejection. +# Should not be too low as a hihg quantity of low quality +# stars will be rejected. ie 9 is a conservative rejection. +# USE_SNR_WEIGHTS : Boolean to determine if the SNR weighting strategy will +# be used. +# For now, it needs the SNR estimations from SExtractor. +# +# +# [INSTANCE] +# N_COMP_LOC : Number of components of the Local model. If LOC_MODEL is poly, +# will be the max degree D of the polynomial. +# D_COMP_GLOB : Max degree of the global polynomial model. +# KSIG_LOC : Denoising parameter of the local model. +# ie 1 is a normal denoising, 3 is a hard denoising. +# KSIG_GLOB : Denoising parameter of the global model. +# ie 1 is a normal denoising, 3 is a hard denoising. +# FILTER_PATH : Path for predefined filters. +# +# +# [FIT] +# LOC_MODEL : Defines the type of local model to use, it can be: 'rca', +# 'poly' or 'hybrid'. +# When the poly model is used, N_COMP_LOC should be used +# as the D_LOC (max degree of the poly model) +# PSF_SIZE : First guess of the PSF size. A size estimation is done anyways. +# PSF_SIZE_TYPE : Type of the size information. It can be: fwhm, R2, sigma +# N_EIGENVECTS : Number of eigenvectors to keep for the graph constraint +# construction. +# N_ITER_RCA : Number of global epochs in the algorithm. Alternation between +# global and local estimations. +# N_ITER_GLOB : Number of epochs for each global optimization. Alternations +# between A_GLOB and S_GLOB. +# N_ITER_LOC : Number of epochs for each local optimization. Alternations +# between the different A_LOC and S_LOC. +# NB_SUBITER_S_LOC : Iterations for the optimization algorithm over S_LOC. +# NB_SUBITER_A_LOC : Iterations for the optimization algorithm over A_LOC. +# NB_SUBITER_S_GLOB : Iterations for the optimization algorithm over S_GLOB. +# NB_SUBITER_A_GLOB : Iterations for the optimization algorithm over A_GLOB. +# +# +# [VALIDATION] +# VAL_DATA_INPUT_DIR : (Required) Must be a valid directory which contains the +# validation input data (test dataset). +# VAL_PREPROCESSED_OUTPUT_DIR : (Required) Must be a valid directory where the +# preprocessed input file will be saved. +# VAL_MODEL_INPUT_DIR : (Required) Must be a valid directory which contains the +# saved trained models. +# VAL_OUTPUT_DIR : (Required) Must be a valid directory where to save the +# validation outputs, test PSFs and interpolated PSFs. +# VAL_REGEX_FILE_PATTERN : Same as INPUT_REGEX_FILE_PATTERN but for validation. +# VAL_SEPARATOR : Same as INPUT_SEPARATOR but for validation. +# APPLY_DEGRADATION : Whether the PSF models should be degraded +# (sampling/shifts/flux) to match stars; use True if you +# plan on making pixel-based comparisons (residuals etc.). +# MCCD_DEBUG : Debug mode. Returns the local and global contributions. +# GLOBAL_POL_INTERP : Uses polynomial interpolation for the global model +# instead of RBF kernel interpolation. +# From 2ee4e7dc9889ba914b7f4e7c8cf3b6442d4da1b8 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Thu, 22 Apr 2021 10:37:59 +0200 Subject: [PATCH 2/5] Modify outlier from psf sizes to observed star sizes --- shapepipe/modules/MCCD_package/mccd_plot_utilities.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shapepipe/modules/MCCD_package/mccd_plot_utilities.py b/shapepipe/modules/MCCD_package/mccd_plot_utilities.py index b818493dd..05b72546c 100644 --- a/shapepipe/modules/MCCD_package/mccd_plot_utilities.py +++ b/shapepipe/modules/MCCD_package/mccd_plot_utilities.py @@ -149,9 +149,9 @@ def plot_meanshapes(starcat_path, output_path, nb_pixel, w_log, if remove_outliers: shape_std_max = 5. # Outlier rejection based on the size - R2_thresh = shape_std_max * np.std(all_psf_shapes[2, :]) + np.mean( - all_psf_shapes[2, :]) - bad_stars = (abs(all_psf_shapes[2, :]) > R2_thresh) + R2_thresh = shape_std_max * np.std(all_star_shapes[2, :]) + np.mean( + all_star_shapes[2, :]) + bad_stars = (abs(all_star_shapes[2, :]) > R2_thresh) bad_stars_idx = np.nonzero(bad_stars)[0] w_log.info('Nb of outlier stars: %d' % (np.sum(bad_stars))) # Remove outlier PSFs From 2fb3d012000dcc0c46455808979830c0732729dc Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 23 Apr 2021 18:50:27 +0200 Subject: [PATCH 3/5] Change option of fits.open() to avoid memmap --- shapepipe/modules/MCCD_package/mccd_plot_utilities.py | 4 ++-- shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py | 6 +++--- shapepipe/modules/mccd_merge_starcat_runner.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/shapepipe/modules/MCCD_package/mccd_plot_utilities.py b/shapepipe/modules/MCCD_package/mccd_plot_utilities.py index 05b72546c..a50920b90 100644 --- a/shapepipe/modules/MCCD_package/mccd_plot_utilities.py +++ b/shapepipe/modules/MCCD_package/mccd_plot_utilities.py @@ -120,7 +120,7 @@ def plot_meanshapes(starcat_path, output_path, nb_pixel, w_log, plot_histograms=True): r"""Plot meanshapes and histograms.""" # READ FULL STARCAT - starcat = fits.open(starcat_path) + starcat = fits.open(starcat_path, memmap=False) auto_colorbar = False colorbar_ampl = 1. @@ -794,7 +794,7 @@ def rho_stats(starcat_path, output_path, rho_def='HSC', depending on the desired definition to use for tho statistics. """ # Read starcat - starcat = fits.open(starcat_path) + starcat = fits.open(starcat_path, memmap=False) rho_stats_fun = None diff --git a/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py b/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py index 32e95825c..1dfc343f9 100644 --- a/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py +++ b/shapepipe/modules/MCCD_package/shapepipe_auxiliary_mccd.py @@ -202,7 +202,7 @@ def mccd_fit_pipeline(trainstar_path, file_number_string, mccd_parser, use_SNR_weight = mccd_parser.get_extra_kw('use_SNR_weight') # Open fits file - starcat = fits.open(trainstar_path) + starcat = fits.open(trainstar_path, memmap=False) mccd.auxiliary_fun.mccd_fit( starcat=starcat[1], @@ -227,7 +227,7 @@ def mccd_validation_pipeline(teststar_path, mccd_model_path, mccd_parser, # Get MCCD parameters save_extension = '.fits' mccd_val_kw = mccd_parser.get_val_kw() - testcat = fits.open(teststar_path) + testcat = fits.open(teststar_path, memmap=False) # Check if there is the fitted model if os.path.isfile(mccd_model_path): @@ -259,7 +259,7 @@ def mccd_interpolation_pipeline(mccd_model_path, galcat_path, pos_params, # Import MCCD model mccd_model = mccd.mccd_quickload(mccd_model_path) # Open galaxy catalog - galcat = fits.open(galcat_path) + galcat = fits.open(galcat_path, memmap=False) # Extract positions x_pos = galcat[2].data[pos_params[0]] diff --git a/shapepipe/modules/mccd_merge_starcat_runner.py b/shapepipe/modules/mccd_merge_starcat_runner.py index 9e8b5b911..36318e3ff 100644 --- a/shapepipe/modules/mccd_merge_starcat_runner.py +++ b/shapepipe/modules/mccd_merge_starcat_runner.py @@ -60,7 +60,7 @@ def mccd_merge_starcat_runner(input_file_list, run_dirs, file_number_string, my_mask[i, j] = True for name in input_file_list: - starcat_j = fits.open(name[0]) + starcat_j = fits.open(name[0], memmap=False) stars = np.copy(starcat_j[hdu_table].data['VIGNET_LIST']) stars[stars < -1e6] = 0 From 3b74ae197b9040a9e0982e9600fc11932eb39b36 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Fri, 30 Apr 2021 09:58:29 +0200 Subject: [PATCH 4/5] Update candide.md cmake v 3 in requirements for candide --- docs/wiki/candide.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/wiki/candide.md b/docs/wiki/candide.md index b8c9345f8..42ebba05f 100644 --- a/docs/wiki/candide.md +++ b/docs/wiki/candide.md @@ -49,6 +49,11 @@ For the installation of `astromatic` software (`SExtractor`, `PSFEx`) the `BLAS` module load intel/19.0 ``` +To install `pysap`, version 3 or higher of `cmake` needs to be loaded, the default is still 2.8, +```bash +module load cmake +``` + > You can add these commands to your `.bash_profile` to ensure that this module is available when you log in. You can list the modules already loaded by running: From c102656ba1c12d5790ac20610c4e126952005ec5 Mon Sep 17 00:00:00 2001 From: Tobias Liaudat Date: Fri, 30 Apr 2021 13:53:06 +0200 Subject: [PATCH 5/5] Update docs/wiki/candide.md Co-authored-by: Samuel Farrens --- docs/wiki/candide.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/wiki/candide.md b/docs/wiki/candide.md index 42ebba05f..b8c9345f8 100644 --- a/docs/wiki/candide.md +++ b/docs/wiki/candide.md @@ -49,11 +49,6 @@ For the installation of `astromatic` software (`SExtractor`, `PSFEx`) the `BLAS` module load intel/19.0 ``` -To install `pysap`, version 3 or higher of `cmake` needs to be loaded, the default is still 2.8, -```bash -module load cmake -``` - > You can add these commands to your `.bash_profile` to ensure that this module is available when you log in. You can list the modules already loaded by running: