diff --git a/example/cfis/config_GitFeGie_symlink.ini b/example/cfis/config_GitFeGie_symlink.ini new file mode 100644 index 000000000..bc39a1e02 --- /dev/null +++ b/example/cfis/config_GitFeGie_symlink.ini @@ -0,0 +1,144 @@ +# ShapePipe configuration file for: get images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_GitFeGie + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner, find_exposures_runner, get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[GET_IMAGES_RUNNER_RUN_1] + +FILE_PATTERN = tile_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = $HOME/astro/data/CFIS/tiles_DR2, $HOME/astro/data/CFIS/tiles_DR2 + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{3}\.\d{3} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- + +# Copy/download method, one in 'vos', 'symlink' +RETRIEVE = symlink + +# Copy command options, optional +RETRIEVE_OPTIONS = -L + + +[FIND_EXPOSURES_RUNNER] + +INPUT_MODULE = get_images_runner_run_1 + +FILE_PATTERN = CFIS_image + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Get exposures +[GET_IMAGES_RUNNER_RUN_2] + +INPUT_MODULE = find_exposures_runner + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Paths + +# Output path (optional, default is [FILE]:OUTPUT_DIR +# OUTPUT_PATH = input_images + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = $HOME/astro/data/CFIS/pitcairn_DR2, $HOME/astro/data/CFIS/weights_DR2, $HOME/astro/data/CFIS/flags_DR2 + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag + +# Input file extensions +INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{6} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = image-, weight-, flag- + +# Method to retrieve images, one in 'vos', 'symlink' +RETRIEVE = symlink + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Retrieve command options, optional +RETRIEVE_OPTIONS = -L diff --git a/example/cfis/config_GitFeGie_vos.ini b/example/cfis/config_GitFeGie_vos.ini new file mode 100644 index 000000000..40c98cd16 --- /dev/null +++ b/example/cfis/config_GitFeGie_vos.ini @@ -0,0 +1,149 @@ +# ShapePipe configuration file for: get images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_GitFeGie + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner, find_exposures_runner, get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[GET_IMAGES_RUNNER_RUN_1] + +FILE_PATTERN = tile_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = vos:cfis/tiles_DR3, vos:cfis/tiles_DR3 + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{3}\.\d{3} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- + +# Copy/download method, one in 'vos', 'symlink' +RETRIEVE = vos + +# Copy command options, optional +RETRIEVE_OPTIONS = --certfile=$VM_HOME/.ssl/cadcproxy.pem + + +[FIND_EXPOSURES_RUNNER] + +INPUT_MODULE = get_images_runner_run_1 + +FILE_PATTERN = CFIS_image + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Get exposures +[GET_IMAGES_RUNNER_RUN_2] + +INPUT_DIR = last:find_exposures_runner + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Paths + +# Output path (optional, default is [FILE]:OUTPUT_DIR +# OUTPUT_PATH = input_images + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = vos:cfis/pitcairn, vos:cfis/weights, vos:cfis/flags +# LSB images: +#INPUT_PATH = vos:cfis/lsb_individual, vos:cfis/weights, vos:cfis/flags + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag +# LSB images +#INPUT_FILE_PATTERN = 000000s, 000000p.weight, 000000p.flag + +# Input file extensions +INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{6} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = image-, weight-, flag- + +# Method to retrieve images, one in 'vos', 'symlink' +RETRIEVE = vos + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Retrieve command options, optional +RETRIEVE_OPTIONS = --certfile=$VM_HOME/.ssl/cadcproxy.pem + diff --git a/example/cfis/config_MCCD_inv_opt.ini b/example/cfis/config_MCCD_inv_opt.ini deleted file mode 100644 index c9fa539ff..000000000 --- a/example/cfis/config_MCCD_inv_opt.ini +++ /dev/null @@ -1,116 +0,0 @@ -# Configuration file for the MCCD method - -[INPUTS] -INPUT_DIR = . -PREPROCESSED_OUTPUT_DIR = ./output -OUTPUT_DIR = ./output -INPUT_REGEX_FILE_PATTERN = star_split_ratio_80-*-*.fits -INPUT_SEPARATOR = - -MIN_N_STARS = 20 -OUTLIER_STD_MAX = 100. -USE_SNR_WEIGHTS = False - -[INSTANCE] -N_COMP_LOC = 2 -D_COMP_GLOB = 6 -KSIG_LOC = 0.0000 -KSIG_GLOB = 0.000 -FILTER_PATH = None - -[FIT] -LOC_MODEL = poly -PSF_SIZE = 6.2 -PSF_SIZE_TYPE = R2 -N_EIGENVECTS = 5 -N_ITER_RCA = 1 -N_ITER_GLOB = 1 -N_ITER_LOC = 1 -NB_SUBITER_S_LOC = 300 -NB_SUBITER_A_LOC = 400 -NB_SUBITER_S_GLOB = 100 -NB_SUBITER_A_GLOB = 200 - -[VALIDATION] -VAL_DATA_INPUT_DIR = . -VAL_PREPROCESSED_OUTPUT_DIR = ./output -VAL_MODEL_INPUT_DIR = ./output -VAL_OUTPUT_DIR = ./output -VAL_REGEX_FILE_PATTERN = star_split_ratio_20-*-*.fits -VAL_SEPARATOR = - -APPLY_DEGRADATION = True -MCCD_DEBUG = False -GLOBAL_POL_INTERP = False - -# Parameter description: -# -# -# [INPUTS] -# INPUT_DIR : (Required) Must be a valid directory containing the input -# MCCD files. -# PREPROCESSED_OUTPUT_DIR : (Required) Must be a valid directory to write the -# preprocessed input files. -# OUTPUT_DIR : (Required) Must be a valid directory to write the output files. -# The constructed models will be saved. -# INPUT_REGEX_FILE_PATTERN : File pattern of the input files to use. It should -# follow regex (regular expression) standards. -# INPUT_SEPARATOR : Separator of the different fields in the filename, -# ie sexcat[SEP]catalog_id[SEP]CCD_id.fits -# MIN_N_STARS : Minimum number of stars to keep a CCD for the training. -# OUTLIER_STD_MAX : Maximum standard deviation used for the outlier rejection. -# Should not be too low as a hihg quantity of low quality -# stars will be rejected. ie 9 is a conservative rejection. -# USE_SNR_WEIGHTS : Boolean to determine if the SNR weighting strategy will -# be used. -# For now, it needs the SNR estimations from SExtractor. -# -# -# [INSTANCE] -# N_COMP_LOC : Number of components of the Local model. If LOC_MODEL is poly, -# will be the max degree D of the polynomial. -# D_COMP_GLOB : Max degree of the global polynomial model. -# KSIG_LOC : Denoising parameter of the local model. -# ie 1 is a normal denoising, 3 is a hard denoising. -# KSIG_GLOB : Denoising parameter of the global model. -# ie 1 is a normal denoising, 3 is a hard denoising. -# FILTER_PATH : Path for predefined filters. -# -# -# [FIT] -# LOC_MODEL : Defines the type of local model to use, it can be: 'rca', -# 'poly' or 'hybrid'. -# When the poly model is used, N_COMP_LOC should be used -# as the D_LOC (max degree of the poly model) -# PSF_SIZE : First guess of the PSF size. A size estimation is done anyways. -# PSF_SIZE_TYPE : Type of the size information. It can be: fwhm, R2, sigma -# N_EIGENVECTS : Number of eigenvectors to keep for the graph constraint -# construction. -# N_ITER_RCA : Number of global epochs in the algorithm. Alternation between -# global and local estimations. -# N_ITER_GLOB : Number of epochs for each global optimization. Alternations -# between A_GLOB and S_GLOB. -# N_ITER_LOC : Number of epochs for each local optimization. Alternations -# between the different A_LOC and S_LOC. -# NB_SUBITER_S_LOC : Iterations for the optimization algorithm over S_LOC. -# NB_SUBITER_A_LOC : Iterations for the optimization algorithm over A_LOC. -# NB_SUBITER_S_GLOB : Iterations for the optimization algorithm over S_GLOB. -# NB_SUBITER_A_GLOB : Iterations for the optimization algorithm over A_GLOB. -# -# -# [VALIDATION] -# VAL_DATA_INPUT_DIR : (Required) Must be a valid directory which contains the -# validation input data (test dataset). -# VAL_PREPROCESSED_OUTPUT_DIR : (Required) Must be a valid directory where the -# preprocessed input file will be saved. -# VAL_MODEL_INPUT_DIR : (Required) Must be a valid directory which contains the -# saved trained models. -# VAL_OUTPUT_DIR : (Required) Must be a valid directory where to save the -# validation outputs, test PSFs and interpolated PSFs. -# VAL_REGEX_FILE_PATTERN : Same as INPUT_REGEX_FILE_PATTERN but for validation. -# VAL_SEPARATOR : Same as INPUT_SEPARATOR but for validation. -# APPLY_DEGRADATION : Whether the PSF models should be degraded -# (sampling/shifts/flux) to match stars; use True if you -# plan on making pixel-based comparisons (residuals etc.). -# MCCD_DEBUG : Debug mode. Returns the local and global contributions. -# GLOBAL_POL_INTERP : Uses polynomial interpolation for the global model -# instead of RBF kernel interpolation. -# diff --git a/example/cfis/config_MCCD_norm_optim.ini b/example/cfis/config_MCCD_norm_optim.ini deleted file mode 100644 index bcd30b6a3..000000000 --- a/example/cfis/config_MCCD_norm_optim.ini +++ /dev/null @@ -1,117 +0,0 @@ -# Configuration file for the MCCD method -# norm_optim - -[INPUTS] -INPUT_DIR = . -PREPROCESSED_OUTPUT_DIR = ./output -OUTPUT_DIR = ./output -INPUT_REGEX_FILE_PATTERN = star_split_ratio_80-*-*.fits -INPUT_SEPARATOR = - -MIN_N_STARS = 20 -OUTLIER_STD_MAX = 100. -USE_SNR_WEIGHTS = False - -[INSTANCE] -N_COMP_LOC = 4 -D_COMP_GLOB = 6 -KSIG_LOC = 0.0000 -KSIG_GLOB = 0.000 -FILTER_PATH = None - -[FIT] -LOC_MODEL = hybrid -PSF_SIZE = 6.2 -PSF_SIZE_TYPE = R2 -N_EIGENVECTS = 5 -N_ITER_RCA = 1 -N_ITER_GLOB = 2 -N_ITER_LOC = 2 -NB_SUBITER_S_LOC = 300 -NB_SUBITER_A_LOC = 400 -NB_SUBITER_S_GLOB = 100 -NB_SUBITER_A_GLOB = 200 - -[VALIDATION] -VAL_DATA_INPUT_DIR = . -VAL_PREPROCESSED_OUTPUT_DIR = ./output -VAL_MODEL_INPUT_DIR = ./output -VAL_OUTPUT_DIR = ./output -VAL_REGEX_FILE_PATTERN = star_split_ratio_20-*-*.fits -VAL_SEPARATOR = - -APPLY_DEGRADATION = True -MCCD_DEBUG = False -GLOBAL_POL_INTERP = False - -# Parameter description: -# -# -# [INPUTS] -# INPUT_DIR : (Required) Must be a valid directory containing the input -# MCCD files. -# PREPROCESSED_OUTPUT_DIR : (Required) Must be a valid directory to write the -# preprocessed input files. -# OUTPUT_DIR : (Required) Must be a valid directory to write the output files. -# The constructed models will be saved. -# INPUT_REGEX_FILE_PATTERN : File pattern of the input files to use. It should -# follow regex (regular expression) standards. -# INPUT_SEPARATOR : Separator of the different fields in the filename, -# ie sexcat[SEP]catalog_id[SEP]CCD_id.fits -# MIN_N_STARS : Minimum number of stars to keep a CCD for the training. -# OUTLIER_STD_MAX : Maximum standard deviation used for the outlier rejection. -# Should not be too low as a hihg quantity of low quality -# stars will be rejected. ie 9 is a conservative rejection. -# USE_SNR_WEIGHTS : Boolean to determine if the SNR weighting strategy will -# be used. -# For now, it needs the SNR estimations from SExtractor. -# -# -# [INSTANCE] -# N_COMP_LOC : Number of components of the Local model. If LOC_MODEL is poly, -# will be the max degree D of the polynomial. -# D_COMP_GLOB : Max degree of the global polynomial model. -# KSIG_LOC : Denoising parameter of the local model. -# ie 1 is a normal denoising, 3 is a hard denoising. -# KSIG_GLOB : Denoising parameter of the global model. -# ie 1 is a normal denoising, 3 is a hard denoising. -# FILTER_PATH : Path for predefined filters. -# -# -# [FIT] -# LOC_MODEL : Defines the type of local model to use, it can be: 'rca', -# 'poly' or 'hybrid'. -# When the poly model is used, N_COMP_LOC should be used -# as the D_LOC (max degree of the poly model) -# PSF_SIZE : First guess of the PSF size. A size estimation is done anyways. -# PSF_SIZE_TYPE : Type of the size information. It can be: fwhm, R2, sigma -# N_EIGENVECTS : Number of eigenvectors to keep for the graph constraint -# construction. -# N_ITER_RCA : Number of global epochs in the algorithm. Alternation between -# global and local estimations. -# N_ITER_GLOB : Number of epochs for each global optimization. Alternations -# between A_GLOB and S_GLOB. -# N_ITER_LOC : Number of epochs for each local optimization. Alternations -# between the different A_LOC and S_LOC. -# NB_SUBITER_S_LOC : Iterations for the optimization algorithm over S_LOC. -# NB_SUBITER_A_LOC : Iterations for the optimization algorithm over A_LOC. -# NB_SUBITER_S_GLOB : Iterations for the optimization algorithm over S_GLOB. -# NB_SUBITER_A_GLOB : Iterations for the optimization algorithm over A_GLOB. -# -# -# [VALIDATION] -# VAL_DATA_INPUT_DIR : (Required) Must be a valid directory which contains the -# validation input data (test dataset). -# VAL_PREPROCESSED_OUTPUT_DIR : (Required) Must be a valid directory where the -# preprocessed input file will be saved. -# VAL_MODEL_INPUT_DIR : (Required) Must be a valid directory which contains the -# saved trained models. -# VAL_OUTPUT_DIR : (Required) Must be a valid directory where to save the -# validation outputs, test PSFs and interpolated PSFs. -# VAL_REGEX_FILE_PATTERN : Same as INPUT_REGEX_FILE_PATTERN but for validation. -# VAL_SEPARATOR : Same as INPUT_SEPARATOR but for validation. -# APPLY_DEGRADATION : Whether the PSF models should be degraded -# (sampling/shifts/flux) to match stars; use True if you -# plan on making pixel-based comparisons (residuals etc.). -# MCCD_DEBUG : Debug mode. Returns the local and global contributions. -# GLOBAL_POL_INTERP : Uses polynomial interpolation for the global model -# instead of RBF kernel interpolation. -# diff --git a/example/cfis/config_tile_Ma.ini b/example/cfis/config_MaMa.ini similarity index 69% rename from example/cfis/config_tile_Ma.ini rename to example/cfis/config_MaMa.ini index ce064e467..b73fb84ce 100644 --- a/example/cfis/config_tile_Ma.ini +++ b/example/cfis/config_MaMa.ini @@ -1,4 +1,4 @@ -# ShapePipe configuration file for tiles. Mask. +# ShapePipe configuration file for masking of tiles and exposures ## Default ShapePipe options @@ -8,7 +8,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Ma +RUN_NAME = run_sp_MaMa # Add date and time to RUN_NAME, optional, default: False ; RUN_DATETIME = False @@ -18,7 +18,7 @@ RUN_NAME = run_sp_tile_Ma [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner +MODULE = mask_runner, mask_runner # Parallel processing mode, SMP or MPI MODE = SMP @@ -51,10 +51,12 @@ TIMEOUT = 96:00:00 ## Module options -[MASK_RUNNER] + +### Mask tiles +[MASK_RUNNER_RUN_1] # Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner, last:uncompress_fits_runner +INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner # NUMBERING_SCHEME (optional) string with numbering pattern for input files NUMBERING_SCHEME = -000-000 @@ -78,3 +80,27 @@ USE_EXT_STAR = False # File name suffix for the output flag files (optional) SUFFIX = pipeline + +### Mask exposures +[MASK_RUNNER_RUN_2] + +# Parent module +INPUT_DIR = last:split_exp_runner + +# Update numbering convention, accounting for HDU number of +# single-exposure single-HDU files +NUMBERING_SCHEME = -0000000-0 + +# Path of mask config file +MASK_CONFIG_PATH = $SP_CONFIG/config.mask + +# External mask file flag, use if True, otherwise ignore +USE_EXT_FLAG = True + +# External star catalogue flag, use external cat if True, +# obtain from online catalogue if False +USE_EXT_STAR = False + +# File name suffix for the output flag files (optional) +SUFFIX = pipeline + diff --git a/example/cfis/config_exp_Ma.ini b/example/cfis/config_exp_Ma.ini deleted file mode 100644 index 01846cfe2..000000000 --- a/example/cfis/config_exp_Ma.ini +++ /dev/null @@ -1,76 +0,0 @@ -# ShapePipe configuration file for single-exposures. -# Mask images - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_Ma - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[MASK_RUNNER] - -# Parent module -INPUT_MODULE = split_exp_runner - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -SUFFIX = pipeline diff --git a/example/cfis/config_exp_SpMh.ini b/example/cfis/config_exp_SpMh.ini index b2801ffdc..c5057ac1c 100644 --- a/example/cfis/config_exp_SpMh.ini +++ b/example/cfis/config_exp_SpMh.ini @@ -55,7 +55,7 @@ TIMEOUT = 96:00:00 [SPLIT_EXP_RUNNER] -INPUT_DIR = $SP_RUN/output/run_sp_Gie:get_images_runner +INPUT_DIR = last:get_images_runner_run_2 # Matches compressed single-exposure files FILE_EXT = .fitsfz, .fitsfz, .fitsfz diff --git a/example/cfis/config_make_cat_mccd.ini b/example/cfis/config_make_cat_mccd.ini index 0107b043e..c8173eabd 100644 --- a/example/cfis/config_make_cat_mccd.ini +++ b/example/cfis/config_make_cat_mccd.ini @@ -34,7 +34,7 @@ LOG_NAME = log_sp RUN_LOG_NAME = log_run_sp # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = last:sextractor_runner, last:spread_model_runner, last:mccd_interp_runner, last:merge_sep_cats_runner +INPUT_DIR = last:sextractor_runner_run_2, last:spread_model_runner, last:mccd_interp_runner, last:merge_sep_cats_runner # Output directory OUTPUT_DIR = ./output diff --git a/example/cfis/config_mask.ini b/example/cfis/config_mask.ini deleted file mode 100644 index 33e0edc51..000000000 --- a/example/cfis/config_mask.ini +++ /dev/null @@ -1,83 +0,0 @@ -# ShapePipe configuration file: mask -# mask bright stars, diffraction spikes, Messier objects, -# borders, and other artifacts - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -; RUN_NAME = my_shapepipe_run - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -; LOG_NAME = my_shapepipe_log - -# Runner log file name, optional, default: shapepipe_runs -; RUN_LOG_NAME = my_shapepipe_run_log - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:split_exp_runner -#,$SP_RUN/output_star_cat - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 01:00:00 - - -## Module options -[MASK_RUNNER] - -# Input file pattern(s), list of strings with length mathcing number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = image,weight,flag -#,star_cat - -# FILE_EXT (optional) list of string extensions to identify input files -#FILE_EXT = .fits,.fits,.fits,.cat -FILE_EXT = .fits,.fits,.fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -0000000-0 - - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -#USE_EXT_STAR = True -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -SUFFIX = pipeline diff --git a/example/cfis/config_tile_MaSxMiViSmVi.ini b/example/cfis/config_tile_MaSxMiViSmVi.ini deleted file mode 100644 index 289e51c6a..000000000 --- a/example/cfis/config_tile_MaSxMiViSmVi.ini +++ /dev/null @@ -1,269 +0,0 @@ -# ShapePipe configuration file for tiles up to ngmix, using MCCD for the PSF - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_MaSxMiViSmVi - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner, sextractor_runner, mccd_interp_runner, - vignetmaker_runner, spread_model_runner, - vignetmaker_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner, last:uncompress_fits_runner - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options -[MASK_RUNNER] - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_image, CFIS_weight - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -SUFFIX = pipeline - - -[SEXTRACTOR_RUNNER] - -INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner, last:uncompress_fits_image_runner, last:mask_runner - -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag - -FILE_EXT = .fits, .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = sex - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -#CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 - - -[MCCD_INTERP_RUNNER] - -INPUT_MODULE = sextractor_runner - -FILE_PATTERN = sexcat_sexcat - -FILE_EXT = .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = MULTI-EPOCH - -# Column names of position parameters -POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD - -# If True, measure and store ellipticity of the PSF -GET_SHAPES = True - -PSF_MODEL_DIR = input_psf_mccd - -PSF_MODEL_PATTERN = fitted_model - -PSF_MODEL_SEPARATOR = - - -# Multi-epoch mode: Path to file with single-exposure WCS header information -ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - - -[VIGNETMAKER_RUNNER_RUN_1] - -# Create vignets for tiles weights - -INPUT_DIR = last:sextractor_runner, last:uncompress_fits_image_runner - -FILE_PATTERN = sexcat_sexcat, CFIS_weight - -FILE_EXT = .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = CLASSIC - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = PIX -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is _vignet.fits -SUFFIX = weight - - -[SPREAD_MODEL_RUNNER] - -INPUT_MODULE = sextractor_runner, mccd_interp_runner, vignetmaker_runner_run_1 - -FILE_PATTERN = sexcat_sexcat, galaxy_psf, weight_vignet - -FILE_EXT = .fits, .sqlite, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Pixel scale in arcsec -PIXEL_SCALE = 0.186 - -# Output mode: -# new: create a new catalog with: [number, mag, sm, sm_err] -# add: create a copy of the input SExtractor with the column sm and sm_err -OUTPUT_MODE = new - - -[VIGNETMAKER_RUNNER_RUN_2] - -# Create multi-epoch vignets for tiles corresponding to -# positions on single-exposures - -INPUT_MODULE = sextractor_runner - -FILE_PATTERN = sexcat_sexcat - -FILE_EXT = .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = MULTI-EPOCH - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = SPHE -POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is vignet.fits -SUFFIX = - -# Additional parameters for path and file pattern corresponding to single-exposure -# run outputs -ME_IMAGE_DIR = input_split_exp, input_split_exp, input_split_exp, input_sextractor -ME_IMAGE_PATTERN = flag, image, weight, sexcat_background -ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite diff --git a/example/cfis/config_tile_MaSxPiViSmVi.ini b/example/cfis/config_tile_MaSxPiViSmVi.ini index 626b21250..5770cad9d 100644 --- a/example/cfis/config_tile_MaSxPiViSmVi.ini +++ b/example/cfis/config_tile_MaSxPiViSmVi.ini @@ -81,7 +81,7 @@ SUFFIX = pipeline [SEXTRACTOR_RUNNER] -INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner, last:uncompress_fits_image_runner, last:mask_runner +INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner, last:uncompress_fits_runner, last:mask_runner FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag @@ -175,7 +175,8 @@ CHI2_THRESH = 2 # Multi-epoch mode parameters # Input psf file directory -ME_DOT_PSF_DIR = input_psfex +#ME_DOT_PSF_DIR = input_psfex +ME_DOT_PSF_DIR = psfex_runner # Input psf file pattern ME_DOT_PSF_PATTERN = star_split_ratio_80 @@ -188,7 +189,7 @@ ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite # Create vignets for tiles weights -INPUT_DIR = last:sextractor_runner, last:uncompress_fits_image_runner +INPUT_DIR = last:sextractor_runner, last:uncompress_fits_runner FILE_PATTERN = sexcat_sexcat, CFIS_weight @@ -272,6 +273,7 @@ SUFFIX = # Additional parameters for path and file pattern corresponding to single-exposure # run outputs -ME_IMAGE_DIR = input_split_exp, input_split_exp, input_split_exp, input_sextractor +#ME_IMAGE_DIR = input_split_exp, input_split_exp, input_split_exp, input_sextractor +IMAGE_DIR = split_exp_runner, splot_exp_runner, split_exp_runner, sextractor_runner ME_IMAGE_PATTERN = flag, image, weight, sexcat_background ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite diff --git a/example/cfis/config_tile_SxMiViSmVi.ini b/example/cfis/config_tile_MiViSmVi.ini similarity index 66% rename from example/cfis/config_tile_SxMiViSmVi.ini rename to example/cfis/config_tile_MiViSmVi.ini index 49cd81f21..f42c0ac61 100644 --- a/example/cfis/config_tile_SxMiViSmVi.ini +++ b/example/cfis/config_tile_MiViSmVi.ini @@ -9,7 +9,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_SxMiViSmVi +RUN_NAME = run_sp_MiViSmVi # Add date and time to RUN_NAME, optional, default: False ; RUN_DATETIME = False @@ -19,7 +19,7 @@ RUN_NAME = run_sp_SxMiViSmVi [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, mccd_interp_runner, +MODULE = mccd_interp_runner, vignetmaker_runner, spread_model_runner, vignetmaker_runner @@ -47,7 +47,7 @@ OUTPUT_DIR = $SP_RUN/output [JOB] # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 +SMP_BATCH_SIZE = 12 # Timeout value (optional), default is None, i.e. no timeout limit applied TIMEOUT = 96:00:00 @@ -55,73 +55,9 @@ TIMEOUT = 96:00:00 ## Module options -[SEXTRACTOR_RUNNER] - -INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner, last:uncompress_fits_runner, $SP_RUN/output/run_sp_tile_Ma:mask_runner - -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag - -FILE_EXT = .fits, .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = sex - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -#CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 - - [MCCD_INTERP_RUNNER] -INPUT_MODULE = sextractor_runner +INPUT_DIR = last:sextractor_runner_run_2 FILE_PATTERN = sexcat_sexcat @@ -142,7 +78,7 @@ POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD # If True, measure and store ellipticity of the PSF GET_SHAPES = True -PSF_MODEL_DIR = input_psf_mccd +PSF_MODEL_DIR = mccd_fit_val_runner PSF_MODEL_PATTERN = fitted_model @@ -156,7 +92,7 @@ ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite # Create vignets for tiles weights -INPUT_DIR = last:sextractor_runner, last:uncompress_fits_runner +INPUT_DIR = last:sextractor_runner_run_2, last:uncompress_fits_runner FILE_PATTERN = sexcat_sexcat, CFIS_weight @@ -187,7 +123,7 @@ SUFFIX = weight [SPREAD_MODEL_RUNNER] -INPUT_MODULE = sextractor_runner, mccd_interp_runner, vignetmaker_runner +INPUT_DIR = last:sextractor_runner_run_2, last:mccd_interp_runner, last:vignetmaker_runner_run_1 FILE_PATTERN = sexcat_sexcat, galaxy_psf, weight_vignet @@ -210,7 +146,7 @@ OUTPUT_MODE = new # Create multi-epoch vignets for tiles corresponding to # positions on single-exposures -INPUT_MODULE = sextractor_runner +INPUT_DIR = last:sextractor_runner_run_2 FILE_PATTERN = sexcat_sexcat @@ -240,6 +176,6 @@ SUFFIX = # Additional parameters for path and file pattern corresponding to single-exposure # run outputs -ME_IMAGE_DIR = input_split_exp, input_split_exp, input_split_exp, input_sextractor +ME_IMAGE_DIR = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner_run_1 ME_IMAGE_PATTERN = flag, image, weight, sexcat_background ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite diff --git a/example/cfis/config_tile_Ng_template.ini b/example/cfis/config_tile_Ng_template.ini index b600d2288..e6f56d332 100644 --- a/example/cfis/config_tile_Ng_template.ini +++ b/example/cfis/config_tile_Ng_template.ini @@ -35,7 +35,7 @@ LOG_NAME = log_sp RUN_LOG_NAME = log_run_sp # Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner +INPUT_DIR = last:get_images_runner_run_1 FILE_PATTERN = sexcat_sexcat, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet @@ -63,7 +63,7 @@ TIMEOUT = 96:00:00 # NUMBERING_SCHEME (optional) string with numbering pattern for input files NUMBERING_SCHEME = -000-000 -INPUT_DIR = last:sextractor_runner,last:X_interp_runner,last:vignetmaker_runner_run_2 +INPUT_DIR = last:sextractor_runner_run_2,last:X_interp_runner,last:vignetmaker_runner_run_2 # Multi-epoch mode: Path to file with single-exposure WCS header information LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite @@ -81,7 +81,7 @@ ID_OBJ_MAX = X # Moment-based (KSB) shapes with galsim [GALSIM_SHAPES_V2_RUNNER] -INPUT_DIR = last:sextractor_runner, last:vignetmaker_runner_run_1, last:X_interp_runner,last:vignetmaker_runner_run_2 +INPUT_DIR = last:sextractor_runner_run_2, last:vignetmaker_runner_run_1, last:X_interp_runner,last:vignetmaker_runner_run_2 FILE_PATTERN = sexcat_sexcat, weight_vignet, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet diff --git a/example/cfis/config_tile_SxPiViSmVi.ini b/example/cfis/config_tile_PiViSmVi.ini similarity index 65% rename from example/cfis/config_tile_SxPiViSmVi.ini rename to example/cfis/config_tile_PiViSmVi.ini index d11e2acff..96946a171 100644 --- a/example/cfis/config_tile_SxPiViSmVi.ini +++ b/example/cfis/config_tile_PiViSmVi.ini @@ -9,7 +9,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_SxPsViSmVi +RUN_NAME = run_sp_tile_PsViSmVi # Add date and time to RUN_NAME, optional, default: False ; RUN_DATETIME = False @@ -19,8 +19,9 @@ RUN_NAME = run_sp_SxPsViSmVi [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, psfex_interp_runner, - vignetmaker_runner, spread_model_runner, +#MODULE = psfex_interp_runner, + +MODULE = psfex_interp_runner, vignetmaker_runner, spread_model_runner, vignetmaker_runner # Parallel processing mode, SMP or MPI @@ -47,7 +48,7 @@ OUTPUT_DIR = $SP_RUN/output [JOB] # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 +SMP_BATCH_SIZE = 16 # Timeout value (optional), default is None, i.e. no timeout limit applied TIMEOUT = 96:00:00 @@ -55,73 +56,9 @@ TIMEOUT = 96:00:00 ## Module options -[SEXTRACTOR_RUNNER] - -INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner, last:uncompress_fits_runner, $SP_RUN/output/run_sp_tile_Ma:mask_runner - -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag - -FILE_EXT = .fits, .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = sex - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -#CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 - - [PSFEX_INTERP_RUNNER] -INPUT_MODULE = sextractor_runner +INPUT_DIR = last:sextractor_runner_run_1 FILE_PATTERN = sexcat_sexcat @@ -150,8 +87,7 @@ CHI2_THRESH = 2 # Multi-epoch mode parameters -# Input psf file directory -ME_DOT_PSF_DIR = $SP_RUN/input_psfex +ME_DOT_PSF_DIR = psfex_runner # Input psf file pattern ME_DOT_PSF_PATTERN = star_split_ratio_80 @@ -160,11 +96,10 @@ ME_DOT_PSF_PATTERN = star_split_ratio_80 ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite -[VIGNETMAKER_RUNNER_RUN_1] - # Create vignets for tiles weights +[VIGNETMAKER_RUNNER_RUN_1] -INPUT_DIR = last:sextractor_runner, last:uncompress_fits_runner +INPUT_DIR = last:sextractor_runner_run_1, last:uncompress_fits_runner FILE_PATTERN = sexcat_sexcat, CFIS_weight @@ -195,7 +130,7 @@ SUFFIX = weight [SPREAD_MODEL_RUNNER] -INPUT_MODULE = sextractor_runner, psfex_interp_runner, vignetmaker_runner +INPUT_DIR = last:sextractor_runner_run_1, psfex_interp_runner, vignetmaker_runner FILE_PATTERN = sexcat_sexcat, galaxy_psf, weight_vignet @@ -218,7 +153,7 @@ OUTPUT_MODE = new # Create multi-epoch vignets for tiles corresponding to # positions on single-exposures -INPUT_MODULE = sextractor_runner +INPUT_DIR = last:sextractor_runner_run_1 FILE_PATTERN = sexcat_sexcat @@ -248,6 +183,6 @@ SUFFIX = # Additional parameters for path and file pattern corresponding to single-exposure # run outputs -ME_IMAGE_DIR = input_split_exp, input_split_exp, input_split_exp, input_sextractor +ME_IMAGE_DIR = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner_run_2 ME_IMAGE_PATTERN = flag, image, weight, sexcat_background ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite diff --git a/example/cfis/config_exp_mccd.ini b/example/cfis/config_tile_Sx_exp_mccd.ini similarity index 71% rename from example/cfis/config_exp_mccd.ini rename to example/cfis/config_tile_Sx_exp_mccd.ini index 4dda513d3..1b4865ae1 100644 --- a/example/cfis/config_exp_mccd.ini +++ b/example/cfis/config_tile_Sx_exp_mccd.ini @@ -9,7 +9,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_SxSePsf +RUN_NAME = run_sp_SxSePsfSx # Add date and time to RUN_NAME, optional, default: True ; RUN_DATETIME = False @@ -19,9 +19,12 @@ RUN_NAME = run_sp_exp_SxSePsf [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, setools_runner, - mccd_preprocessing_runner, mccd_fit_val_runner, - merge_starcat_runner, mccd_plots_runner +MODULE = sextractor_runner, merge_starcat_runner, mccd_plots_runner, sextractor_runner + +#MODULE = sextractor_runner, setools_runner, +#mccd_preprocessing_runner, mccd_fit_val_runner, +#merge_starcat_runner, mccd_plots_runner, +#sextractor_runner # Run mode, SMP or MPI MODE = SMP @@ -55,15 +58,12 @@ TIMEOUT = 96:00:00 ## Module options -[SEXTRACTOR_RUNNER] +[SEXTRACTOR_RUNNER_RUN_1] -# Somehow this works but not -# - omitting -# - $SP_RUN/output -#INPUT_DIR = . +INPUT_DIR = last:split_exp_runner, last:mask_runner_run_2 # Input from two modules -INPUT_MODULE = split_exp_runner, mask_runner +INPUT_MODULE = split_exp_runner, mask_runner_run_2 # Read pipeline flag files created by mask module FILE_PATTERN = image, weight, pipeline_flag @@ -174,6 +174,8 @@ NUMBERING_SCHEME = -0000000 [MERGE_STARCAT_RUNNER] +INPUT_DIR = last:mccd_fit_val_runner + # Path to MCCD config file CONFIG_PATH = $SP_CONFIG/config_MCCD.ini @@ -200,6 +202,7 @@ VERBOSE = False NUMBERING_SCHEME = -0000000 PSF = mccd + PLOT_MEANSHAPES = True # X_GRID, Y_GRID: correspond to the number of bins in each direction of each @@ -213,7 +216,7 @@ PLOT_HISTOGRAMS = True # before drawing the plots. REMOVE_OUTLIERS = False -PLOT_RHO_STATS = True +PLOT_RHO_STATS = False # RHO_STATS_STYLE: can be 'HSC' or 'DES' RHO_STATS_STYLE = HSC @@ -245,3 +248,68 @@ PSF_MODEL_SEPARATOR = - # For multi-epoch purposes ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite + + +## Detection on tile +[SEXTRACTOR_RUNNER_RUN_2] + +INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, last:mask_runner_run_1 + +FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag + +FILE_EXT = .fits, .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# SExtractor executable path +EXEC_PATH = sex + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_tile.sex +DOT_PARAM_FILE = $SP_CONFIG/default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +ZP_FROM_HEADER = False + +BKG_FROM_HEADER = False + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, +# MINIBACK_RMS, -BACKGROUND, #FILTERED, +# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +#CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# Multi-epoch mode: Path to file with single-exposure WCS header information +LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = XWIN_WORLD,YWIN_WORLD + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_tile_Sx_exp_psfex.ini similarity index 69% rename from example/cfis/config_exp_psfex.ini rename to example/cfis/config_tile_Sx_exp_psfex.ini index 916ba53e9..110060f40 100644 --- a/example/cfis/config_exp_psfex.ini +++ b/example/cfis/config_tile_Sx_exp_psfex.ini @@ -9,7 +9,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_SxSePsf +RUN_NAME = run_sp_tile_Sx_exp_SxSePsf # Add date and time to RUN_NAME, optional, default: True ; RUN_DATETIME = False @@ -19,7 +19,7 @@ RUN_NAME = run_sp_exp_SxSePsf [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, setools_runner, +MODULE = sextractor_runner, sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner # Run mode, SMP or MPI @@ -54,7 +54,73 @@ TIMEOUT = 96:00:00 ## Module options -[SEXTRACTOR_RUNNER] +[SEXTRACTOR_RUNNER_RUN_1] + +INPUT_MODULE = get_images_runner_run_1, uncompress_fits_runner, mask_runner_run_1 + +INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, last:mask_runner_run_1 + +FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag + +FILE_EXT = .fits, .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# SExtractor executable path +EXEC_PATH = sex + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_tile.sex +DOT_PARAM_FILE = $SP_CONFIG/default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +ZP_FROM_HEADER = False + +BKG_FROM_HEADER = False + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, +# MINIBACK_RMS, -BACKGROUND, #FILTERED, +# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +#CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# Multi-epoch mode: Path to file with single-exposure WCS header information +LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = XWIN_WORLD,YWIN_WORLD + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 + + +[SEXTRACTOR_RUNNER_RUN_2] # Somehow this works but not # - omitting @@ -128,7 +194,7 @@ MAKE_POST_PROCESS = FALSE [SETOOLS_RUNNER] -INPUT_MODULE = sextractor_runner +INPUT_MODULE = sextractor_runner_run_2 # Note: Make sure this doe not match the SExtractor background images # (sexcat_background*) diff --git a/example/cfis/config_unfz_w.ini b/example/cfis/config_tile_Uz.ini similarity index 93% rename from example/cfis/config_unfz_w.ini rename to example/cfis/config_tile_Uz.ini index df2d23f37..aef899d0f 100644 --- a/example/cfis/config_unfz_w.ini +++ b/example/cfis/config_tile_Uz.ini @@ -34,7 +34,7 @@ LOG_NAME = log_sp RUN_LOG_NAME = log_run_sp # Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner +INPUT_DIR = . # Output directory OUTPUT_DIR = $SP_RUN/output @@ -53,6 +53,9 @@ TIMEOUT = 96:00:00 ## Module options [UNCOMPRESS_FITS_RUNNER] +INPUT_DIR = last:get_images_runner_run_1 +#INPUT_MODULE = get_images_runner_run_1 + FILE_PATTERN = CFIS_weight FILE_EXT = .fitsfz diff --git a/scripts/sh/canfar_sp.bash b/scripts/sh/canfar_sp.bash deleted file mode 100644 index 9190642e2..000000000 --- a/scripts/sh/canfar_sp.bash +++ /dev/null @@ -1,334 +0,0 @@ -#!/usr/bin/env bash - -# Name: canfar_sp.bash -# Description: Process one or more tiles with all -# contributing exposures on canfar. -# This is the job submission script for -# the canfar batch system. Can also be -# called in interactive mode on a virtual -# machine. -# Author: Martin Kilbinger -# Date: 03/2020 -# Package: ShapePipe - - -### Set-up ### - -## Variables - -# Tile numbers - -if [ $# == 0 ]; then - echo "Usages:" - echo " bash $(basename "$0") TILE_ID_1 [TILE_ID_2 [...]]" - echo " TILE_ID = xxx.yyy" - echo " Examples:" - echo " $(basename "$0") 244.252" - echo " $(basename "$0") 244.252 239.293" - echo " . $(basename "$0") -e" - echo " Assign environment variables" - exit 1 -fi - -# Copy command line arguments -TILE_ARR=($@) - -# For tar archives. Should be unique to each job -export ID=`echo ${TILE_ARR[@]} | tr ' ' '_'` - - -## Paths - -# VM home, required for canfar run. -# On other machines set to $HOME -export VM_HOME=/home/ubuntu -if [ ! -d "$VM_HOME" ]; then - export VM_HOME=$HOME -fi - -# Results upload subdirectory on vos -RESULTS=results - -## Path variables used in shapepipe config files - -# Run path and location of input image directories -export SP_RUN=`pwd` - -# Config file path -export SP_CONFIG=$SP_RUN/cfis - -## Other variables - -# Input tile numbers ASCII file -export TILE_NUMBERS_PATH=tile_numbers.txt - -# Output -OUTPUT=$SP_RUN/output - -# For tar archives -output_rel=`realpath --relative-to=. $OUTPUT` - -# Stop on error -STOP=1 - -# Verbose mode (1: verbose, 0: quiet) -VERBOSE=1 - -if [ $VERBOSE == 1 ]; then - vflag="-v" -else - vflag="" -fi - -# VCP options -export CERTFILE=$VM_HOME/.ssl/cadcproxy.pem -export VCP="vcp --certfile=$CERTFILE" - - -## Functions - -# Print string, executes command, and prints return value. -function command () { - cmd=$1 - str=$2 - verbose=$3 - - #RED='\033[0;31m' - #GREEN='\033[0;32m' - #NC='\033[0m' # No Color - # Color escape characters show up in log files - RED='' - GREEN='' - NC='' - - - if [ $# == 3 ]; then - if [ $verbose == 1 ]; then - echo "$str: running '$cmd'" - fi - $cmd - else - if [ $verbose == 1 ]; then - echo "$str: running '$cmd $4 \"$5 $6\"'" - fi - $cmd $4 "$5 $6" - fi - res=$? - - if [ $verbose == 1 ]; then - if [ $res == 0 ]; then - echo -e "${GREEN}success, return value = $res${NC}" - else - echo -e "${RED}error, return value = $res${NC}" - if [ $STOP == 1 ]; then - echo "${RED}exiting $(basename "$0"), error in command '$cmd'${NC}" - exit $res - else - echo "${RED}continuing $(basename "$0"), error in command '$cmd'${NC}" - fi - fi - fi - - return $res -} - -# Run shapepipe command. If error occurs, upload sp log files before stopping script. -command_sp() { - cmd=$1 - str=$2 - verbose=$3 - id=$4 - - STOP=0 - command "$1" "$2" "$3" - res=$? - if [ $res != 0 ]; then - upload_logs $id $verbose - echo "exiting '$(basename "$0"), '$cmd' returned $res, log files for id=$id uploaded" - exit $res - fi - -} - -# Tar and upload files to vos -function upload() { - base=$1 - shift - ID=$1 - shift - verbose=$1 - shift - upl=("$@") - - n_upl=(`ls -l ${upl[@]} | wc`) - if [ $n_upl == 0 ]; then - if [ $STOP == 1 ]; then - echo "Exiting script, no file found for '$base' tar ball" - exit 3 - fi - fi - tar czf ${base}_${ID}.tgz ${upl[@]} - command "$VCP ${base}_${ID}.tgz vos:cfis/cosmostat/kilbinger/$RESULTS" "Upload $base to $RESULTS, $n_upl files in tar ball" "$verbose" -} - -# Upload log files -function upload_logs() { - id=$1 - verbose=$2 - - upl="$output_rel/*/*/logs $output_rel/*/logs" - upload "logs" "$id" "$verbose" "${upl[@]}" -} - -# Print script variables -function print_env() { - echo "*** Setting ***" - echo "Data:" - echo " TILE_ARR=${TILE_ARR[@]}" - echo "Paths:" - echo " VM_HOME=$VM_HOME" - echo " SP_RUN=$SP_RUN" - echo " TILE_NUMBERS_PATH=$TILE_NUMBERS_PATH" - echo " OUTPUT=$OUTPUT" - echo " SP_CONFIG=$SP_CONFIG" - echo "Other variables:" - echo " VCP=$VCP" - echo " CERTFILE=$CERTFILE" - echo " STOP=$STOP" - echo " verbose=$VERBOSE" - echo "***" -} - - -### Start ### - -# Activate conda environment -echo "Activate conda 'shapepipe' environment" -source $VM_HOME/miniconda3/bin/activate shapepipe - -print_env - -if [ "$1" == "-e" ]; then - echo "Exiting" - return 0 -fi - -echo "Start" - -n_tile=${#TILE_ARR[@]} -echo "Processing $n_tile tile(s)" - -# Create input and output directories -echo "Create directories for processing" -mkdir -p $SP_RUN -cd $SP_RUN -mkdir -p $OUTPUT - -# Write tile numbers to ASCII input file -rm -rf $TILE_NUMBERS_PATH -for TILE in ${TILE_ARR[@]}; do - echo $TILE >> $TILE_NUMBERS_PATH -done - -# Download config files -$VCP vos:cfis/cosmostat/kilbinger/cfis . - -### Run pipeline - -## Prepare images - -# Get tiles -command_sp "shapepipe_run -c $SP_CONFIG/config_get_tiles.ini" "Run shapepipe (prepare: get tiles)" "$VERBOSE" "$ID" - -# Uncompress tile weights -command_sp "shapepipe_run -c $SP_CONFIG/config_unfz_w.ini" "Run shapepipe (prepare: uncompress tile weights)" "$VERBOSE" "$ID" - -# Find exposures -command_sp "shapepipe_run -c $SP_CONFIG/config_find_exp.ini" "Run shapepipe (prepare: find exposures)" "$VERBOSE" "$ID" - -# Get exposures -command_sp "shapepipe_run -c $SP_CONFIG/config_get_exp.ini" "Run shapepipe (prepare: get exposures)" "$VERBOSE" "$ID" - -## Exposures - -# Run all modules -command_sp "shapepipe_run -c $SP_CONFIG/config_exp.ini" "Run shapepipe (exp)" "$VERBOSE" "$ID" - - -# The following are very a bad hacks to get additional input files -input_psfex=`find . -name star_split_ratio_80-*.psf | head -n 1` -command_sp "ln -s `dirname $input_psfex` input_psfex" "Link psfex output" "$VERBOSE" "$ID" - -input_split_exp=`find output -name flag-*.fits | head -n 1` -command_sp "ln -s `dirname $input_split_exp` input_split_exp" "Link split_exp output" "$VERBOSE" "$ID" - -input_sextractor=`find . -name sexcat_sexcat-*.fits | head -n 1` -command_sp "ln -s `dirname $input_sextractor` input_sextractor" "Link sextractor output" "$VERBOSE" "$ID" - - -## Tiles - -# Everything up to shapes -command_sp "shapepipe_run -c $SP_CONFIG/config_tile_MaSxPsViSmVi.ini" "Run shapepipe (tile: up to ngmix)" "$VERBOSE" "$ID" - -# Shapes, run 8 parallel processes -for k in $(seq 1 8); do - command_sp "shapepipe_run -c $SP_CONFIG/config_tile_Ng${k}u.ini" "Run shapepipe (tile: ngmix $k)" "$VERBOSE" "$ID" & -done -wait - -# Merge separated shapes catalogues -command_sp "shapepipe_run -c $SP_CONFIG/config_merge_sep_cats.ini" "Run shapepipe (tile: merge sep cats)" "$VERBOSE" "$ID" - -# Create final shape catalogue by merging all tile information -command_sp "shapepipe_run -c $SP_CONFIG/config_make_cat.ini" "Run shapepipe (tile: create final cat)" "$VERBOSE" "$ID" - - -## Upload results - -# module and pipeline log files -upload_logs "$ID" "$VERBOSE" - -# psfex for diagnostics, validation with leakage -# psefxinterp for validation with residuals, rho stats -# SETools masks (selection), stats and plots -# pipeline_flags are the tile masks, for random cats -# Final shape catalog - -NAMES=( - "psfex" - "psfexinterp_exp" - "setools_mask" - "setools_stat" - "setools_plot" - "pipeline_flag" - "final_cat" - ) -DIRS=( - "*/psfex_runner/output" - "*/psfexinterp_runner/output" - "*/setools_runner/output/mask" - "*/setools_runner/output/stat" - "*/setools_runner/output/plot" - "*/mask_runner/output" - "*/make_catalog_runner/output" - ) -PATTERNS=( - "star_split_ratio_80-*" - "validation_psf*" - "*" - "*" - "*" - "pipeline_flag-???-???*" - "final_cat-*" - ) - -for n in "${!NAMES[@]}"; do - name=${NAMES[$n]} - dir=${DIRS[$n]} - pattern=${PATTERNS[$n]} - upl=$output_rel/$dir/$pattern - upload "$name" "$ID" "$VERBOSE" "${upl[@]}" -done - -echo "End" diff --git a/scripts/sh/job_sp.bash b/scripts/sh/job_sp.bash index 8ffda5205..ae08135f5 100755 --- a/scripts/sh/job_sp.bash +++ b/scripts/sh/job_sp.bash @@ -228,12 +228,6 @@ command_sp() { str=$2 command "$1" "$2" - #res=$? - #if [ $res != 0 ]; then - #upload_logs $ID $VERBOSE - #echo "exiting 'canfar_sp.bash', '$cmd' returned $res, log files for id=$ID uploaded" - #exit $res - #fi } # Tar and upload files to vos @@ -287,13 +281,6 @@ function print_env() { ### Start ### -# Activate conda environment -if [ "$CONDA_DEFAULT_ENV" != "shapepipe" ]; then - echo "Activate conda 'shapepipe' environment" - source $VM_HOME/miniconda3/bin/activate shapepipe -fi - - if [ $do_env == 1 ]; then print_env echo "Exiting" @@ -330,14 +317,8 @@ if [[ $do_job != 0 ]]; then fi fi - ### Retrieve tiles - command_sp "shapepipe_run -c $SP_CONFIG/config_get_tiles_$retrieve.ini" "Run shapepipe (get tiles)" - - ### Find exposures - command_sp "shapepipe_run -c $SP_CONFIG/config_find_exp.ini" "Run shapepipe (find exposures)" - - ### Retrieve exposures - command_sp "shapepipe_run -c $SP_CONFIG/config_get_exp_$retrieve.ini" "Run shapepipe (get exposures)" + ### Retrieve files + command_sp "shapepipe_run -c $SP_CONFIG/config_GitFeGie_$retrieve.ini" "Retrieve images" fi @@ -346,7 +327,7 @@ fi if [[ $do_job != 0 ]]; then ### Uncompress tile weights - command_sp "shapepipe_run -c $SP_CONFIG/config_unfz_w.ini" "Run shapepipe (uncompress tile weights)" + command_sp "shapepipe_run -c $SP_CONFIG/config_tile_Uz.ini" "Run shapepipe (uncompress tile weights)" ### Split images into single-HDU files, merge headers for WCS info command_sp "shapepipe_run -c $SP_CONFIG/config_exp_SpMh.ini" "Run shapepipe (split images, merge headers)" @@ -357,11 +338,8 @@ fi (( do_job= $job & 4 )) if [[ $do_job != 0 ]]; then - ### Mask tiles - command_sp "shapepipe_run -c $SP_CONFIG/config_tile_Ma.ini" "Run shapepipe (mask tiles)" - - ### Mask exposures - command_sp "shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini" "Run shapepipe (mask exposures)" + ### Mask tiles and exposures + command_sp "shapepipe_run -c $SP_CONFIG/config_MaMa.ini" "Run shapepipe (mask)" fi @@ -373,27 +351,9 @@ if [[ $do_job != 0 ]]; then ### Star detection, selection, PSF model. setools can exit with an error for CCD with insufficient stars, ### the script should continue STOP=0 - command_sp "shapepipe_run -c $SP_CONFIG/config_exp_$psf.ini" "Run shapepipe (exp $psf)" + command_sp "shapepipe_run -c $SP_CONFIG/config_tile_Sx_exp_${psf}.ini" "Run shapepipe (tile detection, exp $psf)" STOP=1 - ### The following are very a bad hacks to get additional input file paths - echo "Looking for PSF ($psf) output files" - if [ "$psf" == "psfex" ]; then - input_psfex=`ls -1 ./output/*/psfex_runner/output/star_split_ratio_80*.psf | head -n 1` - command_sp "ln -s `dirname $input_psfex` input_psfex" "Link psfex output" - else - input_psf_mccd=`find . -name "fitted_model*.npy" | head -n 1` - command_sp "ln -s `dirname $input_psf_mccd` input_psf_mccd" "Link MCCD output" - fi - - echo "Looking for single-exposure single-HDU output files" - input_split_exp=`ls -1 ./output/*/split_exp_runner/output/flag*.fits | head -n 1` - command_sp "ln -s `dirname $input_split_exp` input_split_exp" "Link split_exp output" - - echo "Looking for SExtractor output files" - input_sextractor=`ls -1 ./output/*/sextractor_runner/output/sexcat_sexcat*.fits | head -n 1` - command_sp "ln -s `dirname $input_sextractor` input_sextractor" "Link sextractor output" - fi ## Process tiles up to shape measurement @@ -403,7 +363,7 @@ if [[ $do_job != 0 ]]; then ### PSF model letter: 'P' (psfex) or 'M' (mccd) letter=${psf:0:1} Letter=${letter^} - command_sp "shapepipe_run -c $SP_CONFIG/config_tile_Sx${Letter}iViSmVi.ini" "Run shapepipe (tile PsfInterp=$Letter}: up to ngmix+galsim)" + command_sp "shapepipe_run -c $SP_CONFIG/config_tile_${Letter}iViSmVi.ini" "Run shapepipe (tile PsfInterp=$Letter}: up to ngmix+galsim)" fi diff --git a/shapepipe/modules/get_images_package/get_images.py b/shapepipe/modules/get_images_package/get_images.py index add70f03a..b776c4a8c 100644 --- a/shapepipe/modules/get_images_package/get_images.py +++ b/shapepipe/modules/get_images_package/get_images.py @@ -276,7 +276,7 @@ def retrieve(self, all_inputs, all_outputs): for in_per_type, out_per_type in zip(all_inputs, all_outputs): for idx in range(len(in_per_type)): if self._check_existing_dir: - out_base = os.path.basename(in_per_type[idx]) + out_base = os.path.basename(out_per_type[idx]) path = glob.glob( f'{self._check_existing_dir}/**/{out_base}', recursive=True, diff --git a/shapepipe/modules/get_images_runner.py b/shapepipe/modules/get_images_runner.py index 5b8ff8fde..2f5dbd1eb 100644 --- a/shapepipe/modules/get_images_runner.py +++ b/shapepipe/modules/get_images_runner.py @@ -47,7 +47,7 @@ def get_images_runner( if config.has_option(module_config_sec, 'N_TRY'): n_try = config.getint(module_config_sec, 'N_TRY') else: - n_try = None + n_try = 3 # Paths input_dir = config.getlist(module_config_sec, 'INPUT_PATH') diff --git a/shapepipe/modules/mccd_interp_runner.py b/shapepipe/modules/mccd_interp_runner.py index 054968d98..49780bec6 100644 --- a/shapepipe/modules/mccd_interp_runner.py +++ b/shapepipe/modules/mccd_interp_runner.py @@ -8,6 +8,8 @@ import os +from shapepipe.pipeline.run_log import get_last_dir + from shapepipe.modules.mccd_package import \ mccd_interpolation_script as mccd_interp from shapepipe.modules.mccd_package import shapepipe_auxiliary_mccd as aux_mccd @@ -38,10 +40,11 @@ def mccd_interp_runner( output_dir = run_dirs['output'] if mode == 'CLASSIC': - psf_model_dir = config.getexpanded( + module = config.getexpanded( module_config_sec, 'PSF_MODEL_DIR' ) + psf_model_dir = get_last_dir(run_dirs['run_log'], module) psf_model_pattern = config.get( module_config_sec, 'PSF_MODEL_PATTERN' @@ -89,10 +92,11 @@ def mccd_interp_runner( ) elif mode == 'MULTI-EPOCH': - psf_model_dir = config.getexpanded( + module = config.getexpanded( module_config_sec, 'PSF_MODEL_DIR' ) + psf_model_dir = get_last_dir(run_dirs['run_log'], module) psf_model_pattern = config.get( module_config_sec, 'PSF_MODEL_PATTERN' diff --git a/shapepipe/modules/psfex_interp_package/psfex_interp.py b/shapepipe/modules/psfex_interp_package/psfex_interp.py index 8897039e2..f00ef1d96 100644 --- a/shapepipe/modules/psfex_interp_package/psfex_interp.py +++ b/shapepipe/modules/psfex_interp_package/psfex_interp.py @@ -529,7 +529,7 @@ def process_me(self, dot_psf_dir, dot_psf_pattern, f_wcs_path): if os.path.exists(dot_psf_dir): self._dot_psf_dir = dot_psf_dir else: - raise ValueError(f'Cound not find file {dot_psf_dir}.') + raise ValueError(f'Cound not find directory {dot_psf_dir}.') self._dot_psf_pattern = dot_psf_pattern diff --git a/shapepipe/modules/psfex_interp_runner.py b/shapepipe/modules/psfex_interp_runner.py index 5879e4cbb..f18b543fb 100644 --- a/shapepipe/modules/psfex_interp_runner.py +++ b/shapepipe/modules/psfex_interp_runner.py @@ -9,6 +9,8 @@ from shapepipe.modules.module_decorator import module_runner from shapepipe.modules.psfex_interp_package import psfex_interp +from shapepipe.pipeline.run_log import get_last_dir + @module_runner( version='1.1', @@ -60,10 +62,11 @@ def psfex_interp_runner( elif mode == 'MULTI-EPOCH': # Fetch multi-epoch parameters - dot_psf_dir = config.getexpanded( + module = config.getexpanded( module_config_sec, 'ME_DOT_PSF_DIR', ) + dot_psf_dir = get_last_dir(run_dirs['run_log'], module) dot_psf_pattern = config.get( module_config_sec, 'ME_DOT_PSF_PATTERN', diff --git a/shapepipe/modules/setools_runner.py b/shapepipe/modules/setools_runner.py index 736c817a0..a076bd08e 100644 --- a/shapepipe/modules/setools_runner.py +++ b/shapepipe/modules/setools_runner.py @@ -11,8 +11,8 @@ @module_runner( + input_module='sextractor_runner', version='1.1', - input_module='sextractor_runner_exp', file_pattern=['sexcat'], file_ext=['.fits'], depends=['numpy', 'matplotlib'], diff --git a/shapepipe/modules/vignetmaker_runner.py b/shapepipe/modules/vignetmaker_runner.py index 9bf857590..0a2ea87be 100644 --- a/shapepipe/modules/vignetmaker_runner.py +++ b/shapepipe/modules/vignetmaker_runner.py @@ -9,6 +9,8 @@ from shapepipe.modules.module_decorator import module_runner from shapepipe.modules.vignetmaker_package import vignetmaker as vm +from shapepipe.pipeline.run_log import get_last_dir + @module_runner( version='1.1', @@ -47,7 +49,6 @@ def vignetmaker_runner( # Without masking else: - # Fetch stamp size stamp_size = config.getint(module_config_sec, 'STAMP_SIZE') - 1 # Check stamp size @@ -89,7 +90,11 @@ def vignetmaker_runner( # Run in MULTI-EPOCH mode elif mode == 'MULTI-EPOCH': # Fetch image directory and patterns - image_dir = config.getlist(module_config_sec, 'ME_IMAGE_DIR') + modules = config.getlist(module_config_sec, 'ME_IMAGE_DIR') + image_dir = [] + for module in modules: + last_dir = get_last_dir(run_dirs['run_log'], module) + image_dir.append(last_dir) image_pattern = config.getlist( module_config_sec, 'ME_IMAGE_PATTERN', diff --git a/shapepipe/pipeline/file_handler.py b/shapepipe/pipeline/file_handler.py index ee7e8c391..67dc78112 100644 --- a/shapepipe/pipeline/file_handler.py +++ b/shapepipe/pipeline/file_handler.py @@ -15,7 +15,7 @@ import numpy as np from shapepipe.modules.module_runners import get_module_runners -from shapepipe.pipeline.run_log import RunLog +from shapepipe.pipeline.run_log import RunLog, get_list, get_all, get_last from shapepipe.pipeline.shared import split_module_run from shapepipe.utilities.file_system import mkdir @@ -394,7 +394,8 @@ def _check_input_dir_list(self, dir_list): elif 'last' in dir.lower(): module_run, module, _ = self._get_module_run_name(dir) - last_module = self._run_log.get_last(module) + runs = get_list(self._run_log_file) + last_module = get_last(runs, module) input_dir.append( self.setpath( self.setpath(last_module, module_run), @@ -404,7 +405,8 @@ def _check_input_dir_list(self, dir_list): elif 'all' in dir.lower(): module_run, module, _ = self._get_module_run_name(dir) - all_runs = self._run_log.get_all(module) + runs = get_list(self._run_log_file) + all_runs = get_all(runs, module) input_dir.extend([ self.setpath( self.setpath(run.split(' ')[0], module_run), @@ -692,7 +694,8 @@ def _create_module_run_dirs(self, module, run_name): 'run': self.run_dir, 'log': self._log_dir, 'tmp': self._tmp_dir, - 'output': self.output_dir + 'output': self.output_dir, + 'run_log': self._run_log_file } def _set_module_input_dir(self, module, run_name): diff --git a/shapepipe/pipeline/run_log.py b/shapepipe/pipeline/run_log.py index af332c57b..b5a5fcbbe 100644 --- a/shapepipe/pipeline/run_log.py +++ b/shapepipe/pipeline/run_log.py @@ -33,7 +33,7 @@ def __init__(self, run_log_file, module_list, current_run): self._module_list = ','.join(module_list) self.current_run = current_run self._write() - self._get_list() + get_list(run_log_file) def _write(self): """Write. @@ -44,69 +44,6 @@ def _write(self): with open(self.run_log_file, 'a') as run_log: run_log.write(f'{self.current_run} {self._module_list}\n') - def _get_list(self): - """Get List. - - Get a list of all runs in the run log. - - """ - with open(self.run_log_file, 'r') as run_log: - lines = run_log.readlines() - - self._runs = [line.rstrip() for line in lines] - - def get_all(self, module): - """Get All. - - Get all previous pipeline runs of a given model. - - Parameters - ---------- - module : str - Module name - - Returns - ------- - all_runs: list of str - All run paths for a given module - - """ - module, _ = split_module_run(module) - - all_runs = [ - run for run in self._runs - if module in run.split()[1].split(',') - ] - if len(all_runs) == 0: - raise RuntimeError( - f'No previous run of module \'{module}\' found' - ) - - all_runs = all_runs[::-1] - - return all_runs - - def get_last(self, module): - """Get Last. - - Get the last run of the pipeline for a given module. - - Parameters - ---------- - module : str - Module name - - Returns - ------- - str - The last run for a given module - - """ - all_runs = self.get_all(module) - last_run = all_runs[0] - - return last_run.split(' ')[0] - def get_run(self, search_string): """Get Run. @@ -144,3 +81,116 @@ def get_run(self, search_string): ) return runs[0].split(' ')[0] + + +def get_list(run_log_file): + """Get List. + + Get a list of all runs in the run log. + + Parameters + ---------- + run_log_file : str + Run log file name + + Returns + ------- + list + Run log file entries + + """ + with open(run_log_file, 'r') as run_log: + lines = run_log.readlines() + + runs = [line.rstrip() for line in lines] + + return runs + + +def get_all(runs, module): + """Get All. + + Get all previous pipeline runs of a given model. + + Parameters + ---------- + runs : list + Log file entries consisting of directory and module(s) + module : str + Module name + + Raises + ------ + RuntimeError + If no previous runs are found + + Returns + ------- + list + All run paths for a given module + + """ + module_base, _ = split_module_run(module) + + all_runs = [ + run for run in runs + if module_base in run.split()[1].split(',') + ] + if len(all_runs) == 0: + raise RuntimeError( + f'No previous run of module \'{module_base}\' found' + ) + + all_runs = all_runs[::-1] + + return all_runs + + +def get_last(runs, module): + """Get Last. + + Get the last run of the pipeline for a given module. + + Parameters + ---------- + runs : list + Log file entries consisting of directory and module(s) + module : str + Module name + + Returns + ------- + str + The last run for a given module + + """ + all_runs = get_all(runs, module) + last_run = all_runs[0] + + return last_run.split(' ')[0] + + +def get_last_dir(run_log_file, module): + """Get Last Dir. + + Return directory path corresponding to last run of given module. + + Parameters + ---------- + run_log_file : str + Run log file name + module : str + Module name + + Returns + ------- + str + Directory name of last module run + + """ + runs = get_list(run_log_file) + all_runs = get_all(runs, module) + last_run = all_runs[0].split(' ')[0] + last_dir = f'{last_run}/{module}/output' + + return last_dir