From 35e2bc3adc94a9148a5221077a3fa31b3b4b6bd8 Mon Sep 17 00:00:00 2001 From: Axel Date: Wed, 30 Jun 2021 15:25:43 +0200 Subject: [PATCH 1/7] SExtractor clean up testing version --- .../modules/SExtractor_runner/__init__.py | 11 + .../SExtractor_runner/sextractor_script.py | 402 ++++++++++++++ shapepipe/modules/__init__.py | 3 +- shapepipe/modules/sextractor_runner.py | 497 +++++++++++------- 4 files changed, 711 insertions(+), 202 deletions(-) create mode 100644 shapepipe/modules/SExtractor_runner/__init__.py create mode 100644 shapepipe/modules/SExtractor_runner/sextractor_script.py diff --git a/shapepipe/modules/SExtractor_runner/__init__.py b/shapepipe/modules/SExtractor_runner/__init__.py new file mode 100644 index 000000000..91b5763b1 --- /dev/null +++ b/shapepipe/modules/SExtractor_runner/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- + +"""SEXTRACTOR MODULE + +This module contains methods to run SEXtractor. + +:Author: Axel Guinot + +""" + +__all__ = ['sextractor_script'] \ No newline at end of file diff --git a/shapepipe/modules/SExtractor_runner/sextractor_script.py b/shapepipe/modules/SExtractor_runner/sextractor_script.py new file mode 100644 index 000000000..11e352539 --- /dev/null +++ b/shapepipe/modules/SExtractor_runner/sextractor_script.py @@ -0,0 +1,402 @@ +# -*- coding: utf-8 -*- + +"""SEXTRACTOR SCRIPT + +This module build SExtractor command line. + +:Author: Axel Guinot & Martin Kilbinger + +""" + +import re +from shapepipe.pipeline.execute import execute +from shapepipe.modules.module_decorator import module_runner +from shapepipe.pipeline import file_io as io + +import numpy as np +from sqlitedict import SqliteDict +from astropy.io import fits + + +def get_header_value(image_path, key): + """Get header value + + This function read a value from the header image. + + Parameters + ---------- + image_path: str + Path to the input image + key: str + Key from which the value is requested (has to be float) + + Returns + ------- + val: float + Value associated to the key provided + + """ + + h = fits.getheader(image_path) + + val = h[key] + + try: + val = float(val) + except: + raise ValueError('The key {} does not return a float value. Got {}'.format(key, val)) + + return val + + +def make_post_process(cat_path, f_wcs_path, pos_params, ccd_size): + """Make post process + + This function will add one hdu by epoch to the SExtractor catalog. + Only works for tiles. + The columns will be: NUMBER same as SExtractor NUMBER + EXP_NAME name of the single exposure for this epoch + CCD_N extansion where the object is + + Parameters + ---------- + cat_path: str + Path to the outputed SExtractor catalog + f_wcs_path: str + Path to the log file containing wcs for all single exp CCDs + pos_params: list + World coordinates to use to match the objects. + ccd_size: list + Size of a ccd [nx, ny] + + Raises + ------ + IOError + + """ + + cat = io.FITSCatalog(cat_path, SEx_catalog=True, + open_mode=io.BaseCatalog.OpenMode.ReadWrite) + cat.open() + + f_wcs = SqliteDict(f_wcs_path) + key_list = list(f_wcs.keys()) + if len(key_list) == 0: + raise IOError('Could not read sql file \'{}\'' + ''.format(f_wcs_path)) + n_hdu = len(f_wcs[key_list[0]]) + + hist = [] + for i in cat.get_data(1)[0][0]: + if re.split('HISTORY', i)[0] == '': + hist.append(i) + + exp_list = [] + pattern = r'([0-9]*)p\.(.*)' + for i in hist: + m = re.search(pattern, i) + exp_list.append(m.group(1)) + + obj_id = np.copy(cat.get_data()['NUMBER']) + + n_epoch = np.zeros(len(obj_id), dtype='int32') + for i, exp in enumerate(exp_list): + pos_tmp = np.ones(len(obj_id), dtype='int32') * -1 + for j in range(n_hdu): + w = f_wcs[exp][j]['WCS'] + pix_tmp = w.all_world2pix(cat.get_data()[pos_params[0]], + cat.get_data()[pos_params[1]], 0) + ind = ((pix_tmp[0] > int(ccd_size[0])) & + (pix_tmp[0] < int(ccd_size[1])) & + (pix_tmp[1] > int(ccd_size[2])) & + (pix_tmp[1] < int(ccd_size[3]))) + pos_tmp[ind] = j + n_epoch[ind] += 1 + exp_name = np.array([exp_list[i] for n in range(len(obj_id))]) + a = np.array([(obj_id[ii], exp_name[ii], pos_tmp[ii]) + for ii in range(len(exp_name))], + dtype=[('NUMBER', obj_id.dtype), + ('EXP_NAME', exp_name.dtype), + ('CCD_N', pos_tmp.dtype)]) + cat.save_as_fits(data=a, ext_name='EPOCH_{}'.format(i)) + cat.open() + + f_wcs.close() + + cat.add_col('N_EPOCH', n_epoch) + + cat.close() + + +class sextractor_caller(): + """SExtractor Caller + + This class construct the command line to call SExtractor based on the + input files and parameters. + + Parameters + ---------- + path_input_files: list + List with all the path for the input files + path_output_file: str + Path for the output catalogue + path_dot_sex: str + Path to the ".sex" config file + path_dot_param: str + Path to the ".param" config file + path_dot_conv: str + Path to the ".conv" kernel file + use_weight: bool + Weither a weight is profided for the measurement + use_flag: bool + Weither a flag is provided for the measurement + use_psf: bool + Weither a psf is provided for the model + use_detection: bool + Weither a detection image is provided + use_detection_weight: bool + Weither a detection weight is provided + use_zero_point: bool + Weither to use a zero point from the input image + zero_point_key: str + Header key corresponding to the zero point + use_backgroup: bool + Weither to use a background value form the input image + background_key: str + Header key corresponding to the background value + check_image: str + If provided, add SExtractor check image to the output + + """ + + def __init__(self, + path_input_files, + path_output_dir, number_string, + path_dot_sex, path_dot_param, path_dot_conv, + use_weight, use_flag, use_psf, + use_detection_image, use_detection_weight, + use_zero_point, use_background, + zero_point_key=None, background_key=None, + check_image=None, output_suffix=None): + + self.cmd_line = '' + self._cmd_line_extra = '' + self._extra = 1 + + self._meas_img_path = path_input_files[0] + self._all_input_path = path_input_files + + self._path_output_dir = path_output_dir + self._num_str = number_string + self.path_output_file = self.get_output_name(output_suffix) + + self._path_dot_sex = path_dot_sex + self._path_dot_param = path_dot_param + self._path_dot_conv = path_dot_conv + + self.set_input_files(use_weight, use_flag, use_psf, + use_detection_image, use_detection_weight) + + # Collect optional arguments for SExtractor + self.get_zero_point(use_zero_point, zero_point_key) + self.get_background(use_background, background_key) + self.get_check_image(check_image) + + def get_output_name(self, output_suffix=None): + """Get output names + + Construct the output file path. + + Parameters + ---------- + output_suffix: str + Suffix to add to the output name, can be None + + Returns + ------- + output_file_path: str + Full path of the output file + + """ + if isinstance(output_suffix, type(None)): + self.suffix = '' + else: + if (output_suffix.lower() != None) & (output_suffix != ''): + self.suffix = output_suffix + '_' + else: + self.suffix = '' + + output_file_name = self.suffix + 'sexcat{}.fits'.format(self._num_str) + output_file_path = '{}/{}'.format(self._path_output_dir, output_file_name) + + return output_file_path + + def set_input_files(self, use_weight, use_flag, use_psf, + use_detect_img, use_detect_weight): + """Set input files + + Setup all the input image files. + + Parameters + ---------- + use_weight: bool + Weither a weight is profided for the measurement + use_flag: bool + Weither a flag is provided for the measurement + use_psf: bool + Weither a psf is provided for the model + use_detect_img: bool + Weither a detection image is provided + use_detect_weight: bool + Weither a detection weight is provided + + Raise + ----- + ValueError + + """ + if use_weight: + weight_image = self._all_input_path[self._extra] + self._extra += 1 + + if use_flag: + self._cmd_line_extra += ' -FLAG_IMAGE {0}'.format(self._all_input_path[self._extra]) + self._extra += 1 + + if use_psf: + self._cmd_line_extra += ' -PSF_NAME {0}'.format(self._all_input_path[self._extra]) + self._extra += 1 + + # Check for separate files for detection and measurement + + # First, consistency checks + if use_detect_weight and not use_detect_img: + raise ValueError('DETECTION_WEIGHT cannot be True if DETECTION_IMAGE is False') + if use_detect_weight and not use_weight: + raise ValueError('DETECTION_WEIGHT cannot be True if WEIGHT_FILE is False') + + # Check for separate image file for detection and measurement + if use_detect_img: + self._detect_img_path = self._all_input_path[self._extra] + self._extra += 1 + else: + self._detect_img_path = self._meas_img_path + + # Check for separate weight file corresponding to the detection image. + # If False, use measurement weight image. + # Note: This could be changed, and no weight image could be used, but + # this might lead to more user errors. + if use_weight: + if use_detect_weight: + detect_weight_path = self._all_input_path[self._extra] + self._extra += 1 + else: + detect_weight_path = weight_image + self._cmd_line_extra += ' -WEIGHT_IMAGE {0},{1}'\ + ''.format(detect_weight_path, weight_image) + else: + self._cmd_line_extra += ' -WEIGHT_TYPE None' + + if self._extra != len(self._all_input_path): + raise ValueError('Incoherence between input file number and keys ' + 'related to extra files: 1 regular + {} extra ' + 'files not compatible with total file list ' + 'length of {}' + ''.format(self._extra-1, len(self._all_input_path))) + + def get_zero_point(self, use_zp, zp_key=None): + """Get Zero Point + + Use a zero point from input image header. + + Parameters + ---------- + use_zp: bool + If True, add the zero point to the command line + zp_key: str + Header key corresponding to the zero point + + """ + if use_zp and isinstance(zp_key, type(None)): + zp_value = get_header_value(self._meas_img_path, zp_key) + self._cmd_line_extra += ' -MAG_ZEROPOINT {}'.format(zp_value) + + def get_background(self, use_bkg, bkg_key=None): + """Get Background + + Use a background value from input image header. + + Parameters + ---------- + use_bkg: bool + If True, add the background value to the command line + bkg_key: str + Header key corresponding to the background value + + """ + if use_bkg and isinstance(bkg_key, type(None)): + bkg_value = get_header_value(self._meas_img_path, bkg_key) + self._cmd_line_extra += ' -BACK_TYPE MANUAL' \ + ' -BACK_VALUE {0}'.format(bkg_value) + + def get_check_image(self, check_image): + """Get check image + + Handle the check images if any are requested. + + Parameters + ---------- + check_image: list + List of SExtractor keys corresponding to check images + + """ + + if (len(check_image) == 1) & (check_image[0] == ''): + check_type = ['NONE'] + check_name = ['none'] + else: + check_type = [] + check_name = [] + for i in check_image: + check_type.append(i.upper()) + check_name.append(self._path_output_dir + '/' + self.suffix + + i.lower() + + self._num_str + '.fits') + + self._cmd_line_extra += (' -CHECKIMAGE_TYPE {0} -CHECKIMAGE_NAME {1}' + ''.format(','.join(check_type), ','.join(check_name))) + + def make_command_line(self, exec_path): + """ Make command line + + Main that construct the command line to run SExtractor + + Parameters + ---------- + exec_path: str + Path to SExtractor executable + + Return + ------ + command_line: str + Full command line to call SExtractor + + """ + + # Base arguments for SExtractor + command_line_base = ('{0} {1},{2} -c {3} -PARAMETERS_NAME {4} -FILTER_NAME {5} ' + '-CATALOG_NAME {6}' + ''.format(exec_path, + self._detect_img_path, + self._meas_img_path, + self._path_dot_sex, + self._path_dot_param, + self._path_dot_conv, + self.path_output_file)) + + command_line = '{} {}'.format(command_line_base, self._cmd_line_extra) + + return command_line + + + diff --git a/shapepipe/modules/__init__.py b/shapepipe/modules/__init__.py index 3a2762ddd..4db83aaef 100644 --- a/shapepipe/modules/__init__.py +++ b/shapepipe/modules/__init__.py @@ -12,7 +12,8 @@ 'module_runners', 'mask_package', 'PSFExInterpolation_package', - 'SETools_package' + 'SETools_package', + 'SExtractor_runner' ] __module_list__ = ['execute_example', 'erase_output_runner', diff --git a/shapepipe/modules/sextractor_runner.py b/shapepipe/modules/sextractor_runner.py index c777f41da..3cd19d832 100644 --- a/shapepipe/modules/sextractor_runner.py +++ b/shapepipe/modules/sextractor_runner.py @@ -11,122 +11,272 @@ import re from shapepipe.pipeline.execute import execute from shapepipe.modules.module_decorator import module_runner -from shapepipe.pipeline import file_io as io - -import numpy as np -from sqlitedict import SqliteDict -from astropy.io import fits - - -def get_header_value(image_path, key): - """Get header value - - This function read a value from the header image. - - Parameters - ---------- - image_path: str - Path to the input image - key: str - Key from which the value is requested (has to be float) - - Returns - ------- - val: float - Value associated to the key provided - - """ - - h = fits.getheader(image_path) - - val = h[key] - - try: - val = float(val) - except: - raise ValueError('The key {} does not return a float value. Got '.format(key, val)) - - return val - - -def make_post_process(cat_path, f_wcs_path, pos_params, ccd_size): - """Make post process - - This function will add one hdu by epoch to the SExtractor catalog. - Only works for tiles. - The columns will be: NUMBER same as SExtractor NUMBER - EXP_NAME name of the single exposure for this epoch - CCD_N extansion where the object is - - Parameters - ---------- - cat_path: str - Path to the outputed SExtractor catalog - f_wcs_path: str - Path to the log file containing wcs for all single exp CCDs - pos_params: list - World coordinates to use to match the objects. - ccd_size: list - Size of a ccd [nx, ny] - - Raises - ------ - IOError - - """ - - cat = io.FITSCatalog(cat_path, SEx_catalog=True, - open_mode=io.BaseCatalog.OpenMode.ReadWrite) - cat.open() - - f_wcs = SqliteDict(f_wcs_path) - key_list = list(f_wcs.keys()) - if len(key_list) == 0: - raise IOError('Could not read sql file \'{}\'' - ''.format(f_wcs_path)) - n_hdu = len(f_wcs[key_list[0]]) - - hist = [] - for i in cat.get_data(1)[0][0]: - if re.split('HISTORY', i)[0] == '': - hist.append(i) - - exp_list = [] - pattern = r'([0-9]*)p\.(.*)' - for i in hist: - m = re.search(pattern, i) - exp_list.append(m.group(1)) - - obj_id = np.copy(cat.get_data()['NUMBER']) - - n_epoch = np.zeros(len(obj_id), dtype='int32') - for i, exp in enumerate(exp_list): - pos_tmp = np.ones(len(obj_id), dtype='int32') * -1 - for j in range(n_hdu): - w = f_wcs[exp][j]['WCS'] - pix_tmp = w.all_world2pix(cat.get_data()[pos_params[0]], - cat.get_data()[pos_params[1]], 0) - ind = ((pix_tmp[0] > int(ccd_size[0])) & - (pix_tmp[0] < int(ccd_size[1])) & - (pix_tmp[1] > int(ccd_size[2])) & - (pix_tmp[1] < int(ccd_size[3]))) - pos_tmp[ind] = j - n_epoch[ind] += 1 - exp_name = np.array([exp_list[i] for n in range(len(obj_id))]) - a = np.array([(obj_id[ii], exp_name[ii], pos_tmp[ii]) - for ii in range(len(exp_name))], - dtype=[('NUMBER', obj_id.dtype), - ('EXP_NAME', exp_name.dtype), - ('CCD_N', pos_tmp.dtype)]) - cat.save_as_fits(data=a, ext_name='EPOCH_{}'.format(i)) - cat.open() - - f_wcs.close() - - cat.add_col('N_EPOCH', n_epoch) - - cat.close() - +# from shapepipe.pipeline import file_io as io + +from shapepipe.modules.SExtractor_runner import sextractor_script as ss + +# import numpy as np +# from sqlitedict import SqliteDict +# from astropy.io import fits + + +# def get_header_value(image_path, key): +# """Get header value + +# This function read a value from the header image. + +# Parameters +# ---------- +# image_path: str +# Path to the input image +# key: str +# Key from which the value is requested (has to be float) + +# Returns +# ------- +# val: float +# Value associated to the key provided + +# """ + +# h = fits.getheader(image_path) + +# val = h[key] + +# try: +# val = float(val) +# except: +# raise ValueError('The key {} does not return a float value. Got '.format(key, val)) + +# return val + + +# def make_post_process(cat_path, f_wcs_path, pos_params, ccd_size): +# """Make post process + +# This function will add one hdu by epoch to the SExtractor catalog. +# Only works for tiles. +# The columns will be: NUMBER same as SExtractor NUMBER +# EXP_NAME name of the single exposure for this epoch +# CCD_N extansion where the object is + +# Parameters +# ---------- +# cat_path: str +# Path to the outputed SExtractor catalog +# f_wcs_path: str +# Path to the log file containing wcs for all single exp CCDs +# pos_params: list +# World coordinates to use to match the objects. +# ccd_size: list +# Size of a ccd [nx, ny] + +# Raises +# ------ +# IOError + +# """ + +# cat = io.FITSCatalog(cat_path, SEx_catalog=True, +# open_mode=io.BaseCatalog.OpenMode.ReadWrite) +# cat.open() + +# f_wcs = SqliteDict(f_wcs_path) +# key_list = list(f_wcs.keys()) +# if len(key_list) == 0: +# raise IOError('Could not read sql file \'{}\'' +# ''.format(f_wcs_path)) +# n_hdu = len(f_wcs[key_list[0]]) + +# hist = [] +# for i in cat.get_data(1)[0][0]: +# if re.split('HISTORY', i)[0] == '': +# hist.append(i) + +# exp_list = [] +# pattern = r'([0-9]*)p\.(.*)' +# for i in hist: +# m = re.search(pattern, i) +# exp_list.append(m.group(1)) + +# obj_id = np.copy(cat.get_data()['NUMBER']) + +# n_epoch = np.zeros(len(obj_id), dtype='int32') +# for i, exp in enumerate(exp_list): +# pos_tmp = np.ones(len(obj_id), dtype='int32') * -1 +# for j in range(n_hdu): +# w = f_wcs[exp][j]['WCS'] +# pix_tmp = w.all_world2pix(cat.get_data()[pos_params[0]], +# cat.get_data()[pos_params[1]], 0) +# ind = ((pix_tmp[0] > int(ccd_size[0])) & +# (pix_tmp[0] < int(ccd_size[1])) & +# (pix_tmp[1] > int(ccd_size[2])) & +# (pix_tmp[1] < int(ccd_size[3]))) +# pos_tmp[ind] = j +# n_epoch[ind] += 1 +# exp_name = np.array([exp_list[i] for n in range(len(obj_id))]) +# a = np.array([(obj_id[ii], exp_name[ii], pos_tmp[ii]) +# for ii in range(len(exp_name))], +# dtype=[('NUMBER', obj_id.dtype), +# ('EXP_NAME', exp_name.dtype), +# ('CCD_N', pos_tmp.dtype)]) +# cat.save_as_fits(data=a, ext_name='EPOCH_{}'.format(i)) +# cat.open() + +# f_wcs.close() + +# cat.add_col('N_EPOCH', n_epoch) + +# cat.close() + + +# @module_runner(input_module='mask_runner', version='1.0.1', +# file_pattern=['image', 'weight', 'flag'], +# file_ext=['.fits', '.fits', '.fits'], +# executes=['sex'], depends=['numpy']) +# def sextractor_runner(input_file_list, run_dirs, file_number_string, +# config, w_log): + +# num = file_number_string + +# exec_path = config.getexpanded("SEXTRACTOR_RUNNER", "EXEC_PATH") +# dot_sex = config.getexpanded("SEXTRACTOR_RUNNER", "DOT_SEX_FILE") +# dot_param = config.getexpanded("SEXTRACTOR_RUNNER", "DOT_PARAM_FILE") +# dot_conv = config.getexpanded("SEXTRACTOR_RUNNER", "DOT_CONV_FILE") + +# weight_file = config.getboolean("SEXTRACTOR_RUNNER", "WEIGHT_IMAGE") +# flag_file = config.getboolean("SEXTRACTOR_RUNNER", "FLAG_IMAGE") +# psf_file = config.getboolean("SEXTRACTOR_RUNNER", "PSF_FILE") +# detection_image = config.getboolean("SEXTRACTOR_RUNNER", "DETECTION_IMAGE") +# detection_weight = config.getboolean("SEXTRACTOR_RUNNER", "DETECTION_WEIGHT") + +# zp_from_header = config.getboolean("SEXTRACTOR_RUNNER", "ZP_FROM_HEADER") +# if zp_from_header: +# zp_key = config.get("SEXTRACTOR_RUNNER", "ZP_KEY") +# zp_value = get_header_value(input_file_list[0], zp_key) + +# bkg_from_header = config.getboolean("SEXTRACTOR_RUNNER", "BKG_FROM_HEADER") +# if bkg_from_header: +# bkg_key = config.get("SEXTRACTOR_RUNNER", "BKG_KEY") +# bkg_value = get_header_value(input_file_list[0], bkg_key) + +# if config.has_option('SEXTRACTOR_RUNNER', "CHECKIMAGE"): +# check_image = config.getlist("SEXTRACTOR_RUNNER", "CHECKIMAGE") +# else: +# check_image = [''] + +# if config.has_option('SEXTRACTOR_RUNNER', 'SUFFIX'): +# suffix = config.get('SEXTRACTOR_RUNNER', 'SUFFIX') +# if (suffix.lower() != 'none') & (suffix != ''): +# suffix = suffix + '_' +# else: +# suffix = '' +# else: +# suffix = '' + +# output_file_name = suffix + 'sexcat{0}.fits'.format(num) +# output_file_path = '{0}/{1}'.format(run_dirs['output'], output_file_name) + +# measurement_image = input_file_list[0] + +# # Collect optional arguments for SExtractor +# command_line_extra = '' +# if zp_from_header: +# command_line_extra += ' -MAG_ZEROPOINT {0}'.format(zp_value) + +# if bkg_from_header: +# command_line_extra += ' -BACK_TYPE MANUAL -BACK_VALUE {0}'.format(bkg_value) + +# extra = 1 +# if weight_file: +# weight_image = input_file_list[extra] +# extra += 1 +# if flag_file: +# command_line_extra += ' -FLAG_IMAGE {0}'.format(input_file_list[extra]) +# extra += 1 +# if psf_file: +# command_line_extra += ' -PSF_NAME {0}'.format(input_file_list[extra]) +# extra += 1 + +# # Check for separate files for detection and measurement + +# # First, consistency checks +# if detection_weight and not detection_image: +# raise ValueError('DETECTION_WEIGHT cannot be True if DETECTION_IMAGE is False') +# if detection_weight and not weight_file: +# raise ValueError('DETECTION_WEIGHT cannot be True if WEIGHT_FILE is False') + +# # Check for separate image file for detection and measurement +# if detection_image: +# detection_image_path = input_file_list[extra] +# extra += 1 +# else: +# detection_image_path = measurement_image + +# # Check for separate weight file corresponding to the detection image. +# # If False, use measurement weight image. +# # Note: This could be changed, and no weight image could be used, but +# # this might lead to more user errors. +# if weight_file: +# if detection_weight: +# detection_weight_path = input_file_list[extra] +# extra += 1 +# else: +# detection_weight_path = weight_image +# command_line_extra += ' -WEIGHT_IMAGE {0},{1}'\ +# ''.format(detection_weight_path, weight_image) +# else: +# command_line_extra += ' -WEIGHT_TYPE None' + +# if extra != len(input_file_list): +# raise ValueError('Incoherence between input file number and keys ' +# 'related to extra files: 1 regular + {} extra ' +# 'files not compatible with total file list ' +# 'length of {}' +# ''.format(extra-1, len(input_file_list))) + +# if (len(check_image) == 1) & (check_image[0] == ''): +# check_type = ['NONE'] +# check_name = ['none'] +# else: +# check_type = [] +# check_name = [] +# for i in check_image: +# check_type.append(i.upper()) +# check_name.append(run_dirs['output'] + '/' + suffix + i.lower() + +# num + '.fits') + +# command_line_extra += (' -CHECKIMAGE_TYPE {0} -CHECKIMAGE_NAME {1}' +# ''.format(','.join(check_type), ','.join(check_name))) + +# # Base arguments for SExtractor +# command_line_base = ('{0} {1},{2} -c {3} -PARAMETERS_NAME {4} -FILTER_NAME {5} ' +# '-CATALOG_NAME {6}' +# ''.format(exec_path, detection_image_path, measurement_image, +# dot_sex, dot_param, dot_conv, output_file_path)) + +# command_line = '{} {}'.format(command_line_base, command_line_extra) +# w_log.info('Calling command \'{}\''.format(command_line)) + +# stderr, stdout = execute(command_line) + +# check_error = re.findall('error', stdout.lower()) +# check_error2 = re.findall('all done', stdout.lower()) + +# if check_error == []: +# stderr2 = '' +# else: +# stderr2 = stdout +# if check_error2 == []: +# stderr2 = stdout + +# if config.getboolean("SEXTRACTOR_RUNNER", "MAKE_POST_PROCESS"): +# f_wcs_path = config.getexpanded("SEXTRACTOR_RUNNER", "LOG_WCS") +# pos_params = config.getlist("SEXTRACTOR_RUNNER", "WORLD_POSITION") +# ccd_size = config.getlist("SEXTRACTOR_RUNNER", "CCD_SIZE") +# make_post_process(output_file_path, f_wcs_path, pos_params, ccd_size) + +# return stdout, stderr2 @module_runner(input_module='mask_runner', version='1.0.1', file_pattern=['image', 'weight', 'flag'], @@ -151,12 +301,16 @@ def sextractor_runner(input_file_list, run_dirs, file_number_string, zp_from_header = config.getboolean("SEXTRACTOR_RUNNER", "ZP_FROM_HEADER") if zp_from_header: zp_key = config.get("SEXTRACTOR_RUNNER", "ZP_KEY") - zp_value = get_header_value(input_file_list[0], zp_key) + # zp_value = get_header_value(input_file_list[0], zp_key) + else: + zp_key = None bkg_from_header = config.getboolean("SEXTRACTOR_RUNNER", "BKG_FROM_HEADER") if bkg_from_header: bkg_key = config.get("SEXTRACTOR_RUNNER", "BKG_KEY") - bkg_value = get_header_value(input_file_list[0], bkg_key) + # bkg_value = get_header_value(input_file_list[0], bkg_key) + else: + bkg_key = None if config.has_option('SEXTRACTOR_RUNNER', "CHECKIMAGE"): check_image = config.getlist("SEXTRACTOR_RUNNER", "CHECKIMAGE") @@ -172,88 +326,27 @@ def sextractor_runner(input_file_list, run_dirs, file_number_string, else: suffix = '' - output_file_name = suffix + 'sexcat{0}.fits'.format(num) - output_file_path = '{0}/{1}'.format(run_dirs['output'], output_file_name) - - measurement_image = input_file_list[0] - - # Collect optional arguments for SExtractor - command_line_extra = '' - if zp_from_header: - command_line_extra += ' -MAG_ZEROPOINT {0}'.format(zp_value) - if bkg_from_header: - command_line_extra += ' -BACK_TYPE MANUAL -BACK_VALUE {0}'.format(bkg_value) - - extra = 1 - if weight_file: - weight_image = input_file_list[extra] - extra += 1 - if flag_file: - command_line_extra += ' -FLAG_IMAGE {0}'.format(input_file_list[extra]) - extra += 1 - if psf_file: - command_line_extra += ' -PSF_NAME {0}'.format(input_file_list[extra]) - extra += 1 - - # Check for separate files for detection and measurement - - # First, consistency checks - if detection_weight and not detection_image: - raise ValueError('DETECTION_WEIGHT cannot be True if DETECTION_IMAGE is False') - if detection_weight and not weight_file: - raise ValueError('DETECTION_WEIGHT cannot be True if WEIGHT_FILE is False') - - # Check for separate image file for detection and measurement - if detection_image: - detection_image_path = input_file_list[extra] - extra += 1 - else: - detection_image_path = measurement_image - - # Check for separate weight file corresponding to the detection image. - # If False, use measurement weight image. - # Note: This could be changed, and no weight image could be used, but - # this might lead to more user errors. - if weight_file: - if detection_weight: - detection_weight_path = input_file_list[extra] - extra += 1 - else: - detection_weight_path = weight_image - command_line_extra += ' -WEIGHT_IMAGE {0},{1}'\ - ''.format(detection_weight_path, weight_image) - else: - command_line_extra += ' -WEIGHT_TYPE None' - - if extra != len(input_file_list): - raise ValueError('Incoherence between input file number and keys ' - 'related to extra files: 1 regular + {} extra ' - 'files not compatible with total file list ' - 'length of {}' - ''.format(extra-1, len(input_file_list))) - - if (len(check_image) == 1) & (check_image[0] == ''): - check_type = ['NONE'] - check_name = ['none'] - else: - check_type = [] - check_name = [] - for i in check_image: - check_type.append(i.upper()) - check_name.append(run_dirs['output'] + '/' + suffix + i.lower() + - num + '.fits') - - command_line_extra += (' -CHECKIMAGE_TYPE {0} -CHECKIMAGE_NAME {1}' - ''.format(','.join(check_type), ','.join(check_name))) - - # Base arguments for SExtractor - command_line_base = ('{0} {1},{2} -c {3} -PARAMETERS_NAME {4} -FILTER_NAME {5} ' - '-CATALOG_NAME {6}' - ''.format(exec_path, detection_image_path, measurement_image, - dot_sex, dot_param, dot_conv, output_file_path)) - - command_line = '{} {}'.format(command_line_base, command_line_extra) + SE_caller = ss.sextractor_caller(input_file_list, + run_dirs['output'], + num, + dot_sex, + dot_param, + dot_conv, + weight_file, + flag_file, + psf_file, + detection_image, + detection_weight, + zp_from_header, + zp_key, + bkg_from_header, + bkg_key, + check_image, + suffix) + + + command_line = SE_caller.make_command_line(exec_path) w_log.info('Calling command \'{}\''.format(command_line)) stderr, stdout = execute(command_line) @@ -272,6 +365,8 @@ def sextractor_runner(input_file_list, run_dirs, file_number_string, f_wcs_path = config.getexpanded("SEXTRACTOR_RUNNER", "LOG_WCS") pos_params = config.getlist("SEXTRACTOR_RUNNER", "WORLD_POSITION") ccd_size = config.getlist("SEXTRACTOR_RUNNER", "CCD_SIZE") - make_post_process(output_file_path, f_wcs_path, pos_params, ccd_size) + ss.make_post_process(SE_caller.path_output_file, + f_wcs_path, + pos_params, ccd_size) - return stdout, stderr2 + return stdout, stderr2 \ No newline at end of file From 4c5caf083bc35c3b9f5d70d6bca328cb4c0a0995 Mon Sep 17 00:00:00 2001 From: Axel Date: Wed, 30 Jun 2021 15:55:33 +0200 Subject: [PATCH 2/7] Bug fix --- shapepipe/modules/sextractor_runner.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/shapepipe/modules/sextractor_runner.py b/shapepipe/modules/sextractor_runner.py index 3cd19d832..78661c5ab 100644 --- a/shapepipe/modules/sextractor_runner.py +++ b/shapepipe/modules/sextractor_runner.py @@ -319,12 +319,14 @@ def sextractor_runner(input_file_list, run_dirs, file_number_string, if config.has_option('SEXTRACTOR_RUNNER', 'SUFFIX'): suffix = config.get('SEXTRACTOR_RUNNER', 'SUFFIX') - if (suffix.lower() != 'none') & (suffix != ''): - suffix = suffix + '_' - else: - suffix = '' + # if (suffix.lower() != 'none') & (suffix != ''): + # suffix = suffix + '_' + # else: + # suffix = '' + # else: + # suffix = '' else: - suffix = '' + suffix = None SE_caller = ss.sextractor_caller(input_file_list, From 5d716372fa78929cebfbdf58d9639d6ea29afb6c Mon Sep 17 00:00:00 2001 From: Axel Date: Wed, 30 Jun 2021 15:58:25 +0200 Subject: [PATCH 3/7] remove old code --- shapepipe/modules/sextractor_runner.py | 271 ------------------------- 1 file changed, 271 deletions(-) diff --git a/shapepipe/modules/sextractor_runner.py b/shapepipe/modules/sextractor_runner.py index 78661c5ab..28433f32c 100644 --- a/shapepipe/modules/sextractor_runner.py +++ b/shapepipe/modules/sextractor_runner.py @@ -11,272 +11,9 @@ import re from shapepipe.pipeline.execute import execute from shapepipe.modules.module_decorator import module_runner -# from shapepipe.pipeline import file_io as io from shapepipe.modules.SExtractor_runner import sextractor_script as ss -# import numpy as np -# from sqlitedict import SqliteDict -# from astropy.io import fits - - -# def get_header_value(image_path, key): -# """Get header value - -# This function read a value from the header image. - -# Parameters -# ---------- -# image_path: str -# Path to the input image -# key: str -# Key from which the value is requested (has to be float) - -# Returns -# ------- -# val: float -# Value associated to the key provided - -# """ - -# h = fits.getheader(image_path) - -# val = h[key] - -# try: -# val = float(val) -# except: -# raise ValueError('The key {} does not return a float value. Got '.format(key, val)) - -# return val - - -# def make_post_process(cat_path, f_wcs_path, pos_params, ccd_size): -# """Make post process - -# This function will add one hdu by epoch to the SExtractor catalog. -# Only works for tiles. -# The columns will be: NUMBER same as SExtractor NUMBER -# EXP_NAME name of the single exposure for this epoch -# CCD_N extansion where the object is - -# Parameters -# ---------- -# cat_path: str -# Path to the outputed SExtractor catalog -# f_wcs_path: str -# Path to the log file containing wcs for all single exp CCDs -# pos_params: list -# World coordinates to use to match the objects. -# ccd_size: list -# Size of a ccd [nx, ny] - -# Raises -# ------ -# IOError - -# """ - -# cat = io.FITSCatalog(cat_path, SEx_catalog=True, -# open_mode=io.BaseCatalog.OpenMode.ReadWrite) -# cat.open() - -# f_wcs = SqliteDict(f_wcs_path) -# key_list = list(f_wcs.keys()) -# if len(key_list) == 0: -# raise IOError('Could not read sql file \'{}\'' -# ''.format(f_wcs_path)) -# n_hdu = len(f_wcs[key_list[0]]) - -# hist = [] -# for i in cat.get_data(1)[0][0]: -# if re.split('HISTORY', i)[0] == '': -# hist.append(i) - -# exp_list = [] -# pattern = r'([0-9]*)p\.(.*)' -# for i in hist: -# m = re.search(pattern, i) -# exp_list.append(m.group(1)) - -# obj_id = np.copy(cat.get_data()['NUMBER']) - -# n_epoch = np.zeros(len(obj_id), dtype='int32') -# for i, exp in enumerate(exp_list): -# pos_tmp = np.ones(len(obj_id), dtype='int32') * -1 -# for j in range(n_hdu): -# w = f_wcs[exp][j]['WCS'] -# pix_tmp = w.all_world2pix(cat.get_data()[pos_params[0]], -# cat.get_data()[pos_params[1]], 0) -# ind = ((pix_tmp[0] > int(ccd_size[0])) & -# (pix_tmp[0] < int(ccd_size[1])) & -# (pix_tmp[1] > int(ccd_size[2])) & -# (pix_tmp[1] < int(ccd_size[3]))) -# pos_tmp[ind] = j -# n_epoch[ind] += 1 -# exp_name = np.array([exp_list[i] for n in range(len(obj_id))]) -# a = np.array([(obj_id[ii], exp_name[ii], pos_tmp[ii]) -# for ii in range(len(exp_name))], -# dtype=[('NUMBER', obj_id.dtype), -# ('EXP_NAME', exp_name.dtype), -# ('CCD_N', pos_tmp.dtype)]) -# cat.save_as_fits(data=a, ext_name='EPOCH_{}'.format(i)) -# cat.open() - -# f_wcs.close() - -# cat.add_col('N_EPOCH', n_epoch) - -# cat.close() - - -# @module_runner(input_module='mask_runner', version='1.0.1', -# file_pattern=['image', 'weight', 'flag'], -# file_ext=['.fits', '.fits', '.fits'], -# executes=['sex'], depends=['numpy']) -# def sextractor_runner(input_file_list, run_dirs, file_number_string, -# config, w_log): - -# num = file_number_string - -# exec_path = config.getexpanded("SEXTRACTOR_RUNNER", "EXEC_PATH") -# dot_sex = config.getexpanded("SEXTRACTOR_RUNNER", "DOT_SEX_FILE") -# dot_param = config.getexpanded("SEXTRACTOR_RUNNER", "DOT_PARAM_FILE") -# dot_conv = config.getexpanded("SEXTRACTOR_RUNNER", "DOT_CONV_FILE") - -# weight_file = config.getboolean("SEXTRACTOR_RUNNER", "WEIGHT_IMAGE") -# flag_file = config.getboolean("SEXTRACTOR_RUNNER", "FLAG_IMAGE") -# psf_file = config.getboolean("SEXTRACTOR_RUNNER", "PSF_FILE") -# detection_image = config.getboolean("SEXTRACTOR_RUNNER", "DETECTION_IMAGE") -# detection_weight = config.getboolean("SEXTRACTOR_RUNNER", "DETECTION_WEIGHT") - -# zp_from_header = config.getboolean("SEXTRACTOR_RUNNER", "ZP_FROM_HEADER") -# if zp_from_header: -# zp_key = config.get("SEXTRACTOR_RUNNER", "ZP_KEY") -# zp_value = get_header_value(input_file_list[0], zp_key) - -# bkg_from_header = config.getboolean("SEXTRACTOR_RUNNER", "BKG_FROM_HEADER") -# if bkg_from_header: -# bkg_key = config.get("SEXTRACTOR_RUNNER", "BKG_KEY") -# bkg_value = get_header_value(input_file_list[0], bkg_key) - -# if config.has_option('SEXTRACTOR_RUNNER', "CHECKIMAGE"): -# check_image = config.getlist("SEXTRACTOR_RUNNER", "CHECKIMAGE") -# else: -# check_image = [''] - -# if config.has_option('SEXTRACTOR_RUNNER', 'SUFFIX'): -# suffix = config.get('SEXTRACTOR_RUNNER', 'SUFFIX') -# if (suffix.lower() != 'none') & (suffix != ''): -# suffix = suffix + '_' -# else: -# suffix = '' -# else: -# suffix = '' - -# output_file_name = suffix + 'sexcat{0}.fits'.format(num) -# output_file_path = '{0}/{1}'.format(run_dirs['output'], output_file_name) - -# measurement_image = input_file_list[0] - -# # Collect optional arguments for SExtractor -# command_line_extra = '' -# if zp_from_header: -# command_line_extra += ' -MAG_ZEROPOINT {0}'.format(zp_value) - -# if bkg_from_header: -# command_line_extra += ' -BACK_TYPE MANUAL -BACK_VALUE {0}'.format(bkg_value) - -# extra = 1 -# if weight_file: -# weight_image = input_file_list[extra] -# extra += 1 -# if flag_file: -# command_line_extra += ' -FLAG_IMAGE {0}'.format(input_file_list[extra]) -# extra += 1 -# if psf_file: -# command_line_extra += ' -PSF_NAME {0}'.format(input_file_list[extra]) -# extra += 1 - -# # Check for separate files for detection and measurement - -# # First, consistency checks -# if detection_weight and not detection_image: -# raise ValueError('DETECTION_WEIGHT cannot be True if DETECTION_IMAGE is False') -# if detection_weight and not weight_file: -# raise ValueError('DETECTION_WEIGHT cannot be True if WEIGHT_FILE is False') - -# # Check for separate image file for detection and measurement -# if detection_image: -# detection_image_path = input_file_list[extra] -# extra += 1 -# else: -# detection_image_path = measurement_image - -# # Check for separate weight file corresponding to the detection image. -# # If False, use measurement weight image. -# # Note: This could be changed, and no weight image could be used, but -# # this might lead to more user errors. -# if weight_file: -# if detection_weight: -# detection_weight_path = input_file_list[extra] -# extra += 1 -# else: -# detection_weight_path = weight_image -# command_line_extra += ' -WEIGHT_IMAGE {0},{1}'\ -# ''.format(detection_weight_path, weight_image) -# else: -# command_line_extra += ' -WEIGHT_TYPE None' - -# if extra != len(input_file_list): -# raise ValueError('Incoherence between input file number and keys ' -# 'related to extra files: 1 regular + {} extra ' -# 'files not compatible with total file list ' -# 'length of {}' -# ''.format(extra-1, len(input_file_list))) - -# if (len(check_image) == 1) & (check_image[0] == ''): -# check_type = ['NONE'] -# check_name = ['none'] -# else: -# check_type = [] -# check_name = [] -# for i in check_image: -# check_type.append(i.upper()) -# check_name.append(run_dirs['output'] + '/' + suffix + i.lower() + -# num + '.fits') - -# command_line_extra += (' -CHECKIMAGE_TYPE {0} -CHECKIMAGE_NAME {1}' -# ''.format(','.join(check_type), ','.join(check_name))) - -# # Base arguments for SExtractor -# command_line_base = ('{0} {1},{2} -c {3} -PARAMETERS_NAME {4} -FILTER_NAME {5} ' -# '-CATALOG_NAME {6}' -# ''.format(exec_path, detection_image_path, measurement_image, -# dot_sex, dot_param, dot_conv, output_file_path)) - -# command_line = '{} {}'.format(command_line_base, command_line_extra) -# w_log.info('Calling command \'{}\''.format(command_line)) - -# stderr, stdout = execute(command_line) - -# check_error = re.findall('error', stdout.lower()) -# check_error2 = re.findall('all done', stdout.lower()) - -# if check_error == []: -# stderr2 = '' -# else: -# stderr2 = stdout -# if check_error2 == []: -# stderr2 = stdout - -# if config.getboolean("SEXTRACTOR_RUNNER", "MAKE_POST_PROCESS"): -# f_wcs_path = config.getexpanded("SEXTRACTOR_RUNNER", "LOG_WCS") -# pos_params = config.getlist("SEXTRACTOR_RUNNER", "WORLD_POSITION") -# ccd_size = config.getlist("SEXTRACTOR_RUNNER", "CCD_SIZE") -# make_post_process(output_file_path, f_wcs_path, pos_params, ccd_size) - -# return stdout, stderr2 @module_runner(input_module='mask_runner', version='1.0.1', file_pattern=['image', 'weight', 'flag'], @@ -301,14 +38,12 @@ def sextractor_runner(input_file_list, run_dirs, file_number_string, zp_from_header = config.getboolean("SEXTRACTOR_RUNNER", "ZP_FROM_HEADER") if zp_from_header: zp_key = config.get("SEXTRACTOR_RUNNER", "ZP_KEY") - # zp_value = get_header_value(input_file_list[0], zp_key) else: zp_key = None bkg_from_header = config.getboolean("SEXTRACTOR_RUNNER", "BKG_FROM_HEADER") if bkg_from_header: bkg_key = config.get("SEXTRACTOR_RUNNER", "BKG_KEY") - # bkg_value = get_header_value(input_file_list[0], bkg_key) else: bkg_key = None @@ -319,12 +54,6 @@ def sextractor_runner(input_file_list, run_dirs, file_number_string, if config.has_option('SEXTRACTOR_RUNNER', 'SUFFIX'): suffix = config.get('SEXTRACTOR_RUNNER', 'SUFFIX') - # if (suffix.lower() != 'none') & (suffix != ''): - # suffix = suffix + '_' - # else: - # suffix = '' - # else: - # suffix = '' else: suffix = None From 1e5e01059af4211064fffb4cb44aad4a732625c3 Mon Sep 17 00:00:00 2001 From: Axel Date: Mon, 26 Jul 2021 17:39:04 +0200 Subject: [PATCH 4/7] Fix syntax and docstring --- .../SExtractor_runner/sextractor_script.py | 152 ++++++++++-------- shapepipe/modules/__init__.py | 75 ++++----- shapepipe/modules/sextractor_runner.py | 60 +++---- 3 files changed, 160 insertions(+), 127 deletions(-) diff --git a/shapepipe/modules/SExtractor_runner/sextractor_script.py b/shapepipe/modules/SExtractor_runner/sextractor_script.py index 11e352539..d59ebdc76 100644 --- a/shapepipe/modules/SExtractor_runner/sextractor_script.py +++ b/shapepipe/modules/SExtractor_runner/sextractor_script.py @@ -9,8 +9,6 @@ """ import re -from shapepipe.pipeline.execute import execute -from shapepipe.modules.module_decorator import module_runner from shapepipe.pipeline import file_io as io import numpy as np @@ -44,7 +42,9 @@ def get_header_value(image_path, key): try: val = float(val) except: - raise ValueError('The key {} does not return a float value. Got {}'.format(key, val)) + raise ValueError( + f'The key {key} does not return a float value. Got {val}' + ) return val @@ -75,15 +75,16 @@ def make_post_process(cat_path, f_wcs_path, pos_params, ccd_size): """ - cat = io.FITSCatalog(cat_path, SEx_catalog=True, - open_mode=io.BaseCatalog.OpenMode.ReadWrite) + cat = io.FITSCatalog( + cat_path, SEx_catalog=True, + open_mode=io.BaseCatalog.OpenMode.ReadWrite, + ) cat.open() f_wcs = SqliteDict(f_wcs_path) key_list = list(f_wcs.keys()) if len(key_list) == 0: - raise IOError('Could not read sql file \'{}\'' - ''.format(f_wcs_path)) + raise IOError(f'Could not read sql file \'{f_wcs_path}\'') n_hdu = len(f_wcs[key_list[0]]) hist = [] @@ -138,8 +139,10 @@ class sextractor_caller(): ---------- path_input_files: list List with all the path for the input files - path_output_file: str - Path for the output catalogue + path_output_dir: str + Path for the output directory + number_string: str + Pipeline intern numerotation path_dot_sex: str Path to the ".sex" config file path_dot_param: str @@ -152,7 +155,7 @@ class sextractor_caller(): Weither a flag is provided for the measurement use_psf: bool Weither a psf is provided for the model - use_detection: bool + use_detection_image: bool Weither a detection image is provided use_detection_weight: bool Weither a detection weight is provided @@ -166,18 +169,21 @@ class sextractor_caller(): Header key corresponding to the background value check_image: str If provided, add SExtractor check image to the output + output_suffix: str + If provided, add a suffix to the output files """ - - def __init__(self, - path_input_files, - path_output_dir, number_string, - path_dot_sex, path_dot_param, path_dot_conv, - use_weight, use_flag, use_psf, - use_detection_image, use_detection_weight, - use_zero_point, use_background, - zero_point_key=None, background_key=None, - check_image=None, output_suffix=None): + def __init__( + self, + path_input_files, + path_output_dir, number_string, + path_dot_sex, path_dot_param, path_dot_conv, + use_weight, use_flag, use_psf, + use_detection_image, use_detection_weight, + use_zero_point, use_background, + zero_point_key=None, background_key=None, + check_image=None, output_suffix=None + ): self.cmd_line = '' self._cmd_line_extra = '' @@ -221,18 +227,21 @@ def get_output_name(self, output_suffix=None): if isinstance(output_suffix, type(None)): self.suffix = '' else: - if (output_suffix.lower() != None) & (output_suffix != ''): + if (output_suffix.lower() is not None) & (output_suffix != ''): self.suffix = output_suffix + '_' else: self.suffix = '' - output_file_name = self.suffix + 'sexcat{}.fits'.format(self._num_str) - output_file_path = '{}/{}'.format(self._path_output_dir, output_file_name) + output_file_name = self.suffix + f'sexcat{self._num_str}.fits' + output_file_path = f'{self._path_output_dir}/{output_file_name}' return output_file_path - def set_input_files(self, use_weight, use_flag, use_psf, - use_detect_img, use_detect_weight): + def set_input_files( + self, + use_weight, use_flag, use_psf, + use_detect_img, use_detect_weight + ): """Set input files Setup all the input image files. @@ -253,27 +262,39 @@ def set_input_files(self, use_weight, use_flag, use_psf, Raise ----- ValueError - + """ if use_weight: weight_image = self._all_input_path[self._extra] self._extra += 1 - + if use_flag: - self._cmd_line_extra += ' -FLAG_IMAGE {0}'.format(self._all_input_path[self._extra]) + self._cmd_line_extra += ( + f' -FLAG_IMAGE ' + + f'{self._all_input_path[self._extra]}' + ) self._extra += 1 if use_psf: - self._cmd_line_extra += ' -PSF_NAME {0}'.format(self._all_input_path[self._extra]) + self._cmd_line_extra += ( + ' -PSF_NAME ' + + f'{self._all_input_path[self._extra]}' + ) self._extra += 1 # Check for separate files for detection and measurement # First, consistency checks if use_detect_weight and not use_detect_img: - raise ValueError('DETECTION_WEIGHT cannot be True if DETECTION_IMAGE is False') + raise ValueError( + 'DETECTION_WEIGHT cannot be True ' + + 'if DETECTION_IMAGE is False' + ) if use_detect_weight and not use_weight: - raise ValueError('DETECTION_WEIGHT cannot be True if WEIGHT_FILE is False') + raise ValueError( + 'DETECTION_WEIGHT cannot be True ' + + 'if WEIGHT_FILE is False' + ) # Check for separate image file for detection and measurement if use_detect_img: @@ -292,17 +313,20 @@ def set_input_files(self, use_weight, use_flag, use_psf, self._extra += 1 else: detect_weight_path = weight_image - self._cmd_line_extra += ' -WEIGHT_IMAGE {0},{1}'\ - ''.format(detect_weight_path, weight_image) + self._cmd_line_extra += ( + f' -WEIGHT_IMAGE {detect_weight_path}' + + f',{weight_image}' + ) else: self._cmd_line_extra += ' -WEIGHT_TYPE None' if self._extra != len(self._all_input_path): - raise ValueError('Incoherence between input file number and keys ' - 'related to extra files: 1 regular + {} extra ' - 'files not compatible with total file list ' - 'length of {}' - ''.format(self._extra-1, len(self._all_input_path))) + raise ValueError( + f'Incoherence between input file number and keys ' + + f'related to extra files: 1 regular + {self._extra-1} extra ' + + f'files not compatible with total file list ' + + f'length of {len(self._all_input_path)}' + ) def get_zero_point(self, use_zp, zp_key=None): """Get Zero Point @@ -319,7 +343,7 @@ def get_zero_point(self, use_zp, zp_key=None): """ if use_zp and isinstance(zp_key, type(None)): zp_value = get_header_value(self._meas_img_path, zp_key) - self._cmd_line_extra += ' -MAG_ZEROPOINT {}'.format(zp_value) + self._cmd_line_extra += f' -MAG_ZEROPOINT {zp_value}' def get_background(self, use_bkg, bkg_key=None): """Get Background @@ -332,13 +356,14 @@ def get_background(self, use_bkg, bkg_key=None): If True, add the background value to the command line bkg_key: str Header key corresponding to the background value - + """ if use_bkg and isinstance(bkg_key, type(None)): bkg_value = get_header_value(self._meas_img_path, bkg_key) - self._cmd_line_extra += ' -BACK_TYPE MANUAL' \ - ' -BACK_VALUE {0}'.format(bkg_value) - + self._cmd_line_extra += ( + f' -BACK_TYPE MANUAL -BACK_VALUE {bkg_value}' + ) + def get_check_image(self, check_image): """Get check image @@ -348,7 +373,7 @@ def get_check_image(self, check_image): ---------- check_image: list List of SExtractor keys corresponding to check images - + """ if (len(check_image) == 1) & (check_image[0] == ''): @@ -359,13 +384,17 @@ def get_check_image(self, check_image): check_name = [] for i in check_image: check_type.append(i.upper()) - check_name.append(self._path_output_dir + '/' + self.suffix - + i.lower() + - self._num_str + '.fits') + check_name.append( + self._path_output_dir + '/' + self.suffix + + i.lower() + + self._num_str + '.fits' + ) + + self._cmd_line_extra += ( + f" -CHECKIMAGE_TYPE {','.join(check_type)} " + + f"-CHECKIMAGE_NAME {','.join(check_name)}" + ) - self._cmd_line_extra += (' -CHECKIMAGE_TYPE {0} -CHECKIMAGE_NAME {1}' - ''.format(','.join(check_type), ','.join(check_name))) - def make_command_line(self, exec_path): """ Make command line @@ -384,19 +413,14 @@ def make_command_line(self, exec_path): """ # Base arguments for SExtractor - command_line_base = ('{0} {1},{2} -c {3} -PARAMETERS_NAME {4} -FILTER_NAME {5} ' - '-CATALOG_NAME {6}' - ''.format(exec_path, - self._detect_img_path, - self._meas_img_path, - self._path_dot_sex, - self._path_dot_param, - self._path_dot_conv, - self.path_output_file)) - - command_line = '{} {}'.format(command_line_base, self._cmd_line_extra) - - return command_line - + command_line_base = ( + f'{exec_path} {self._detect_img_path},{self._meas_img_path} ' + + f'-c {self._path_dot_sex} ' + + f'-PARAMETERS_NAME {self._path_dot_param}' + + f' -FILTER_NAME {self._path_dot_conv} ' + + f'-CATALOG_NAME {self.path_output_file}' + ) + command_line = f'{command_line_base} {self._cmd_line_extra}' + return command_line diff --git a/shapepipe/modules/__init__.py b/shapepipe/modules/__init__.py index 4db83aaef..9c6e3ca7b 100644 --- a/shapepipe/modules/__init__.py +++ b/shapepipe/modules/__init__.py @@ -8,39 +8,42 @@ """ -__all__ = ['module_decorator', - 'module_runners', - 'mask_package', - 'PSFExInterpolation_package', - 'SETools_package', - 'SExtractor_runner' - ] -__module_list__ = ['execute_example', - 'erase_output_runner', - 'find_exposures_runner', - 'galsim_shapes_runner', - 'galsim_shapes_v2_runner', - 'get_images_runner', - 'get_images_runner2', - 'make_catalog_runner', - 'mask_runner', - 'mask_runner_exp', - 'match_external_runner', - 'merge_sep_cats_runner', - 'merge_headers_runner', - 'merge_star_cat_runner', - 'ngmix_runner', - 'paste_cat_runner', - 'psfex_runner', - 'psfex_interp_runner', - 'python_example', - 'serial_example', - 'setools_runner', - 'sextractor_runner', - 'sextractor_runner_exp', - 'split_exp_runner', - 'spread_model_runner', - 'swarp_runner', - 'vignetmaker_runner', - 'vignetmaker_runner2', - 'uncompress_fits_image_runner'] +__all__ = [ + 'module_decorator', + 'module_runners', + 'mask_package', + 'PSFExInterpolation_package', + 'SETools_package', + 'SExtractor_runner', +] +__module_list__ = [ + 'execute_example', + 'erase_output_runner', + 'find_exposures_runner', + 'galsim_shapes_runner', + 'galsim_shapes_v2_runner', + 'get_images_runner', + 'get_images_runner2', + 'make_catalog_runner', + 'mask_runner', + 'mask_runner_exp', + 'match_external_runner', + 'merge_sep_cats_runner', + 'merge_headers_runner', + 'merge_star_cat_runner', + 'ngmix_runner', + 'paste_cat_runner', + 'psfex_runner', + 'psfex_interp_runner', + 'python_example', + 'serial_example', + 'setools_runner', + 'sextractor_runner', + 'sextractor_runner_exp', + 'split_exp_runner', + 'spread_model_runner', + 'swarp_runner', + 'vignetmaker_runner', + 'vignetmaker_runner2', + 'uncompress_fits_image_runner', +] diff --git a/shapepipe/modules/sextractor_runner.py b/shapepipe/modules/sextractor_runner.py index 28433f32c..c088e63c7 100644 --- a/shapepipe/modules/sextractor_runner.py +++ b/shapepipe/modules/sextractor_runner.py @@ -15,10 +15,12 @@ from shapepipe.modules.SExtractor_runner import sextractor_script as ss -@module_runner(input_module='mask_runner', version='1.0.1', - file_pattern=['image', 'weight', 'flag'], - file_ext=['.fits', '.fits', '.fits'], - executes=['sex'], depends=['numpy']) +@module_runner( + input_module='mask_runner', version='1.0.1', + file_pattern=['image', 'weight', 'flag'], + file_ext=['.fits', '.fits', '.fits'], + executes=['sex'], depends=['numpy'], +) def sextractor_runner(input_file_list, run_dirs, file_number_string, config, w_log): @@ -33,7 +35,9 @@ def sextractor_runner(input_file_list, run_dirs, file_number_string, flag_file = config.getboolean("SEXTRACTOR_RUNNER", "FLAG_IMAGE") psf_file = config.getboolean("SEXTRACTOR_RUNNER", "PSF_FILE") detection_image = config.getboolean("SEXTRACTOR_RUNNER", "DETECTION_IMAGE") - detection_weight = config.getboolean("SEXTRACTOR_RUNNER", "DETECTION_WEIGHT") + detection_weight = config.getboolean( + "SEXTRACTOR_RUNNER", "DETECTION_WEIGHT" + ) zp_from_header = config.getboolean("SEXTRACTOR_RUNNER", "ZP_FROM_HEADER") if zp_from_header: @@ -57,26 +61,26 @@ def sextractor_runner(input_file_list, run_dirs, file_number_string, else: suffix = None + SE_caller = ss.sextractor_caller( + input_file_list, + run_dirs['output'], + num, + dot_sex, + dot_param, + dot_conv, + weight_file, + flag_file, + psf_file, + detection_image, + detection_weight, + zp_from_header, + zp_key, + bkg_from_header, + bkg_key, + check_image, + suffix, + ) - SE_caller = ss.sextractor_caller(input_file_list, - run_dirs['output'], - num, - dot_sex, - dot_param, - dot_conv, - weight_file, - flag_file, - psf_file, - detection_image, - detection_weight, - zp_from_header, - zp_key, - bkg_from_header, - bkg_key, - check_image, - suffix) - - command_line = SE_caller.make_command_line(exec_path) w_log.info('Calling command \'{}\''.format(command_line)) @@ -96,8 +100,10 @@ def sextractor_runner(input_file_list, run_dirs, file_number_string, f_wcs_path = config.getexpanded("SEXTRACTOR_RUNNER", "LOG_WCS") pos_params = config.getlist("SEXTRACTOR_RUNNER", "WORLD_POSITION") ccd_size = config.getlist("SEXTRACTOR_RUNNER", "CCD_SIZE") - ss.make_post_process(SE_caller.path_output_file, - f_wcs_path, - pos_params, ccd_size) + ss.make_post_process( + SE_caller.path_output_file, + f_wcs_path, + pos_params, ccd_size + ) return stdout, stderr2 \ No newline at end of file From d661c8b911e11abceddb65d7c25fa5a5c94ca38f Mon Sep 17 00:00:00 2001 From: Axel Date: Mon, 26 Jul 2021 17:51:35 +0200 Subject: [PATCH 5/7] Github comments --- .../SExtractor_runner/sextractor_script.py | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/shapepipe/modules/SExtractor_runner/sextractor_script.py b/shapepipe/modules/SExtractor_runner/sextractor_script.py index d59ebdc76..9127d4c8a 100644 --- a/shapepipe/modules/SExtractor_runner/sextractor_script.py +++ b/shapepipe/modules/SExtractor_runner/sextractor_script.py @@ -2,7 +2,7 @@ """SEXTRACTOR SCRIPT -This module build SExtractor command line. +This module builds the SExtractor command line. :Author: Axel Guinot & Martin Kilbinger @@ -19,7 +19,7 @@ def get_header_value(image_path, key): """Get header value - This function read a value from the header image. + This function reads a value from the header image. Parameters ---------- @@ -56,7 +56,7 @@ def make_post_process(cat_path, f_wcs_path, pos_params, ccd_size): Only works for tiles. The columns will be: NUMBER same as SExtractor NUMBER EXP_NAME name of the single exposure for this epoch - CCD_N extansion where the object is + CCD_N extension where the object is Parameters ---------- @@ -132,7 +132,7 @@ def make_post_process(cat_path, f_wcs_path, pos_params, ccd_size): class sextractor_caller(): """SExtractor Caller - This class construct the command line to call SExtractor based on the + This class constructs the command line to call SExtractor based on the input files and parameters. Parameters @@ -187,7 +187,6 @@ def __init__( self.cmd_line = '' self._cmd_line_extra = '' - self._extra = 1 self._meas_img_path = path_input_files[0] self._all_input_path = path_input_files @@ -264,23 +263,25 @@ def set_input_files( ValueError """ + extra = 1 + if use_weight: - weight_image = self._all_input_path[self._extra] - self._extra += 1 + weight_image = self._all_input_path[extra] + extra += 1 if use_flag: self._cmd_line_extra += ( - f' -FLAG_IMAGE ' - + f'{self._all_input_path[self._extra]}' + ' -FLAG_IMAGE ' + + f'{self._all_input_path[extra]}' ) - self._extra += 1 + extra += 1 if use_psf: self._cmd_line_extra += ( ' -PSF_NAME ' - + f'{self._all_input_path[self._extra]}' + + f'{self._all_input_path[extra]}' ) - self._extra += 1 + extra += 1 # Check for separate files for detection and measurement @@ -298,8 +299,8 @@ def set_input_files( # Check for separate image file for detection and measurement if use_detect_img: - self._detect_img_path = self._all_input_path[self._extra] - self._extra += 1 + self._detect_img_path = self._all_input_path[extra] + extra += 1 else: self._detect_img_path = self._meas_img_path @@ -309,8 +310,8 @@ def set_input_files( # this might lead to more user errors. if use_weight: if use_detect_weight: - detect_weight_path = self._all_input_path[self._extra] - self._extra += 1 + detect_weight_path = self._all_input_path[extra] + extra += 1 else: detect_weight_path = weight_image self._cmd_line_extra += ( @@ -320,11 +321,11 @@ def set_input_files( else: self._cmd_line_extra += ' -WEIGHT_TYPE None' - if self._extra != len(self._all_input_path): + if extra != len(self._all_input_path): raise ValueError( - f'Incoherence between input file number and keys ' - + f'related to extra files: 1 regular + {self._extra-1} extra ' - + f'files not compatible with total file list ' + 'Incoherence between input file number and keys ' + + f'related to extra files: 1 regular + {extra-1} extra ' + + 'files not compatible with total file list ' + f'length of {len(self._all_input_path)}' ) From bb1b4935aad416d1a62128b658896a14faf1a1e5 Mon Sep 17 00:00:00 2001 From: Axel Date: Tue, 27 Jul 2021 11:57:54 +0200 Subject: [PATCH 6/7] Fixed pythonstyle errors --- shapepipe/modules/SExtractor_runner/__init__.py | 2 +- .../SExtractor_runner/sextractor_script.py | 16 ++++++++-------- .../modules/find_exposures_package/__init__.py | 1 - shapepipe/modules/sextractor_runner.py | 2 +- shapepipe/modules/split_exp_package/__init__.py | 1 - 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/shapepipe/modules/SExtractor_runner/__init__.py b/shapepipe/modules/SExtractor_runner/__init__.py index 91b5763b1..5792fe4d4 100644 --- a/shapepipe/modules/SExtractor_runner/__init__.py +++ b/shapepipe/modules/SExtractor_runner/__init__.py @@ -8,4 +8,4 @@ """ -__all__ = ['sextractor_script'] \ No newline at end of file +__all__ = ['sextractor_script'] diff --git a/shapepipe/modules/SExtractor_runner/sextractor_script.py b/shapepipe/modules/SExtractor_runner/sextractor_script.py index 9127d4c8a..0facc9257 100644 --- a/shapepipe/modules/SExtractor_runner/sextractor_script.py +++ b/shapepipe/modules/SExtractor_runner/sextractor_script.py @@ -76,8 +76,8 @@ def make_post_process(cat_path, f_wcs_path, pos_params, ccd_size): """ cat = io.FITSCatalog( - cat_path, SEx_catalog=True, - open_mode=io.BaseCatalog.OpenMode.ReadWrite, + cat_path, SEx_catalog=True, + open_mode=io.BaseCatalog.OpenMode.ReadWrite, ) cat.open() @@ -288,13 +288,13 @@ def set_input_files( # First, consistency checks if use_detect_weight and not use_detect_img: raise ValueError( - 'DETECTION_WEIGHT cannot be True ' - + 'if DETECTION_IMAGE is False' + 'DETECTION_WEIGHT cannot be True ' + + 'if DETECTION_IMAGE is False' ) if use_detect_weight and not use_weight: raise ValueError( - 'DETECTION_WEIGHT cannot be True ' - + 'if WEIGHT_FILE is False' + 'DETECTION_WEIGHT cannot be True ' + + 'if WEIGHT_FILE is False' ) # Check for separate image file for detection and measurement @@ -315,8 +315,8 @@ def set_input_files( else: detect_weight_path = weight_image self._cmd_line_extra += ( - f' -WEIGHT_IMAGE {detect_weight_path}' - + f',{weight_image}' + f' -WEIGHT_IMAGE {detect_weight_path}' + + f',{weight_image}' ) else: self._cmd_line_extra += ' -WEIGHT_TYPE None' diff --git a/shapepipe/modules/find_exposures_package/__init__.py b/shapepipe/modules/find_exposures_package/__init__.py index ca63497cc..bea12540b 100644 --- a/shapepipe/modules/find_exposures_package/__init__.py +++ b/shapepipe/modules/find_exposures_package/__init__.py @@ -9,4 +9,3 @@ """ __all__ = ['find_exposures.py'] - diff --git a/shapepipe/modules/sextractor_runner.py b/shapepipe/modules/sextractor_runner.py index c088e63c7..44f171503 100644 --- a/shapepipe/modules/sextractor_runner.py +++ b/shapepipe/modules/sextractor_runner.py @@ -106,4 +106,4 @@ def sextractor_runner(input_file_list, run_dirs, file_number_string, pos_params, ccd_size ) - return stdout, stderr2 \ No newline at end of file + return stdout, stderr2 diff --git a/shapepipe/modules/split_exp_package/__init__.py b/shapepipe/modules/split_exp_package/__init__.py index 16e61ec9c..b00b58b93 100644 --- a/shapepipe/modules/split_exp_package/__init__.py +++ b/shapepipe/modules/split_exp_package/__init__.py @@ -9,4 +9,3 @@ """ __all__ = ['split_exp_script.py'] - From bf0cc3c1065e90420dbbb1da8058651fb4088029 Mon Sep 17 00:00:00 2001 From: Axel Date: Fri, 6 Aug 2021 10:37:19 +0200 Subject: [PATCH 7/7] Sam's comments --- .../SExtractor_runner/sextractor_script.py | 48 ++++++++++++------- shapepipe/modules/sextractor_runner.py | 6 ++- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/shapepipe/modules/SExtractor_runner/sextractor_script.py b/shapepipe/modules/SExtractor_runner/sextractor_script.py index 0facc9257..f93f4d4f4 100644 --- a/shapepipe/modules/SExtractor_runner/sextractor_script.py +++ b/shapepipe/modules/SExtractor_runner/sextractor_script.py @@ -41,7 +41,7 @@ def get_header_value(image_path, key): try: val = float(val) - except: + except Exception: raise ValueError( f'The key {key} does not return a float value. Got {val}' ) @@ -87,21 +87,21 @@ def make_post_process(cat_path, f_wcs_path, pos_params, ccd_size): raise IOError(f'Could not read sql file \'{f_wcs_path}\'') n_hdu = len(f_wcs[key_list[0]]) - hist = [] + history = [] for i in cat.get_data(1)[0][0]: if re.split('HISTORY', i)[0] == '': - hist.append(i) + history.append(i) exp_list = [] pattern = r'([0-9]*)p\.(.*)' - for i in hist: - m = re.search(pattern, i) + for hist in history: + m = re.search(pattern, hist) exp_list.append(m.group(1)) obj_id = np.copy(cat.get_data()['NUMBER']) n_epoch = np.zeros(len(obj_id), dtype='int32') - for i, exp in enumerate(exp_list): + for idx, exp in enumerate(exp_list): pos_tmp = np.ones(len(obj_id), dtype='int32') * -1 for j in range(n_hdu): w = f_wcs[exp][j]['WCS'] @@ -176,13 +176,22 @@ class sextractor_caller(): def __init__( self, path_input_files, - path_output_dir, number_string, - path_dot_sex, path_dot_param, path_dot_conv, - use_weight, use_flag, use_psf, - use_detection_image, use_detection_weight, - use_zero_point, use_background, - zero_point_key=None, background_key=None, - check_image=None, output_suffix=None + path_output_dir, + number_string, + path_dot_sex, + path_dot_param, + path_dot_conv, + use_weight, + use_flag, + use_psf, + use_detection_image, + use_detection_weight, + use_zero_point, + use_background, + zero_point_key=None, + background_key=None, + check_image=None, + output_suffix=None ): self.cmd_line = '' @@ -238,8 +247,11 @@ def get_output_name(self, output_suffix=None): def set_input_files( self, - use_weight, use_flag, use_psf, - use_detect_img, use_detect_weight + use_weight, + use_flag, + use_psf, + use_detect_img, + use_detect_weight ): """Set input files @@ -383,11 +395,11 @@ def get_check_image(self, check_image): else: check_type = [] check_name = [] - for i in check_image: - check_type.append(i.upper()) + for key in check_image: + check_type.append(key.upper()) check_name.append( self._path_output_dir + '/' + self.suffix - + i.lower() + + key.lower() + self._num_str + '.fits' ) diff --git a/shapepipe/modules/sextractor_runner.py b/shapepipe/modules/sextractor_runner.py index 44f171503..95b24334b 100644 --- a/shapepipe/modules/sextractor_runner.py +++ b/shapepipe/modules/sextractor_runner.py @@ -16,10 +16,12 @@ @module_runner( - input_module='mask_runner', version='1.0.1', + input_module='mask_runner', + version='1.0.1', file_pattern=['image', 'weight', 'flag'], file_ext=['.fits', '.fits', '.fits'], - executes=['sex'], depends=['numpy'], + executes=['sex'], + depends=['numpy'], ) def sextractor_runner(input_file_list, run_dirs, file_number_string, config, w_log):