From b5215abfe1700e44292b39b0ab56b7e4eb7afbe1 Mon Sep 17 00:00:00 2001 From: andre zamoranovitorelli Date: Thu, 1 Jul 2021 09:06:13 +0200 Subject: [PATCH 01/16] first cleanup pass --- shapepipe/modules/psfex_runner.py | 89 ++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 32 deletions(-) diff --git a/shapepipe/modules/psfex_runner.py b/shapepipe/modules/psfex_runner.py index e566155b0..17f66cac8 100644 --- a/shapepipe/modules/psfex_runner.py +++ b/shapepipe/modules/psfex_runner.py @@ -12,48 +12,72 @@ import os from shapepipe.pipeline.execute import execute from shapepipe.modules.module_decorator import module_runner +from shapepipe.modules.psfex_package.psfex import PSFex_caller -@module_runner(input_module='setools_runner', version='1.0', - file_pattern=['star_selection'], file_ext=['.fits'], - executes='psfex') -def psfex_runner(input_file_list, run_dirs, file_number_string, - config, w_log): +@module_runner(input_module='setools_runner', + version='1.0', + file_pattern=['star_selection'], + file_ext=['.fits'], + executes='psfex' +) - exec_path = config.getexpanded("PSFEX_RUNNER", "EXEC_PATH") - dot_sex = config.getexpanded("PSFEX_RUNNER", "DOT_PSFEX_FILE") - outcat_name = '{0}/psfex_cat{1}.cat'.format(run_dirs['output'], - file_number_string) +def psfex_runner(input_file_list, + run_dirs, + file_number_string, + config, + w_log + ): + """ + Runs the psfex wrapper package. - command_line = ('{0} {1} -c {2} -PSF_DIR {3} -OUTCAT_NAME {4}' - ''.format(exec_path, input_file_list[0], dot_sex, - run_dirs['output'], outcat_name)) + Args: + input_file_list, run_dirs, file_number_string, + config, w_log - if config.has_option('PSFEX_RUNNER', "CHECKIMAGE"): - check_image = config.getlist("PSFEX_RUNNER", "CHECKIMAGE") - else: - check_image = [''] - if (len(check_image) == 1) & (check_image[0] == ''): - check_type = ['NONE'] - check_name = ['none'] - else: - suffix = re.split(file_number_string, os.path.splitext( - os.path.split(input_file_list[0])[-1])[0])[0] - check_type = [] - check_name = [] - for i in check_image: - check_type.append(i.upper()) - check_name.append(run_dirs['output'] + '/' + suffix + '_' + - i.lower() + file_number_string+'.fits') - command_line += (' -CHECKIMAGE_TYPE {0} -CHECKIMAGE_NAME {1}' - ''.format(','.join(check_type), ','.join(check_name))) + Returns: + + stdout, stderr: str + + """ + inputs = [input_file_list, + run_dirs, + file_number_string, + config, + w_log] - w_log.info('Running command \'{}\'' - ''.format(command_line)) + #extract psfex run configurations + psfex_executable_path = config.getexpanded("PSFEX_RUNNER", + "EXEC_PATH") + output_dir = run_dirs['output'] + outcatalog_name = f'psfex_cat{file_number_string}.cat' + psfex_config_file = config.getexpanded("PSFEX_RUNNER", + "DOT_PSFEX_FILE") + input_file_path = input_file_list[0] + + #check image options + if config.has_option('PSFEX_RUNNER', "CHECKIMAGE"): + check_image_list = config.getlist("PSFEX_RUNNER", "CHECKIMAGE") + else: + check_image_list = [''] + + #prepare the psfex command line + PSFex_call = PSFex_caller(psfex_executable_path, + input_file_path, + psfex_config_file, + output_dir, + outcatalog_name, + check_image_list) + + #generates the psfex command + command_line = PSFex_call.generate_command() + + w_log.info(f'Running command \'{command_line}\'') stderr, stdout = execute(command_line) + #move psfex errors reported as stdout to stderr check_error = re.findall('error', stdout.lower()) check_error2 = re.findall('all done', stdout.lower()) @@ -61,6 +85,7 @@ def psfex_runner(input_file_list, run_dirs, file_number_string, stderr2 = '' else: stderr2 = stdout + if check_error2 == []: stderr2 = stdout From fd282d8b05dd8af9387651ed018795be90472e8c Mon Sep 17 00:00:00 2001 From: andre zamoranovitorelli Date: Thu, 1 Jul 2021 09:25:32 +0200 Subject: [PATCH 02/16] adding package files --- shapepipe/modules/psfex_package/__init__.py | 8 ++ shapepipe/modules/psfex_package/psfex.py | 107 ++++++++++++++++++++ shapepipe/modules/psfex_runner.py | 16 ++- 3 files changed, 126 insertions(+), 5 deletions(-) create mode 100644 shapepipe/modules/psfex_package/__init__.py create mode 100644 shapepipe/modules/psfex_package/psfex.py diff --git a/shapepipe/modules/psfex_package/__init__.py b/shapepipe/modules/psfex_package/__init__.py new file mode 100644 index 000000000..579749286 --- /dev/null +++ b/shapepipe/modules/psfex_package/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +"""VIGNET MAKER PACKAGE +This package contains the module(s) for ``psfex``. +:Authors: Axel Guinot, Tobias Liaudat +""" + +__all__ = ['psfex'] diff --git a/shapepipe/modules/psfex_package/psfex.py b/shapepipe/modules/psfex_package/psfex.py new file mode 100644 index 000000000..865bef7be --- /dev/null +++ b/shapepipe/modules/psfex_package/psfex.py @@ -0,0 +1,107 @@ + +"""PSFEX PACKAGE +This module contains a wrapper class to prepare the psfex command line. +:Author: Axel Guinot +""" +import os +import re + +class PSFex_caller: + """This class contains functions to generate a PSFex command line. + Parameters: + ---------- + psfex_executable_path: str + Full path to the PSFEx executable + input_file_path: str + Full path of the input file + psfex_config_file: str + Full path of the PSFEx configuration file (.psfex) + output_dir: str + Full path of the output directory + outcatalog_name: str + Full path pf the output catalogue + check_image_list: list of str + List of check images + + Returns: + command_line: str + Command line with correct and complete paths for PSFEx execution. + """ + + + def __init__(self, + psfex_executable_path, + input_file_path, + psfex_config_file, + output_dir, + outcatalog_name, + check_image_list): + self.psfex_executable_path = psfex_executable_path + self.input_file_path = input_file_path + self.psfex_config_file = psfex_config_file + self.output_dir = output_dir + self.outcatalog_name = outcatalog_name + self.check_image_list = check_image_list + + def generate_command(self): + """ Generates a command line for running PSFEx """ + + + #prepare command line + command_line = (f'{self.psfex_executable_path} ' + + f'{self.input_file_path} ' + + f'-c {self.psfex_config_file} ' + + f'-PSF_DIR {self.output_dir} ' + + f'-OUTCAT_NAME {self.outcatalog_name}') + + if ((len(self.check_image_list) == 1) + & (self.check_image_list[0] == '')) : + check_type_list = ['NONE'] + check_name_list = ['none'] + else: + #get pattern for filenaming from first file in list + input_file_name = os.path.split(input_file_path)[-1] + input_file_noext = os.path.splitext(input_file_name)[0] + suffix = re.split(file_number_string,input_file_noext)[0] + + check_type_list = [] + check_name_list = [] + for check_image in self.check_image_list: + check_type_list.append(check_image.upper()) + check_name_list.append(f'{output_dir}' + + f'/{suffix}' + + f'{check_image.lower()}' + + f'{file_number_string}.fits') + + #add checks to command line + command_line += (f' -CHECKIMAGE_TYPE {",".join(check_type_list)}' + + f' -CHECKIMAGE_NAME {",".join(check_name_list)}') + + self.command_line = command_line + return command_line + + def exec(self): + """ + Executes the command line. + """ + self.generate_command() + stderr, stdout = execute(self.command_line) + + return stderr, stdout + + def parse_errors(stderr, stdout): + """ + Move errors from output from psfex to errors. + """ + check_error = re.findall('error', stdout.lower()) + check_error2 = re.findall('all done', stdout.lower()) + + if check_error == []: + stderr2 = '' + else: + stderr2 = stdout + + if check_error2 == []: + stderr2 = stdout + + return stdout, stderr2 diff --git a/shapepipe/modules/psfex_runner.py b/shapepipe/modules/psfex_runner.py index 17f66cac8..9bd12e067 100644 --- a/shapepipe/modules/psfex_runner.py +++ b/shapepipe/modules/psfex_runner.py @@ -29,16 +29,22 @@ def psfex_runner(input_file_list, w_log ): """ - Runs the psfex wrapper package. + Runs the psfex wrapper package. - Args: - input_file_list, run_dirs, file_number_string, - config, w_log + Parameters: + ---------- + input_file_list: + + run_dirs: + file_number_string: + config: + w_log: Returns: stdout, stderr: str + Strings with the output and error output of execution. """ inputs = [input_file_list, @@ -70,7 +76,7 @@ def psfex_runner(input_file_list, output_dir, outcatalog_name, check_image_list) - + #generates the psfex command command_line = PSFex_call.generate_command() From 6bd97de8d2a52cd71c47eae1e04ec9cff9980fa2 Mon Sep 17 00:00:00 2001 From: Andre Zamorano Vitorelli Date: Thu, 1 Jul 2021 09:26:52 +0200 Subject: [PATCH 03/16] Update psfex.py --- shapepipe/modules/psfex_package/psfex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shapepipe/modules/psfex_package/psfex.py b/shapepipe/modules/psfex_package/psfex.py index 865bef7be..934b3b566 100644 --- a/shapepipe/modules/psfex_package/psfex.py +++ b/shapepipe/modules/psfex_package/psfex.py @@ -59,7 +59,7 @@ def generate_command(self): check_type_list = ['NONE'] check_name_list = ['none'] else: - #get pattern for filenaming from first file in list + #get pattern for filenaming from file in list input_file_name = os.path.split(input_file_path)[-1] input_file_noext = os.path.splitext(input_file_name)[0] suffix = re.split(file_number_string,input_file_noext)[0] From de34c316ee3a2323612f694552c34e87a07e6814 Mon Sep 17 00:00:00 2001 From: Andre Zamorano Vitorelli Date: Thu, 1 Jul 2021 09:30:56 +0200 Subject: [PATCH 04/16] docstring to runner --- shapepipe/modules/psfex_runner.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/shapepipe/modules/psfex_runner.py b/shapepipe/modules/psfex_runner.py index 9bd12e067..819b1099c 100644 --- a/shapepipe/modules/psfex_runner.py +++ b/shapepipe/modules/psfex_runner.py @@ -33,12 +33,16 @@ def psfex_runner(input_file_list, Parameters: ---------- - input_file_list: - - run_dirs: - file_number_string: + input_file_list: list + List with the full path to the input file + run_dirs: dict + Dictionary with directories + file_number_string: str + Specific number of the data file being processed config: + Config object w_log: + Log object Returns: From fab1d4b79ab469d887be46b1703d65cc1bed6510 Mon Sep 17 00:00:00 2001 From: Andre Zamorano Vitorelli Date: Thu, 1 Jul 2021 11:06:26 +0200 Subject: [PATCH 05/16] style changes to block comments --- shapepipe/modules/psfex_package/psfex.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shapepipe/modules/psfex_package/psfex.py b/shapepipe/modules/psfex_package/psfex.py index 934b3b566..3e635b800 100644 --- a/shapepipe/modules/psfex_package/psfex.py +++ b/shapepipe/modules/psfex_package/psfex.py @@ -47,7 +47,7 @@ def generate_command(self): """ Generates a command line for running PSFEx """ - #prepare command line + # Prepare command line command_line = (f'{self.psfex_executable_path} ' + f'{self.input_file_path} ' + f'-c {self.psfex_config_file} ' @@ -59,7 +59,7 @@ def generate_command(self): check_type_list = ['NONE'] check_name_list = ['none'] else: - #get pattern for filenaming from file in list + # Get pattern for filenaming from file in list input_file_name = os.path.split(input_file_path)[-1] input_file_noext = os.path.splitext(input_file_name)[0] suffix = re.split(file_number_string,input_file_noext)[0] @@ -73,7 +73,7 @@ def generate_command(self): + f'{check_image.lower()}' + f'{file_number_string}.fits') - #add checks to command line + # Add checks to command line command_line += (f' -CHECKIMAGE_TYPE {",".join(check_type_list)}' + f' -CHECKIMAGE_NAME {",".join(check_name_list)}') From 311433c087a93193524a24373ff7994a6772d9a7 Mon Sep 17 00:00:00 2001 From: Andre Zamorano Vitorelli Date: Thu, 1 Jul 2021 11:08:07 +0200 Subject: [PATCH 06/16] style changes to block comments --- shapepipe/modules/psfex_runner.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/shapepipe/modules/psfex_runner.py b/shapepipe/modules/psfex_runner.py index 819b1099c..158e83b62 100644 --- a/shapepipe/modules/psfex_runner.py +++ b/shapepipe/modules/psfex_runner.py @@ -57,7 +57,7 @@ def psfex_runner(input_file_list, config, w_log] - #extract psfex run configurations + # extract psfex run configurations psfex_executable_path = config.getexpanded("PSFEX_RUNNER", "EXEC_PATH") output_dir = run_dirs['output'] @@ -67,13 +67,13 @@ def psfex_runner(input_file_list, "DOT_PSFEX_FILE") input_file_path = input_file_list[0] - #check image options + # check image options if config.has_option('PSFEX_RUNNER', "CHECKIMAGE"): check_image_list = config.getlist("PSFEX_RUNNER", "CHECKIMAGE") else: check_image_list = [''] - #prepare the psfex command line + # prepare the psfex command line PSFex_call = PSFex_caller(psfex_executable_path, input_file_path, psfex_config_file, @@ -81,13 +81,13 @@ def psfex_runner(input_file_list, outcatalog_name, check_image_list) - #generates the psfex command + # generates the psfex command command_line = PSFex_call.generate_command() w_log.info(f'Running command \'{command_line}\'') stderr, stdout = execute(command_line) - #move psfex errors reported as stdout to stderr + # move psfex errors reported as stdout to stderr check_error = re.findall('error', stdout.lower()) check_error2 = re.findall('all done', stdout.lower()) From 9f3df0e94c82f6755ef47b91f95a6e5eebb0ac92 Mon Sep 17 00:00:00 2001 From: andre zamoranovitorelli Date: Thu, 15 Jul 2021 16:04:15 +0200 Subject: [PATCH 07/16] style changes --- shapepipe/modules/psfex_package/psfex.py | 29 +++++++------ shapepipe/modules/psfex_runner.py | 54 ++++++++++++++---------- 2 files changed, 49 insertions(+), 34 deletions(-) diff --git a/shapepipe/modules/psfex_package/psfex.py b/shapepipe/modules/psfex_package/psfex.py index 3e635b800..7d8efe9fe 100644 --- a/shapepipe/modules/psfex_package/psfex.py +++ b/shapepipe/modules/psfex_package/psfex.py @@ -29,13 +29,15 @@ class PSFex_caller: """ - def __init__(self, + def __init__( + self, psfex_executable_path, input_file_path, psfex_config_file, output_dir, outcatalog_name, - check_image_list): + check_image_list + ): self.psfex_executable_path = psfex_executable_path self.input_file_path = input_file_path self.psfex_config_file = psfex_config_file @@ -48,11 +50,13 @@ def generate_command(self): # Prepare command line - command_line = (f'{self.psfex_executable_path} ' - + f'{self.input_file_path} ' - + f'-c {self.psfex_config_file} ' - + f'-PSF_DIR {self.output_dir} ' - + f'-OUTCAT_NAME {self.outcatalog_name}') + command_line = ( + f'{self.psfex_executable_path} ' + + f'{self.input_file_path} ' + + f'-c {self.psfex_config_file} ' + + f'-PSF_DIR {self.output_dir} ' + + f'-OUTCAT_NAME {self.outcatalog_name}' + ) if ((len(self.check_image_list) == 1) & (self.check_image_list[0] == '')) : @@ -68,14 +72,15 @@ def generate_command(self): check_name_list = [] for check_image in self.check_image_list: check_type_list.append(check_image.upper()) - check_name_list.append(f'{output_dir}' - + f'/{suffix}' - + f'{check_image.lower()}' - + f'{file_number_string}.fits') + check_name_list.append( + f'{output_dir}' + + f'/{suffix}' + + f'{check_image.lower()}' + + f'{file_number_string}.fits') # Add checks to command line command_line += (f' -CHECKIMAGE_TYPE {",".join(check_type_list)}' - + f' -CHECKIMAGE_NAME {",".join(check_name_list)}') + + f' -CHECKIMAGE_NAME {",".join(check_name_list)}') self.command_line = command_line return command_line diff --git a/shapepipe/modules/psfex_runner.py b/shapepipe/modules/psfex_runner.py index 158e83b62..4a5164725 100644 --- a/shapepipe/modules/psfex_runner.py +++ b/shapepipe/modules/psfex_runner.py @@ -15,18 +15,20 @@ from shapepipe.modules.psfex_package.psfex import PSFex_caller -@module_runner(input_module='setools_runner', - version='1.0', - file_pattern=['star_selection'], - file_ext=['.fits'], - executes='psfex' +@module_runner( + input_module='setools_runner', + version='1.0', + file_pattern=['star_selection'], + file_ext=['.fits'], + executes='psfex' ) -def psfex_runner(input_file_list, - run_dirs, - file_number_string, - config, - w_log +def psfex_runner( + input_file_list, + run_dirs, + file_number_string, + config, + w_log ): """ Runs the psfex wrapper package. @@ -58,28 +60,36 @@ def psfex_runner(input_file_list, w_log] # extract psfex run configurations - psfex_executable_path = config.getexpanded("PSFEX_RUNNER", - "EXEC_PATH") + psfex_executable_path = config.getexpanded( + "PSFEX_RUNNER", + "EXEC_PATH" + ) output_dir = run_dirs['output'] - outcatalog_name = f'psfex_cat{file_number_string}.cat' - psfex_config_file = config.getexpanded("PSFEX_RUNNER", - "DOT_PSFEX_FILE") + outcatalog_name = f"psfex_cat{file_number_string}.cat" + + psfex_config_file = config.getexpanded( + "PSFEX_RUNNER", + "DOT_PSFEX_FILE" + ) + input_file_path = input_file_list[0] # check image options if config.has_option('PSFEX_RUNNER', "CHECKIMAGE"): - check_image_list = config.getlist("PSFEX_RUNNER", "CHECKIMAGE") + check_image_list = config.getlist("PSFEX_RUNNER","CHECKIMAGE") else: check_image_list = [''] # prepare the psfex command line - PSFex_call = PSFex_caller(psfex_executable_path, - input_file_path, - psfex_config_file, - output_dir, - outcatalog_name, - check_image_list) + PSFex_call = PSFex_caller( + psfex_executable_path, + input_file_path, + psfex_config_file, + output_dir, + outcatalog_name, + check_image_list + ) # generates the psfex command command_line = PSFex_call.generate_command() From 6da09394c3f0e2cc691184c34ab15ce163f55e8d Mon Sep 17 00:00:00 2001 From: andre zamoranovitorelli Date: Thu, 15 Jul 2021 16:34:52 +0200 Subject: [PATCH 08/16] style changes --- shapepipe/modules/psfex_package/psfex.py | 9 +++++---- shapepipe/modules/psfex_runner.py | 16 +++++----------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/shapepipe/modules/psfex_package/psfex.py b/shapepipe/modules/psfex_package/psfex.py index 7d8efe9fe..73769b59c 100644 --- a/shapepipe/modules/psfex_package/psfex.py +++ b/shapepipe/modules/psfex_package/psfex.py @@ -6,6 +6,7 @@ import os import re + class PSFex_caller: """This class contains functions to generate a PSFex command line. Parameters: @@ -28,7 +29,6 @@ class PSFex_caller: Command line with correct and complete paths for PSFEx execution. """ - def __init__( self, psfex_executable_path, @@ -48,7 +48,6 @@ def __init__( def generate_command(self): """ Generates a command line for running PSFEx """ - # Prepare command line command_line = ( f'{self.psfex_executable_path} ' @@ -58,8 +57,10 @@ def generate_command(self): + f'-OUTCAT_NAME {self.outcatalog_name}' ) - if ((len(self.check_image_list) == 1) - & (self.check_image_list[0] == '')) : + if ( + (len(self.check_image_list) == 1) + & (self.check_image_list[0] == '') + ): check_type_list = ['NONE'] check_name_list = ['none'] else: diff --git a/shapepipe/modules/psfex_runner.py b/shapepipe/modules/psfex_runner.py index 4a5164725..a63047945 100644 --- a/shapepipe/modules/psfex_runner.py +++ b/shapepipe/modules/psfex_runner.py @@ -22,13 +22,12 @@ file_ext=['.fits'], executes='psfex' ) - def psfex_runner( - input_file_list, - run_dirs, - file_number_string, - config, - w_log + input_file_list, + run_dirs, + file_number_string, + config, + w_log ): """ Runs the psfex wrapper package. @@ -53,11 +52,6 @@ def psfex_runner( Strings with the output and error output of execution. """ - inputs = [input_file_list, - run_dirs, - file_number_string, - config, - w_log] # extract psfex run configurations psfex_executable_path = config.getexpanded( From fd3e37b17c28b83ab71f49935656ef425aee58c9 Mon Sep 17 00:00:00 2001 From: andre zamoranovitorelli Date: Thu, 15 Jul 2021 16:52:21 +0200 Subject: [PATCH 09/16] style changes --- shapepipe/modules/psfex_package/psfex.py | 10 ++++++---- shapepipe/modules/psfex_runner.py | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/shapepipe/modules/psfex_package/psfex.py b/shapepipe/modules/psfex_package/psfex.py index 73769b59c..99bbaafa9 100644 --- a/shapepipe/modules/psfex_package/psfex.py +++ b/shapepipe/modules/psfex_package/psfex.py @@ -60,14 +60,14 @@ def generate_command(self): if ( (len(self.check_image_list) == 1) & (self.check_image_list[0] == '') - ): + ): check_type_list = ['NONE'] check_name_list = ['none'] else: # Get pattern for filenaming from file in list input_file_name = os.path.split(input_file_path)[-1] input_file_noext = os.path.splitext(input_file_name)[0] - suffix = re.split(file_number_string,input_file_noext)[0] + suffix = re.split(file_number_string, input_file_noext)[0] check_type_list = [] check_name_list = [] @@ -80,8 +80,10 @@ def generate_command(self): + f'{file_number_string}.fits') # Add checks to command line - command_line += (f' -CHECKIMAGE_TYPE {",".join(check_type_list)}' - + f' -CHECKIMAGE_NAME {",".join(check_name_list)}') + command_line += ( + f' -CHECKIMAGE_TYPE {",".join(check_type_list)}' + + f' -CHECKIMAGE_NAME {",".join(check_name_list)}' + ) self.command_line = command_line return command_line diff --git a/shapepipe/modules/psfex_runner.py b/shapepipe/modules/psfex_runner.py index a63047945..9ffdddb11 100644 --- a/shapepipe/modules/psfex_runner.py +++ b/shapepipe/modules/psfex_runner.py @@ -28,7 +28,7 @@ def psfex_runner( file_number_string, config, w_log - ): +): """ Runs the psfex wrapper package. @@ -71,7 +71,7 @@ def psfex_runner( # check image options if config.has_option('PSFEX_RUNNER', "CHECKIMAGE"): - check_image_list = config.getlist("PSFEX_RUNNER","CHECKIMAGE") + check_image_list = config.getlist("PSFEX_RUNNER", "CHECKIMAGE") else: check_image_list = [''] From 230a2417d4906f0ec7074123d79d6e1872339da9 Mon Sep 17 00:00:00 2001 From: andre zamoranovitorelli Date: Thu, 15 Jul 2021 17:05:32 +0200 Subject: [PATCH 10/16] style changes --- shapepipe/modules/psfex_package/psfex.py | 2 +- shapepipe/modules/psfex_runner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shapepipe/modules/psfex_package/psfex.py b/shapepipe/modules/psfex_package/psfex.py index 99bbaafa9..8a087569c 100644 --- a/shapepipe/modules/psfex_package/psfex.py +++ b/shapepipe/modules/psfex_package/psfex.py @@ -37,7 +37,7 @@ def __init__( output_dir, outcatalog_name, check_image_list - ): + ): self.psfex_executable_path = psfex_executable_path self.input_file_path = input_file_path self.psfex_config_file = psfex_config_file diff --git a/shapepipe/modules/psfex_runner.py b/shapepipe/modules/psfex_runner.py index 9ffdddb11..b4cf0f605 100644 --- a/shapepipe/modules/psfex_runner.py +++ b/shapepipe/modules/psfex_runner.py @@ -76,7 +76,7 @@ def psfex_runner( check_image_list = [''] # prepare the psfex command line - PSFex_call = PSFex_caller( + PSFex_call = PSFex_caller( psfex_executable_path, input_file_path, psfex_config_file, From fa1b2a79f15fae61d299711e22d5d3cde0547ee1 Mon Sep 17 00:00:00 2001 From: andre zamoranovitorelli Date: Thu, 15 Jul 2021 18:31:29 +0200 Subject: [PATCH 11/16] renaming the psfex.py file to psfex_script.py --- shapepipe/modules/psfex_package/__init__.py | 6 +++--- .../modules/psfex_package/{psfex.py => psfex_script.py} | 0 shapepipe/modules/psfex_runner.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename shapepipe/modules/psfex_package/{psfex.py => psfex_script.py} (100%) diff --git a/shapepipe/modules/psfex_package/__init__.py b/shapepipe/modules/psfex_package/__init__.py index 579749286..106208f40 100644 --- a/shapepipe/modules/psfex_package/__init__.py +++ b/shapepipe/modules/psfex_package/__init__.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -"""VIGNET MAKER PACKAGE -This package contains the module(s) for ``psfex``. +"""PSFEX SCRIPT PACKAGE +This package contains the module(s) for ``psfex_script``. :Authors: Axel Guinot, Tobias Liaudat """ -__all__ = ['psfex'] +__all__ = ['psfex_script'] diff --git a/shapepipe/modules/psfex_package/psfex.py b/shapepipe/modules/psfex_package/psfex_script.py similarity index 100% rename from shapepipe/modules/psfex_package/psfex.py rename to shapepipe/modules/psfex_package/psfex_script.py diff --git a/shapepipe/modules/psfex_runner.py b/shapepipe/modules/psfex_runner.py index b4cf0f605..6351d5ba8 100644 --- a/shapepipe/modules/psfex_runner.py +++ b/shapepipe/modules/psfex_runner.py @@ -12,7 +12,7 @@ import os from shapepipe.pipeline.execute import execute from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.psfex_package.psfex import PSFex_caller +from shapepipe.modules.psfex_package.psfex_script import PSFex_caller @module_runner( From 2c68c9024fd037e6d926826737b2bcd68a1d9940 Mon Sep 17 00:00:00 2001 From: andre zamoranovitorelli Date: Fri, 23 Jul 2021 09:59:23 +0200 Subject: [PATCH 12/16] requested changes --- shapepipe/modules/psfex_package/psfex_script.py | 1 + shapepipe/modules/psfex_runner.py | 15 +++------------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/shapepipe/modules/psfex_package/psfex_script.py b/shapepipe/modules/psfex_package/psfex_script.py index 8a087569c..7a650d97c 100644 --- a/shapepipe/modules/psfex_package/psfex_script.py +++ b/shapepipe/modules/psfex_package/psfex_script.py @@ -25,6 +25,7 @@ class PSFex_caller: List of check images Returns: + ------- command_line: str Command line with correct and complete paths for PSFEx execution. """ diff --git a/shapepipe/modules/psfex_runner.py b/shapepipe/modules/psfex_runner.py index 6351d5ba8..4b9323e7f 100644 --- a/shapepipe/modules/psfex_runner.py +++ b/shapepipe/modules/psfex_runner.py @@ -47,7 +47,7 @@ def psfex_runner( Returns: - + ------- stdout, stderr: str Strings with the output and error output of execution. @@ -92,15 +92,6 @@ def psfex_runner( stderr, stdout = execute(command_line) # move psfex errors reported as stdout to stderr - check_error = re.findall('error', stdout.lower()) - check_error2 = re.findall('all done', stdout.lower()) - - if check_error == []: - stderr2 = '' - else: - stderr2 = stdout - - if check_error2 == []: - stderr2 = stdout + stdout, stderr = PSFex_call.parse_errors(stderr, stdout) - return stdout, stderr2 + return stdout, stderr From 65bd772f3725c52a42aa5a1d4418928f2b5c3159 Mon Sep 17 00:00:00 2001 From: andre zamoranovitorelli Date: Fri, 23 Jul 2021 10:18:43 +0200 Subject: [PATCH 13/16] more requested changes --- .../modules/psfex_package/psfex_script.py | 24 +++++++------------ shapepipe/modules/psfex_runner.py | 24 +------------------ 2 files changed, 9 insertions(+), 39 deletions(-) diff --git a/shapepipe/modules/psfex_package/psfex_script.py b/shapepipe/modules/psfex_package/psfex_script.py index 7a650d97c..9bfc1203b 100644 --- a/shapepipe/modules/psfex_package/psfex_script.py +++ b/shapepipe/modules/psfex_package/psfex_script.py @@ -23,11 +23,6 @@ class PSFex_caller: Full path pf the output catalogue check_image_list: list of str List of check images - - Returns: - ------- - command_line: str - Command line with correct and complete paths for PSFEx execution. """ def __init__( @@ -47,7 +42,12 @@ def __init__( self.check_image_list = check_image_list def generate_command(self): - """ Generates a command line for running PSFEx """ + """ Generates a command line for running PSFEx + Returns: + ------- + command_line: str + Command line with correct and complete paths for PSFEx execution. + """ # Prepare command line command_line = ( @@ -78,7 +78,8 @@ def generate_command(self): f'{output_dir}' + f'/{suffix}' + f'{check_image.lower()}' - + f'{file_number_string}.fits') + + f'{file_number_string}.fits' + ) # Add checks to command line command_line += ( @@ -89,15 +90,6 @@ def generate_command(self): self.command_line = command_line return command_line - def exec(self): - """ - Executes the command line. - """ - self.generate_command() - stderr, stdout = execute(self.command_line) - - return stderr, stdout - def parse_errors(stderr, stdout): """ Move errors from output from psfex to errors. diff --git a/shapepipe/modules/psfex_runner.py b/shapepipe/modules/psfex_runner.py index 4b9323e7f..62b7b7aca 100644 --- a/shapepipe/modules/psfex_runner.py +++ b/shapepipe/modules/psfex_runner.py @@ -29,29 +29,7 @@ def psfex_runner( config, w_log ): - """ - Runs the psfex wrapper package. - - Parameters: - ---------- - input_file_list: list - List with the full path to the input file - run_dirs: dict - Dictionary with directories - file_number_string: str - Specific number of the data file being processed - config: - Config object - w_log: - Log object - - - Returns: - ------- - stdout, stderr: str - Strings with the output and error output of execution. - - """ + # extract psfex run configurations psfex_executable_path = config.getexpanded( From eb2363cf228603d597cca408021867151fce1f76 Mon Sep 17 00:00:00 2001 From: andre zamoranovitorelli Date: Fri, 23 Jul 2021 10:34:37 +0200 Subject: [PATCH 14/16] additional requested changes on docstrings and name of PSFEx --- .../modules/psfex_package/psfex_script.py | 29 +++++++++++++++---- shapepipe/modules/psfex_runner.py | 6 ++-- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/shapepipe/modules/psfex_package/psfex_script.py b/shapepipe/modules/psfex_package/psfex_script.py index 9bfc1203b..b3ad05014 100644 --- a/shapepipe/modules/psfex_package/psfex_script.py +++ b/shapepipe/modules/psfex_package/psfex_script.py @@ -7,8 +7,11 @@ import re -class PSFex_caller: - """This class contains functions to generate a PSFex command line. +class PSFEx_caller: + """ PSFEx caller + + This class contains functions to generate a PSFex command line. + Parameters: ---------- psfex_executable_path: str @@ -42,7 +45,10 @@ def __init__( self.check_image_list = check_image_list def generate_command(self): - """ Generates a command line for running PSFEx + """Generate command + + This method generates a command line for running PSFEx. + Returns: ------- command_line: str @@ -90,9 +96,22 @@ def generate_command(self): self.command_line = command_line return command_line + @staticmethod def parse_errors(stderr, stdout): - """ - Move errors from output from psfex to errors. + """Parse errors + + This methoid move errors from output from psfex to errors. + + Parameters: + ---------- + stderr, stdout: str + strings with outputs from the execute command + + Returns: + ------- + stdout, stderr2: str + + """ check_error = re.findall('error', stdout.lower()) check_error2 = re.findall('all done', stdout.lower()) diff --git a/shapepipe/modules/psfex_runner.py b/shapepipe/modules/psfex_runner.py index 62b7b7aca..a6efe6879 100644 --- a/shapepipe/modules/psfex_runner.py +++ b/shapepipe/modules/psfex_runner.py @@ -12,7 +12,7 @@ import os from shapepipe.pipeline.execute import execute from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.psfex_package.psfex_script import PSFex_caller +from shapepipe.modules.psfex_package.psfex_script import PSFEx_caller @module_runner( @@ -54,7 +54,7 @@ def psfex_runner( check_image_list = [''] # prepare the psfex command line - PSFex_call = PSFex_caller( + PSFEx_call = PSFEx_caller( psfex_executable_path, input_file_path, psfex_config_file, @@ -70,6 +70,6 @@ def psfex_runner( stderr, stdout = execute(command_line) # move psfex errors reported as stdout to stderr - stdout, stderr = PSFex_call.parse_errors(stderr, stdout) + stdout, stderr = PSFEx_call.parse_errors(stderr, stdout) return stdout, stderr From 7e77f046f061c97cce08d2abdb3653098e4f2d48 Mon Sep 17 00:00:00 2001 From: andre zamoranovitorelli Date: Fri, 23 Jul 2021 10:37:26 +0200 Subject: [PATCH 15/16] changing PSFex to PSFEx --- shapepipe/modules/psfex_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shapepipe/modules/psfex_runner.py b/shapepipe/modules/psfex_runner.py index a6efe6879..2284ed439 100644 --- a/shapepipe/modules/psfex_runner.py +++ b/shapepipe/modules/psfex_runner.py @@ -64,7 +64,7 @@ def psfex_runner( ) # generates the psfex command - command_line = PSFex_call.generate_command() + command_line = PSFEx_call.generate_command() w_log.info(f'Running command \'{command_line}\'') stderr, stdout = execute(command_line) From d5f5111aef6f696016958dc8727c4d1e765aebb1 Mon Sep 17 00:00:00 2001 From: andre zamoranovitorelli Date: Fri, 23 Jul 2021 11:00:56 +0200 Subject: [PATCH 16/16] style test changes --- shapepipe/modules/psfex_runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/shapepipe/modules/psfex_runner.py b/shapepipe/modules/psfex_runner.py index 2284ed439..948f066b7 100644 --- a/shapepipe/modules/psfex_runner.py +++ b/shapepipe/modules/psfex_runner.py @@ -30,7 +30,6 @@ def psfex_runner( w_log ): - # extract psfex run configurations psfex_executable_path = config.getexpanded( "PSFEX_RUNNER",