From 13d7d665cabba022c8de13dd76c7ed12bfbc0337 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Mon, 19 Jul 2021 17:41:46 +0200 Subject: [PATCH 01/15] get_images in revision --- .../modules/get_images_package/get_images.py | 300 +++++++++++++++++ shapepipe/modules/get_images_runner.py | 318 +++--------------- 2 files changed, 353 insertions(+), 265 deletions(-) create mode 100644 shapepipe/modules/get_images_package/get_images.py diff --git a/shapepipe/modules/get_images_package/get_images.py b/shapepipe/modules/get_images_package/get_images.py new file mode 100644 index 000000000..3c3bbeada --- /dev/null +++ b/shapepipe/modules/get_images_package/get_images.py @@ -0,0 +1,300 @@ +# -*- coding: utf-8 -*- + +"""GET IMAGES + +This module copies all images required for processing + +:Author: Martin Kilbinger + +:Date: 2019 - 2021 + +:Package: ShapePipe + +""" + +from shapepipe.modules.module_decorator import module_runner +from shapepipe.utilities.canfar import vosHandler + +import os +import re +import sys +import glob + + +def read_image_numbers(path): + """Read Image Numbers + Read image numbers from file. + + Parameters + ---------- + path : string + input file path + + Returns + ------- + image_number_list : list of string + image numbers + """ + + image_number_list = [] + with open(path) as f: + for line in f: + image_number_list.append(line.strip()) + + return image_number_list + + +def in2out_pattern(number): + """In2out Pattern + Transform input to output number pattern or image ID + + Parameters + ---------- + number : string + input number + + Returns + ------- + number_final : string + output number + """ + + # replace dots ('.') with dashes ('-') to avoid confusion + # with file extension delimiters + number_final = re.sub(r'\.', '-', number) + + # remove letters in number + number_final = re.sub('[a-zA-Z]', '', number_final) + + return number_final + + +class GetImages(object): + + def __init__( + self, + retrieve_method, + retrieve_options, + input_dir, + image_file_list, + input_numbering, + input_file_pattern, + input_file_ext, + output_file_pattern, + w_log, + check_existing_dir=None, + n_expected=None + ): + """Get Images + + Class handling retrieval of input images + + Parameters + ---------- + retrieve_method : string + copy/download method + retrieve_option : string + retrieve options + input_dir : string + input directory + input_file_list : list of string + input files + input_numbering : string + numbering scheme, python regexp + input_file_pattern : list of strings + file pattern including input number template of input files + input_file_ext : list of strings + input file extensions + output_file_pattern : list of strings + output file patterns + w_log : + log file + check_existing_dir : string, optional, default=None + if not None, only retrieve image if not existing at this path (recursively) + n_expected : int, optional, default=None + number of expected files per type and ID to download/check for existence + """ + + self._retrieve_method = retrieve_method + self._retrieve_options = retrieve_options + self._input_dir = input_dir + self._input_file_list = input_file_list + self._input_numbering = input_numbering + self._input_file_pattern = input_file_pattern + self._input_file_ext = input_file_ext + self._output_file_pattern = output_file_pattern + self._w_log = w_log + self._check_existing_dir = check_existing_dir + self._n_expected = n_expected + + def process(): + """Process + + Main function to process GetImages + """ + + # Input image numbers from all input tile files + all_image_numbers = [] + for input_file in self._input_file_list: + numbers_from_tile = read_image_numbers(input_file[0]) + all_image_numbers.append(numbers_from_tile) + + # List of unique input images + flat_list = [item for sublist in all_image_numbers for item in sublist] + self._w_log.info(f'Number of total image IDs = {len(flat_list)}') + + # Get unique number list + image_number_list = list(set(flat_list)) + self._w_log.info( + f'Number of unique image IDs = {len(image_number_list)}' + ) + + # Create array to make it compatible with input dir + nitem = len(input_dir) + if len(output_dir) == 1: + output_dir = [output_dir] * nitem + + # Check consistency of list lengths + if any(len(lst) != nitem for lst in [ + self._input_dir, + self._input_file_pattern, + self._input_file_ext, + output_file_pattern + ]: + raise ValueError( + f'Lists INPUT_PATH ({len(self._input_dir)}), ' + + f'INPUT_FILE_PATTERN ({len(self._input_file_pattern)}), ' + + f'INPUT_FILE_EXT ({llen(self._input_file_ext)}), ' + + f 'OUTPUT_FILE_PATTERN ({len(self._output_file_pattern)}) ' + + 'need to have equal length' + ) + + def get_file_list(self, dest_dir, use_output_file_pattern=False): + """Get File List + Return lists of file paths to retrieve. + + Parameters + ---------- + dest_dir : list of stringsr + input directory or url + use_output_file_pattern : bool, optional, default=False + if True, use output file base patterns excluding numbering scheme; + if False, use input file patterns + + Returns + ------- + list_all_files : list of list of strings + complete file paths, one list for each input file type + """ + + list_all_files = [] + for i in range(len(dest_dir)): + in_path = dest_dir[i] + in_pattern = self._input_file_pattern[i] + in_ext = self._input_file_ext[i] + + list_files_per_type = [] + for number in self._image_number_list: + + if use_output_file_pattern: + # Transform input to output number patterns + + number_final = in2out_pattern(number) + + # Keep initial dot in extension + x = in_ext[1:] + x2 = re.sub(r'\.', '', x) + ext_final = in_ext[0] + x2 + fbase = '{}{}'.format(sef._output_file_pattern[i], number_final) + else: + fbase = re.sub(self._input_numbering, number, in_pattern) + ext_final = in_ext + + if use_output_file_pattern and self._output_file_pattern[i] == '*': + # retrieve all input files to output dir, do not append + # extension + # fpath = '{}/.'.format(in_path) + fpath = in_path + else: + fpath = '{}/{}{}'.format(in_path, fbase, ext_final) + + list_files_per_type.append(fpath) + list_all_files.append(list_files_per_type) + + return list_all_files + + def retrieve(self, all_inputs, all_outputs): + """Retrieve + Retrieve all files. + + Parameters + ---------- + all_inputs: list of list of strings + input file paths, one list for each input file type + all_outputs: list of list of strings + output file paths, one list for each input file type + """ + + for in_per_type, out_per_type in zip(all_inputs, all_outputs): + for i in range(len(in_per_type)): + if self._check_existing_dir: + out_base = os.path.basename(in_per_type[i]) + path = glob.glob('{}/**/{}' + ''.format(self._check_existing_dir, + out_base), + recursive=True) + if path and len(path) == self._n_expected: + self._w_log.info('{} found, skipping' + ''.format(path[0])) + continue + self._w_log.info('Retrieving {}'.format(in_per_type[i])) + self.retrieve_one(in_per_type[i], out_per_type[i]) + + def retrieve_one(self, in_path, out_path): + """Retrieve One + Retrieve one file. + + Parameters + ---------- + in_path : string + input path + out_path : string + output path + """ + + if self._retrieve_method == 'vos': + sys.argv = [] + sys.argv.append('vcp') + if self._retrieve_options: + for opt in self._retrieve_options.split(' '): + sys.argv.append(opt) + sys.argv.append(in_path) + sys.argv.append(out_path) + + log_cmd = ' '.join(sys.argv) + self._w_log.info(f'Command \'{log_cmd}\'') + + vcp = vosHandler('vcp') + vcp() + + elif self._retrieve_method == 'symlink': + src = in_path + + # Get all input file names if INPUT_FILE_PATTERN contains '*' + all_src = glob.glob(src) + if len(all_src) == 0: + raise IndexError( + f'No input file found corresponding to \'{src}\'' + ) + + dst = out_path + for src in all_src: + if os.path.isdir(dst): + # OUTPUT_FILE_PATTERN is '*', so dst is not regular file + # but directory. Append input file name + dst_name = '{}/{}'.format(dst, os.path.basename(src)) + else: + # dst is regular file + dst_name = dst + os.symlink(src, dst_name) + diff --git a/shapepipe/modules/get_images_runner.py b/shapepipe/modules/get_images_runner.py index 1ac6f673c..46d5ac1c4 100644 --- a/shapepipe/modules/get_images_runner.py +++ b/shapepipe/modules/get_images_runner.py @@ -2,254 +2,48 @@ """GET IMAGES RUNNER -This module copies all images required for processing +Module runner for ``get_images`` :Author: Martin Kilbinger -""" - -from shapepipe.modules.module_decorator import module_runner -from shapepipe.utilities.canfar import vosHandler - -import os -import re -import sys -import glob - - -def in2out_pattern(number): - """Transform input to output number pattern or image ID - - Parameters - ---------- - number : string - input number - - Returns - ------- - number_final : string - output number - """ - - # replace dots ('.') with dashes ('-') to avoid confusion - # with file extension delimiters - number_final = re.sub(r'\.', '-', number) - - # remove letters in number - number_final = re.sub('[a-zA-Z]', '', number_final) - - return number_final - - -class GetImages(object): - - def __init__(self, copy, options, image_number_list, - input_numbering, input_file_pattern, - input_file_ext, w_log, check_existing_dir=None, - n_expected=None): - """GetImages initiatliation. - - Parameters - ---------- - copy : string - copy/download method - option : string - copy options - image_number_list : list of string self._copy = copy - image numbers self._options = options - input_numbering : string self._image_number_list = image_number_list - numbering scheme, python regexp self._input_numbering = input_numbering - input_file_pattern : list of strings self._input_file_ext = input_file_ext - file pattern including input number template of input files self._w_log = w_log - input_file_ext : list of strings - input file extensions - w_log : - log file - check_existing_dir : string, optional, default=None - if not None, only retrieve image if not existing at this path (recursively) - n_expected : int, optional, default=None - number of expected files per type and ID to download/check for existence - """ - - self._copy = copy - self._options = options - self._image_number_list = image_number_list - self._input_numbering = input_numbering - self._input_file_pattern = input_file_pattern - self._input_file_ext = input_file_ext - self._w_log = w_log - self._check_existing_dir = check_existing_dir - self._n_expected = n_expected - - def get_file_list(self, dest_dir, output_file_pattern=None): - """Return lists of file paths to copy. - - Parameters - ---------- - dest_dir: list of stringsr - input directory or url - output_file_pattern: list of strings, optional, default=None - output file base patterns excluding numbering scheme, - if None use input file patterns - - Returns - ------- - list_all_files: list of list of strings - complete file paths, one list for each input file type - """ - - list_all_files = [] - for i in range(len(dest_dir)): - in_path = dest_dir[i] - in_pattern = self._input_file_pattern[i] - in_ext = self._input_file_ext[i] - - list_files_per_type = [] - for number in self._image_number_list: - - if output_file_pattern: - # Transform input to output number patterns - - number_final = in2out_pattern(number) - - # Keep initial dot in extension - x = in_ext[1:] - x2 = re.sub(r'\.', '', x) - ext_final = in_ext[0] + x2 - fbase = '{}{}'.format(output_file_pattern[i], number_final) - else: - fbase = re.sub(self._input_numbering, number, in_pattern) - ext_final = in_ext - - if output_file_pattern and output_file_pattern[i] == '*': - # copy all input files to output dir, do not append - # extension - # fpath = '{}/.'.format(in_path) - fpath = in_path - else: - fpath = '{}/{}{}'.format(in_path, fbase, ext_final) - - list_files_per_type.append(fpath) - list_all_files.append(list_files_per_type) - - return list_all_files - - def copy(self, all_inputs, all_outputs): - """Copy all files. - - Parameters - ---------- - all_inputs: list of list of strings - input file paths, one list for each input file type - all_outputs: list of list of strings - output file paths, one list for each input file type - """ +:Date: 2019 - 2021 - for in_per_type, out_per_type in zip(all_inputs, all_outputs): - for i in range(len(in_per_type)): - if self._check_existing_dir: - out_base = os.path.basename(in_per_type[i]) - path = glob.glob('{}/**/{}' - ''.format(self._check_existing_dir, - out_base), - recursive=True) - if path and len(path) == self._n_expected: - self._w_log.info('{} found, skipping' - ''.format(path[0])) - continue - self._w_log.info('Retrieving {}'.format(in_per_type[i])) - self.copy_one(in_per_type[i], out_per_type[i]) +:Package: ShapePipe - def copy_one(self, in_path, out_path): - - if self._copy == 'vos': - sys.argv = [] - sys.argv.append('vcp') - if self._options: - for opt in self._options.split(' '): - sys.argv.append(opt) - sys.argv.append(in_path) - sys.argv.append(out_path) - - self._w_log.info('Command \'{}\'' - ''.format(' '.join(sys.argv))) - - vcp = vosHandler('vcp') - vcp() - - elif self._copy == 'symlink': - src = in_path - - # Get all input file names if INPUT_FILE_PATTERN contains '*' - all_src = glob.glob(src) - if len(all_src) == 0: - raise IndexError('No input file found corresponding to \'{}\'' - ''.format(src)) - - dst = out_path - for src in all_src: - if os.path.isdir(dst): - # OUTPUT_FILE_PATTERN is '*', so dst is not regular file - # but directory. Append input file name - dst_name = '{}/{}'.format(dst, os.path.basename(src)) - else: - # dst is regular file - dst_name = dst - os.symlink(src, dst_name) - - -def read_image_numbers(path): - """Read image numbers from file. - - Parameters - ---------- - path: string - input file path - - Returns - ------- - image_number_list: list of string - image numbers - """ - - image_number_list = [] - with open(path) as f: - for line in f: - image_number_list.append(line.strip()) - - return image_number_list +""" -@module_runner(version='1.0', - depends=['numpy'], - run_method='serial', - numbering_scheme='_0') -def get_images_runner(input_file_list, run_dirs, file_number_string, - config, w_log): +from shapepipe.modules.module_decorator import module_runner +from shapepipe.modules.get_images_runner import get_images as gi - # Input image numbers from all input tile files - all_image_numbers = [] - for input_file in input_file_list: - numbers_from_tile = read_image_numbers(input_file[0]) - all_image_numbers.append(numbers_from_tile) - flat_list = [item for sublist in all_image_numbers for item in sublist] - w_log.info('Number of images IDs = {}'.format(len(flat_list))) - # Get unique number list - image_number_list = list(set(flat_list)) - w_log.info('Number of unique image IDs = {}'.format(len(image_number_list))) +@module_runner( + version='1.0', + depends=['numpy'], + run_method='serial', + numbering_scheme='_0') +def get_images_runner( + input_file_list, + run_dirs, + file_number_string, + config, + w_log): # Read config file section - # Copying/download method - copy = config.get('GET_IMAGES_RUNNER', 'COPY') - copy_ok = ['vos', 'symlink'] - if copy not in copy_ok: - raise ValueError('key COPY={} is invalid, must be in {}'.format(copy, copy_ok)) + # Copy/download method + retrieve_method = config.get('GET_IMAGES_RUNNER', 'RETRIEVE') + retrieve_ok = ['vos', 'symlink'] + if retrieve_method not in retrieve_ok: + raise ValueError(f'key RETRIEVE={retrieve_method} is invalid, must be in {retrieve_ok}') + + if config.has_option('GET_IMAGES_RUNNER', 'RETRIEVE_OPTIONS'): + retrieve_options = config.getexpanded('GET_IMAGES_RUNNER', 'RETRIEVE_OPTIONS') + else: + retrieve_options = None # Paths input_dir = config.getlist('GET_IMAGES_RUNNER', 'INPUT_PATH') - nitem = len(input_dir) input_file_pattern = config.getlist('GET_IMAGES_RUNNER', 'INPUT_FILE_PATTERN') input_file_ext = config.getlist('GET_IMAGES_RUNNER', 'INPUT_FILE_EXT') output_file_pattern = config.getlist('GET_IMAGES_RUNNER', 'OUTPUT_FILE_PATTERN') @@ -261,32 +55,7 @@ def get_images_runner(input_file_list, run_dirs, file_number_string, else: output_dir = run_dirs['output'] - # Create array to make it compatible with input dir - output_dir = [output_dir] * nitem - - if any(len(lst) != nitem for lst in [input_dir, input_file_pattern, - input_file_ext, output_file_pattern]): - raise ValueError('Lists INPUT_PATH ({}), INPUT_FILE_PATTERN ({}), ' - - 'INPUT_FILE_EXT ({}), OUTPUT_FILE_PATTERN ({}) ' - 'need to have equal length' - ''.format(len(input_dir), - len(input_file_pattern), - len(input_file_ext), - len(output_file_pattern))) - - # Copying/download method - copy = config.get('GET_IMAGES_RUNNER', 'COPY') - copy_ok = ['vos', 'symlink'] - if copy not in copy_ok: - raise ValueError('key COPY={} is invalid, must be in {}'.format(copy, copy_ok)) - - if config.has_option('GET_IMAGES_RUNNER', 'COPY_OPTIONS'): - options = config.getexpanded('GET_IMAGES_RUNNER', 'COPY_OPTIONS') - else: - options = None - - # Check for already retrieved files + # Flags to check for already retrieved files if config.has_option('GET_IMAGES_RUNNER', 'CHECK_EXISTING_DIR'): check_existing_dir = config.getexpanded('GET_IMAGES_RUNNER', 'CHECK_EXISTING_DIR') if config.has_option('GET_IMAGES_RUNNER', 'N_EXPECTED'): @@ -297,13 +66,32 @@ def get_images_runner(input_file_list, run_dirs, file_number_string, check_existing_dir = None n_expected = None - inst = GetImages(copy, options, image_number_list, input_numbering, - input_file_pattern, input_file_ext, w_log, - check_existing_dir=check_existing_dir, n_expected=n_expected) + inst = gi.GetImages( + retrieve_method, + retrieve_options, + input_dir, + input_file_list, + input_numbering, + input_file_pattern, + input_file_ext, + output_file_pattern, + w_log, + check_existing_dir=check_existing_dir, + n_expected=n_expected + ) + + inst.process() + + nitem = len(input_dir) + + # Create array to make it compatible with input dir + if len(output_dir) == 1: + output_dir = [output_dir] * nitem + # Assemble input and output file lists - all_inputs = inst.get_file_list(input_dir) - all_outputs = inst.get_file_list(output_dir, output_file_pattern=output_file_pattern) - inst.copy(all_inputs, all_outputs) + all_inputs = inst.get_file_list(input_dir, output_file_pattern=False) + all_outputs = inst.get_file_list(output_dir, output_file_pattern=True) + inst.retrieve(all_inputs, all_outputs) return None, None From 8bd6537e230541a339756d2155e20f950891dbaf Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Tue, 20 Jul 2021 13:51:33 +0200 Subject: [PATCH 02/15] get_images revision cont --- .../modules/get_images_package/get_images.py | 29 ++++++++++++++----- shapepipe/modules/get_images_runner.py | 10 +------ 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/shapepipe/modules/get_images_package/get_images.py b/shapepipe/modules/get_images_package/get_images.py index 3c3bbeada..0be4181b8 100644 --- a/shapepipe/modules/get_images_package/get_images.py +++ b/shapepipe/modules/get_images_package/get_images.py @@ -95,8 +95,6 @@ def __init__( copy/download method retrieve_option : string retrieve options - input_dir : string - input directory input_file_list : list of string input files input_numbering : string @@ -117,7 +115,6 @@ def __init__( self._retrieve_method = retrieve_method self._retrieve_options = retrieve_options - self._input_dir = input_dir self._input_file_list = input_file_list self._input_numbering = input_numbering self._input_file_pattern = input_file_pattern @@ -127,10 +124,17 @@ def __init__( self._check_existing_dir = check_existing_dir self._n_expected = n_expected - def process(): + def process(input_dir, output_dir): """Process Main function to process GetImages + + Parameters + ---------- + input_dir : string + input directory + output_dir : string + output directory """ # Input image numbers from all input tile files @@ -150,25 +154,36 @@ def process(): ) # Create array to make it compatible with input dir - nitem = len(input_dir) + nitem = len(self._input_dir) + + # Make sure output_dir is list and compatible to input lists if len(output_dir) == 1: output_dir = [output_dir] * nitem # Check consistency of list lengths if any(len(lst) != nitem for lst in [ - self._input_dir, + input_dir, self._input_file_pattern, self._input_file_ext, output_file_pattern ]: raise ValueError( - f'Lists INPUT_PATH ({len(self._input_dir)}), ' + f'Lists INPUT_PATH ({len(input_dir)}), ' + f'INPUT_FILE_PATTERN ({len(self._input_file_pattern)}), ' + f'INPUT_FILE_EXT ({llen(self._input_file_ext)}), ' + f 'OUTPUT_FILE_PATTERN ({len(self._output_file_pattern)}) ' + 'need to have equal length' ) + # Assemble input and output file lists + all_inputs = self.get_file_list(input_dir, output_file_pattern=False) + all_outputs = self.get_file_list(output_dir, output_file_pattern=True) + + # Retrieve files + self.retrieve(all_inputs, all_outputs) + + + def get_file_list(self, dest_dir, use_output_file_pattern=False): """Get File List Return lists of file paths to retrieve. diff --git a/shapepipe/modules/get_images_runner.py b/shapepipe/modules/get_images_runner.py index 46d5ac1c4..a8b17427d 100644 --- a/shapepipe/modules/get_images_runner.py +++ b/shapepipe/modules/get_images_runner.py @@ -69,7 +69,6 @@ def get_images_runner( inst = gi.GetImages( retrieve_method, retrieve_options, - input_dir, input_file_list, input_numbering, input_file_pattern, @@ -80,14 +79,7 @@ def get_images_runner( n_expected=n_expected ) - inst.process() - - nitem = len(input_dir) - - # Create array to make it compatible with input dir - if len(output_dir) == 1: - output_dir = [output_dir] * nitem - + inst.process(input_dir, output_dir) # Assemble input and output file lists all_inputs = inst.get_file_list(input_dir, output_file_pattern=False) From 8eb4b75f0a48359e655829c892c4e6523dd937f6 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Tue, 20 Jul 2021 14:06:22 +0200 Subject: [PATCH 03/15] job_sp : temp change --- scripts/sh/job_sp.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sh/job_sp.bash b/scripts/sh/job_sp.bash index 2740a7363..06dffdbc3 100755 --- a/scripts/sh/job_sp.bash +++ b/scripts/sh/job_sp.bash @@ -311,7 +311,7 @@ if [[ $do_job != 0 ]]; then done ### Download config files - command_sp "$VCP vos:cfis/cosmostat/kilbinger/cfis ." "Get shapepipe config files" + #command_sp "$VCP vos:cfis/cosmostat/kilbinger/cfis ." "Get shapepipe config files" ### Get tiles command_sp "shapepipe_run -c $SP_CONFIG/config_get_tiles_$retrieve.ini" "Run shapepipe (get tiles)" From a4ccfa06b168bb4f70b7080a413c0bbd369b9199 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 20 Jul 2021 18:20:12 +0200 Subject: [PATCH 04/15] get_images works for vos --- example/cfis/config_get_exp_vos.ini | 4 +- example/cfis/config_get_tiles_symlink.ini | 4 +- example/cfis/config_get_tiles_vos.ini | 4 +- scripts/sh/job_sp.bash | 2 +- .../modules/get_images_package/get_images.py | 41 ++++++++++--------- shapepipe/modules/get_images_runner.py | 7 +--- 6 files changed, 30 insertions(+), 32 deletions(-) diff --git a/example/cfis/config_get_exp_vos.ini b/example/cfis/config_get_exp_vos.ini index 261c8a6cb..bc9b821a5 100644 --- a/example/cfis/config_get_exp_vos.ini +++ b/example/cfis/config_get_exp_vos.ini @@ -18,7 +18,7 @@ RUN_DATETIME = True [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = get_images_runner2 +MODULE = get_images_runner # Parallel processing mode, SMP or MPI MODE = SMP @@ -51,7 +51,7 @@ TIMEOUT = 96:00:00 ## Module options -[GET_IMAGES_RUNNER2] +[GET_IMAGES_RUNNER] FILE_PATTERN = exp_numbers diff --git a/example/cfis/config_get_tiles_symlink.ini b/example/cfis/config_get_tiles_symlink.ini index f629a22ab..2d9c96002 100644 --- a/example/cfis/config_get_tiles_symlink.ini +++ b/example/cfis/config_get_tiles_symlink.ini @@ -85,8 +85,8 @@ INPUT_NUMBERING = \d{3}\.\d{3} OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- # Copy/download method, one in 'vos', 'symlink' -COPY = symlink +RETRIEVE = symlink # Copy command options, optional. # -L: follow links -COPY_OPTIONS = -L +RETRIEVE_OPTIONS = -L diff --git a/example/cfis/config_get_tiles_vos.ini b/example/cfis/config_get_tiles_vos.ini index 442662599..9acb31c59 100644 --- a/example/cfis/config_get_tiles_vos.ini +++ b/example/cfis/config_get_tiles_vos.ini @@ -87,7 +87,7 @@ INPUT_NUMBERING = \d{3}\.\d{3} OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- # Copy/download method, one in 'vos', 'symlink' -COPY = vos +RETRIEVE = vos # Copy command options, optional -COPY_OPTIONS = --certfile=$VM_HOME/.ssl/cadcproxy.pem +RETRIEVE_OPTIONS = --certfile=$VM_HOME/.ssl/cadcproxy.pem diff --git a/scripts/sh/job_sp.bash b/scripts/sh/job_sp.bash index 2740a7363..06dffdbc3 100755 --- a/scripts/sh/job_sp.bash +++ b/scripts/sh/job_sp.bash @@ -311,7 +311,7 @@ if [[ $do_job != 0 ]]; then done ### Download config files - command_sp "$VCP vos:cfis/cosmostat/kilbinger/cfis ." "Get shapepipe config files" + #command_sp "$VCP vos:cfis/cosmostat/kilbinger/cfis ." "Get shapepipe config files" ### Get tiles command_sp "shapepipe_run -c $SP_CONFIG/config_get_tiles_$retrieve.ini" "Run shapepipe (get tiles)" diff --git a/shapepipe/modules/get_images_package/get_images.py b/shapepipe/modules/get_images_package/get_images.py index 0be4181b8..76846f77b 100644 --- a/shapepipe/modules/get_images_package/get_images.py +++ b/shapepipe/modules/get_images_package/get_images.py @@ -75,8 +75,7 @@ def __init__( self, retrieve_method, retrieve_options, - input_dir, - image_file_list, + input_file_list, input_numbering, input_file_pattern, input_file_ext, @@ -124,7 +123,7 @@ def __init__( self._check_existing_dir = check_existing_dir self._n_expected = n_expected - def process(input_dir, output_dir): + def process(self, input_dir, output_dir): """Process Main function to process GetImages @@ -154,43 +153,47 @@ def process(input_dir, output_dir): ) # Create array to make it compatible with input dir - nitem = len(self._input_dir) + nitem = len(input_dir) # Make sure output_dir is list and compatible to input lists - if len(output_dir) == 1: - output_dir = [output_dir] * nitem + #if len(output_dir) == 1: + output_dir = [output_dir] * nitem # Check consistency of list lengths - if any(len(lst) != nitem for lst in [ - input_dir, - self._input_file_pattern, - self._input_file_ext, - output_file_pattern - ]: + if any( + len(lst) != nitem for lst in [ + input_dir, + self._input_file_pattern, + self._input_file_ext, + self._output_file_pattern + ] + ): raise ValueError( f'Lists INPUT_PATH ({len(input_dir)}), ' + f'INPUT_FILE_PATTERN ({len(self._input_file_pattern)}), ' + f'INPUT_FILE_EXT ({llen(self._input_file_ext)}), ' - + f 'OUTPUT_FILE_PATTERN ({len(self._output_file_pattern)}) ' + + f'OUTPUT_FILE_PATTERN ({len(self._output_file_pattern)}) ' + 'need to have equal length' ) # Assemble input and output file lists - all_inputs = self.get_file_list(input_dir, output_file_pattern=False) - all_outputs = self.get_file_list(output_dir, output_file_pattern=True) + all_inputs = self.get_file_list(image_number_list, input_dir, use_output_file_pattern=False) + all_outputs = self.get_file_list(image_number_list, output_dir, use_output_file_pattern=True) # Retrieve files self.retrieve(all_inputs, all_outputs) - def get_file_list(self, dest_dir, use_output_file_pattern=False): + def get_file_list(self, image_number_list, dest_dir, use_output_file_pattern=False): """Get File List Return lists of file paths to retrieve. Parameters ---------- - dest_dir : list of stringsr + image_number_list : list of string + image numbers + dest_dir : list of string input directory or url use_output_file_pattern : bool, optional, default=False if True, use output file base patterns excluding numbering scheme; @@ -209,7 +212,7 @@ def get_file_list(self, dest_dir, use_output_file_pattern=False): in_ext = self._input_file_ext[i] list_files_per_type = [] - for number in self._image_number_list: + for number in image_number_list: if use_output_file_pattern: # Transform input to output number patterns @@ -220,7 +223,7 @@ def get_file_list(self, dest_dir, use_output_file_pattern=False): x = in_ext[1:] x2 = re.sub(r'\.', '', x) ext_final = in_ext[0] + x2 - fbase = '{}{}'.format(sef._output_file_pattern[i], number_final) + fbase = '{}{}'.format(self._output_file_pattern[i], number_final) else: fbase = re.sub(self._input_numbering, number, in_pattern) ext_final = in_ext diff --git a/shapepipe/modules/get_images_runner.py b/shapepipe/modules/get_images_runner.py index a8b17427d..28865103a 100644 --- a/shapepipe/modules/get_images_runner.py +++ b/shapepipe/modules/get_images_runner.py @@ -14,7 +14,7 @@ from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.get_images_runner import get_images as gi +from shapepipe.modules.get_images_package import get_images as gi @module_runner( @@ -81,9 +81,4 @@ def get_images_runner( inst.process(input_dir, output_dir) - # Assemble input and output file lists - all_inputs = inst.get_file_list(input_dir, output_file_pattern=False) - all_outputs = inst.get_file_list(output_dir, output_file_pattern=True) - inst.retrieve(all_inputs, all_outputs) - return None, None From fe08032c86d128a647700c87a36ed3d19bfd36e7 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Tue, 20 Jul 2021 19:31:17 +0200 Subject: [PATCH 05/15] config get exp symlink get_images_runner --- example/cfis/config_get_exp_symlink.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/cfis/config_get_exp_symlink.ini b/example/cfis/config_get_exp_symlink.ini index f05a88438..a14943256 100644 --- a/example/cfis/config_get_exp_symlink.ini +++ b/example/cfis/config_get_exp_symlink.ini @@ -18,7 +18,7 @@ RUN_DATETIME = True [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = get_images_runner2 +MODULE = get_images_runner # Parallel processing mode, SMP or MPI MODE = SMP @@ -51,7 +51,7 @@ TIMEOUT = 96:00:00 ## Module options -[GET_IMAGES_RUNNER2] +[GET_IMAGES_RUNNER] FILE_PATTERN = exp_numbers From f87f4b1408af98168bfdb37bc5699902309b7969 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Tue, 20 Jul 2021 19:39:31 +0200 Subject: [PATCH 06/15] removed get_images_runner2 --- example/cfis/config_exp_SpMh.ini | 2 +- example/cfis/config_tile_Ma.ini | 2 +- example/cfis/config_tile_MaSxMiViSmVi.ini | 4 +- example/cfis/config_tile_MaSxPiViSmVi.ini | 4 +- example/cfis/config_tile_Ng1u.ini | 96 -------- example/cfis/config_tile_Ng2u.ini | 96 -------- example/cfis/config_tile_Ng3u.ini | 96 -------- example/cfis/config_tile_Ng4u.ini | 96 -------- example/cfis/config_tile_Ng5u.ini | 95 -------- example/cfis/config_tile_Ng6u.ini | 96 -------- example/cfis/config_tile_Ng7u.ini | 96 -------- example/cfis/config_tile_Ng8u.ini | 95 -------- example/cfis/config_tile_Ng_template.ini | 2 +- example/cfis/config_tile_SxMiViSmVi.ini | 2 +- example/cfis/config_tile_SxPiViSmVi.ini | 2 +- shapepipe/modules/__init__.py | 1 - shapepipe/modules/get_images_runner2.py | 261 ---------------------- 17 files changed, 9 insertions(+), 1037 deletions(-) delete mode 100644 example/cfis/config_tile_Ng1u.ini delete mode 100644 example/cfis/config_tile_Ng2u.ini delete mode 100644 example/cfis/config_tile_Ng3u.ini delete mode 100644 example/cfis/config_tile_Ng4u.ini delete mode 100644 example/cfis/config_tile_Ng5u.ini delete mode 100644 example/cfis/config_tile_Ng6u.ini delete mode 100644 example/cfis/config_tile_Ng7u.ini delete mode 100644 example/cfis/config_tile_Ng8u.ini delete mode 100644 shapepipe/modules/get_images_runner2.py diff --git a/example/cfis/config_exp_SpMh.ini b/example/cfis/config_exp_SpMh.ini index 7d324a461..b2801ffdc 100644 --- a/example/cfis/config_exp_SpMh.ini +++ b/example/cfis/config_exp_SpMh.ini @@ -55,7 +55,7 @@ TIMEOUT = 96:00:00 [SPLIT_EXP_RUNNER] -INPUT_DIR = last:get_images_runner2 +INPUT_DIR = $SP_RUN/output/run_sp_Gie:get_images_runner # Matches compressed single-exposure files FILE_EXT = .fitsfz, .fitsfz, .fitsfz diff --git a/example/cfis/config_tile_Ma.ini b/example/cfis/config_tile_Ma.ini index 955a2bb35..6ad405f6b 100644 --- a/example/cfis/config_tile_Ma.ini +++ b/example/cfis/config_tile_Ma.ini @@ -54,7 +54,7 @@ TIMEOUT = 96:00:00 [MASK_RUNNER] # Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner, last:uncompress_fits_image_runner +INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner, last:uncompress_fits_image_runner # NUMBERING_SCHEME (optional) string with numbering pattern for input files NUMBERING_SCHEME = -000-000 diff --git a/example/cfis/config_tile_MaSxMiViSmVi.ini b/example/cfis/config_tile_MaSxMiViSmVi.ini index c6cd2a1c3..e2ac135ae 100644 --- a/example/cfis/config_tile_MaSxMiViSmVi.ini +++ b/example/cfis/config_tile_MaSxMiViSmVi.ini @@ -36,7 +36,7 @@ LOG_NAME = log_sp RUN_LOG_NAME = log_run_sp # Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner, last:uncompress_fits_image_runner +INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner, last:uncompress_fits_image_runner # Output directory OUTPUT_DIR = $SP_RUN/output @@ -81,7 +81,7 @@ SUFFIX = pipeline [SEXTRACTOR_RUNNER] -INPUT_DIR = last:get_images_runner, last:uncompress_fits_image_runner, last:mask_runner +INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner, last:uncompress_fits_image_runner, last:mask_runner FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag diff --git a/example/cfis/config_tile_MaSxPiViSmVi.ini b/example/cfis/config_tile_MaSxPiViSmVi.ini index ec3d08b64..de09a7af1 100644 --- a/example/cfis/config_tile_MaSxPiViSmVi.ini +++ b/example/cfis/config_tile_MaSxPiViSmVi.ini @@ -36,7 +36,7 @@ LOG_NAME = log_sp RUN_LOG_NAME = log_run_sp # Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner, last:uncompress_fits_image_runner +INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner, last:uncompress_fits_image_runner # Output directory OUTPUT_DIR = $SP_RUN/output @@ -81,7 +81,7 @@ SUFFIX = pipeline [SEXTRACTOR_RUNNER] -INPUT_DIR = last:get_images_runner, last:uncompress_fits_image_runner, last:mask_runner +INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner, last:uncompress_fits_image_runner, last:mask_runner FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag diff --git a/example/cfis/config_tile_Ng1u.ini b/example/cfis/config_tile_Ng1u.ini deleted file mode 100644 index 2747bf5ac..000000000 --- a/example/cfis/config_tile_Ng1u.ini +++ /dev/null @@ -1,96 +0,0 @@ -# ShapePipe configuration file for tiles: ngmix + KSB - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_ngmix_Ng1u - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = ngmix_runner, galsim_shapes_v2_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner - -FILE_PATTERN = sexcat_sexcat, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Model-fitting shapes with ngmix -[NGMIX_RUNNER] - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -INPUT_DIR = last:sextractor_runner,last:mccd_interp_runner,last:vignetmaker_runner2 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 0 -ID_OBJ_MAX = 3999 - - -# Moment-based (KSB) shapes with galsim -[GALSIM_SHAPES_V2_RUNNER] - -INPUT_DIR = last:sextractor_runner, last:vignetmaker_runner, last:mccd_interp_runner,last:vignetmaker_runner2 - -FILE_PATTERN = sexcat_sexcat, weight_vignet, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 0 -ID_OBJ_MAX = 3999 diff --git a/example/cfis/config_tile_Ng2u.ini b/example/cfis/config_tile_Ng2u.ini deleted file mode 100644 index b6efd8fce..000000000 --- a/example/cfis/config_tile_Ng2u.ini +++ /dev/null @@ -1,96 +0,0 @@ -# ShapePipe configuration file for tiles: ngmix + KSB - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_ngmix_Ng2u - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = ngmix_runner, galsim_shapes_v2_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner - -FILE_PATTERN = sexcat_sexcat, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Model-fitting shapes with ngmix -[NGMIX_RUNNER] - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -INPUT_DIR = last:sextractor_runner,last:mccd_interp_runner,last:vignetmaker_runner2 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 4000 -ID_OBJ_MAX = 7999 - - -# Moment-based (KSB) shapes with galsim -[GALSIM_SHAPES_V2_RUNNER] - -INPUT_DIR = last:sextractor_runner, last:vignetmaker_runner, last:mccd_interp_runner,last:vignetmaker_runner2 - -FILE_PATTERN = sexcat_sexcat, weight_vignet, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 4000 -ID_OBJ_MAX = 7999 diff --git a/example/cfis/config_tile_Ng3u.ini b/example/cfis/config_tile_Ng3u.ini deleted file mode 100644 index 95a999420..000000000 --- a/example/cfis/config_tile_Ng3u.ini +++ /dev/null @@ -1,96 +0,0 @@ -# ShapePipe configuration file for tiles: ngmix + KSB - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_ngmix_Ng3u - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = ngmix_runner, galsim_shapes_v2_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner - -FILE_PATTERN = sexcat_sexcat, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Model-fitting shapes with ngmix -[NGMIX_RUNNER] - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -INPUT_DIR = last:sextractor_runner,last:mccd_interp_runner,last:vignetmaker_runner2 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 8000 -ID_OBJ_MAX = 11999 - - -# Moment-based (KSB) shapes with galsim -[GALSIM_SHAPES_V2_RUNNER] - -INPUT_DIR = last:sextractor_runner, last:vignetmaker_runner, last:mccd_interp_runner,last:vignetmaker_runner2 - -FILE_PATTERN = sexcat_sexcat, weight_vignet, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 8000 -ID_OBJ_MAX = 11999 diff --git a/example/cfis/config_tile_Ng4u.ini b/example/cfis/config_tile_Ng4u.ini deleted file mode 100644 index b04d7b7cd..000000000 --- a/example/cfis/config_tile_Ng4u.ini +++ /dev/null @@ -1,96 +0,0 @@ -# ShapePipe configuration file for tiles: ngmix + KSB - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_ngmix_Ng4u - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = ngmix_runner, galsim_shapes_v2_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner - -FILE_PATTERN = sexcat_sexcat, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Model-fitting shapes with ngmix -[NGMIX_RUNNER] - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -INPUT_DIR = last:sextractor_runner,last:mccd_interp_runner,last:vignetmaker_runner2 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 12000 -ID_OBJ_MAX = 15999 - - -# Moment-based (KSB) shapes with galsim -[GALSIM_SHAPES_V2_RUNNER] - -INPUT_DIR = last:sextractor_runner, last:vignetmaker_runner, last:mccd_interp_runner,last:vignetmaker_runner2 - -FILE_PATTERN = sexcat_sexcat, weight_vignet, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 12000 -ID_OBJ_MAX = 15999 diff --git a/example/cfis/config_tile_Ng5u.ini b/example/cfis/config_tile_Ng5u.ini deleted file mode 100644 index 10775c241..000000000 --- a/example/cfis/config_tile_Ng5u.ini +++ /dev/null @@ -1,95 +0,0 @@ -# ShapePipe configuration file for tiles: ngmix + KSB - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_ngmix_Ng5u - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = ngmix_runner, galsim_shapes_v2_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner - -FILE_PATTERN = sexcat_sexcat, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Model-fitting shapes with ngmix -[NGMIX_RUNNER] - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -INPUT_DIR = last:sextractor_runner,last:mccd_interp_runner,last:vignetmaker_runner2 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 16000 -ID_OBJ_MAX = 19999 - -# Moment-based (KSB) shapes with galsim -[GALSIM_SHAPES_V2_RUNNER] - -INPUT_DIR = last:sextractor_runner, last:vignetmaker_runner, last:mccd_interp_runner,last:vignetmaker_runner2 - -FILE_PATTERN = sexcat_sexcat, weight_vignet, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 16000 -ID_OBJ_MAX = 19999 diff --git a/example/cfis/config_tile_Ng6u.ini b/example/cfis/config_tile_Ng6u.ini deleted file mode 100644 index a00826005..000000000 --- a/example/cfis/config_tile_Ng6u.ini +++ /dev/null @@ -1,96 +0,0 @@ -# ShapePipe configuration file for tiles: ngmix + KSB - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_ngmix_Ng6u - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = ngmix_runner, galsim_shapes_v2_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner - -FILE_PATTERN = sexcat_sexcat, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Model-fitting shapes with ngmix -[NGMIX_RUNNER] - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -INPUT_DIR = last:sextractor_runner,last:mccd_interp_runner,last:vignetmaker_runner2 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 20000 -ID_OBJ_MAX = 23999 - - -# Moment-based (KSB) shapes with galsim -[GALSIM_SHAPES_V2_RUNNER] - -INPUT_DIR = last:sextractor_runner, last:vignetmaker_runner, last:mccd_interp_runner,last:vignetmaker_runner2 - -FILE_PATTERN = sexcat_sexcat, weight_vignet, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 20000 -ID_OBJ_MAX = 23999 diff --git a/example/cfis/config_tile_Ng7u.ini b/example/cfis/config_tile_Ng7u.ini deleted file mode 100644 index 025b497f1..000000000 --- a/example/cfis/config_tile_Ng7u.ini +++ /dev/null @@ -1,96 +0,0 @@ -# ShapePipe configuration file for tiles: ngmix + KSB - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_ngmix_Ng7u - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = ngmix_runner, galsim_shapes_v2_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner - -FILE_PATTERN = sexcat_sexcat, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Model-fitting shapes with ngmix -[NGMIX_RUNNER] - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -INPUT_DIR = last:sextractor_runner,last:mccd_interp_runner,last:vignetmaker_runner2 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 24000 -ID_OBJ_MAX = 27999 - - -# Moment-based (KSB) shapes with galsim -[GALSIM_SHAPES_V2_RUNNER] - -INPUT_DIR = last:sextractor_runner, last:vignetmaker_runner, last:mccd_interp_runner,last:vignetmaker_runner2 - -FILE_PATTERN = sexcat_sexcat, weight_vignet, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 24000 -ID_OBJ_MAX = 27999 diff --git a/example/cfis/config_tile_Ng8u.ini b/example/cfis/config_tile_Ng8u.ini deleted file mode 100644 index 3e0dc5fec..000000000 --- a/example/cfis/config_tile_Ng8u.ini +++ /dev/null @@ -1,95 +0,0 @@ -# ShapePipe configuration file for tiles: ngmix + KSB - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_ngmix_Ng8u - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = ngmix_runner, galsim_shapes_v2_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner - -FILE_PATTERN = sexcat_sexcat, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Model-fitting shapes with ngmix -[NGMIX_RUNNER] - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -INPUT_DIR = last:sextractor_runner,last:mccd_interp_runner,last:vignetmaker_runner2 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 28000 -ID_OBJ_MAX = -1 - -# Moment-based (KSB) shapes with galsim -[GALSIM_SHAPES_V2_RUNNER] - -INPUT_DIR = last:sextractor_runner, last:vignetmaker_runner, last:mccd_interp_runner,last:vignetmaker_runner2 - -FILE_PATTERN = sexcat_sexcat, weight_vignet, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet - -FILE_EXT = .fits, .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = 28000 -ID_OBJ_MAX = -1 diff --git a/example/cfis/config_tile_Ng_template.ini b/example/cfis/config_tile_Ng_template.ini index eb37f47a6..cf307a50c 100644 --- a/example/cfis/config_tile_Ng_template.ini +++ b/example/cfis/config_tile_Ng_template.ini @@ -34,7 +34,7 @@ LOG_NAME = log_sp RUN_LOG_NAME = log_run_sp # Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner +INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner FILE_PATTERN = sexcat_sexcat, image_vignet, sexcat_background_vignet, galaxy_psf, weight_vignet, flag_vignet diff --git a/example/cfis/config_tile_SxMiViSmVi.ini b/example/cfis/config_tile_SxMiViSmVi.ini index ddf3c04ac..f454cbd00 100644 --- a/example/cfis/config_tile_SxMiViSmVi.ini +++ b/example/cfis/config_tile_SxMiViSmVi.ini @@ -57,7 +57,7 @@ TIMEOUT = 96:00:00 [SEXTRACTOR_RUNNER] -INPUT_DIR = last:get_images_runner, last:uncompress_fits_image_runner, $SP_RUN/output/run_sp_tile_Ma:mask_runner +INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner, last:uncompress_fits_image_runner, $SP_RUN/output/run_sp_tile_Ma:mask_runner FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag diff --git a/example/cfis/config_tile_SxPiViSmVi.ini b/example/cfis/config_tile_SxPiViSmVi.ini index 3f6f2ccaa..87fce9c1c 100644 --- a/example/cfis/config_tile_SxPiViSmVi.ini +++ b/example/cfis/config_tile_SxPiViSmVi.ini @@ -57,7 +57,7 @@ TIMEOUT = 96:00:00 [SEXTRACTOR_RUNNER] -INPUT_DIR = last:get_images_runner, last:uncompress_fits_image_runner, $SP_RUN/output/run_sp_tile_Ma:mask_runner +INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner, last:uncompress_fits_image_runner, $SP_RUN/output/run_sp_tile_Ma:mask_runner FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag diff --git a/shapepipe/modules/__init__.py b/shapepipe/modules/__init__.py index 3a2762ddd..d48a6a4f3 100644 --- a/shapepipe/modules/__init__.py +++ b/shapepipe/modules/__init__.py @@ -20,7 +20,6 @@ 'galsim_shapes_runner', 'galsim_shapes_v2_runner', 'get_images_runner', - 'get_images_runner2', 'make_catalog_runner', 'mask_runner', 'mask_runner_exp', diff --git a/shapepipe/modules/get_images_runner2.py b/shapepipe/modules/get_images_runner2.py deleted file mode 100644 index 371d7a755..000000000 --- a/shapepipe/modules/get_images_runner2.py +++ /dev/null @@ -1,261 +0,0 @@ -# -*- coding: utf-8 -*- - -"""GET IMAGES RUNNER - -This module copies all images required for processing - -:Author: Martin Kilbinger - -""" - -from shapepipe.modules.module_decorator import module_runner -from shapepipe.utilities.canfar import vosHandler - -import os -import re -import sys -import glob - - -class GetImages(object): - - def __init__(self, retrieve, options, image_number_list, - input_numbering, input_file_pattern, - input_file_ext, w_log, check_existing_dir=None): - """GetImages initialisation. - - Parameters - ---------- - retrieve : string - retrieve/download method - option : string - retrieve options - image_number_list : list of string - image numbers self._options = options - input_numbering : string - numbering scheme, python regexp - input_file_pattern : list of strings - file pattern including input number template of input files - input_file_ext : list of strings - input file extensions - w_log: - log file - check_existing_dir : string, optional, default=None - if not None, only retrieve image if not existing at this directory - - Returns - -------- - self: GetImages class instance - """ - - self._retrieve = retrieve - self._options = options - self._image_number_list = image_number_list - self._input_numbering = input_numbering - self._input_file_pattern = input_file_pattern - self._input_file_ext = input_file_ext - self._w_log = w_log - self._check_existing_dir = check_existing_dir - - def get_file_list(self, dest_dir, output_file_pattern=None): - """Return list of file paths to retrieve. - - Parameters - ---------- - dest_dir: list of stringsr - input directory or url - output_file_pattern: list of strings, optional, default=None - output file base patterns excluding numbering scheme, - if None use input file patterns - - Returns - ------- - list_all_files: list of list of strings - complete file paths, one list for each input file type - """ - - list_all_files = [] - for i in range(len(dest_dir)): - in_path = dest_dir[i] - in_pattern = self._input_file_pattern[i] - in_ext = self._input_file_ext[i] - - list_files_per_type = [] - for number in self._image_number_list: - - if output_file_pattern: - # For output: - # - replace dots ('.') with dashes ('-') to avoid confusion - # with file extension delimiters - # - remove letters in number - - number_final = re.sub(r'\.', '-', number) - number_final = re.sub('[a-zA-Z]', '', number_final) - - # Keep initial dot in extension - x = in_ext[1:] - x2 = re.sub(r'\.', '', x) - ext_final = in_ext[0] + x2 - fbase = '{}{}'.format(output_file_pattern[i], number_final) - else: - fbase = re.sub(self._input_numbering, number, in_pattern) - ext_final = in_ext - - # Remove 'p' for LSB images - # fbase = re.sub('p', '', fbase) - - fpath = '{}/{}{}'.format(in_path, fbase, ext_final) - list_files_per_type.append(fpath) - list_all_files.append(list_files_per_type) - - return list_all_files - - def retrieve(self, all_inputs, all_outputs): - """Retreve all files. - - Parameters - ---------- - all_inputs: list of list of strings - input file paths, one list for each input file type - all_outputs: list of list of strings - output file paths, one list for each input file type - """ - - for in_per_type, out_per_type in zip(all_inputs, all_outputs): - for i in range(len(in_per_type)): - if self._check_existing_dir: - out_base = os.path.basename(in_per_type[i]) - path = glob.glob('{}/**/{}' - ''.format(self._check_existing_dir, - out_base), - recursive=True) - if path: - self._w_log.info('{} found, skipping' - ''.format(path[0])) - continue - self._w_log.info('Retrieving {}'.format(in_per_type[i])) - self.retrieve_one(in_per_type[i], out_per_type[i]) - - def retrieve_one(self, in_path, out_path): - """Retrieve one file. - - Parameters - ---------- - in_path : string - input file path - out_path : string - output file path - """ - - if self._retrieve == 'vos': - sys.argv = [] - sys.argv.append('vcp') - if self._options: - for opt in self._options.split(' '): - sys.argv.append(opt) - sys.argv.append(in_path) - sys.argv.append(out_path) - - vcp = vosHandler('vcp') - vcp() - - elif self._retrieve == 'symlink': - src = in_path - dst = out_path - os.symlink(src, dst) - if not os.path.exists(src): - w_log.info('Warning: Source of symlink \'{}\' ' - 'does not exist' - ''.format(src)) - - -def read_image_numbers(path): - """Read image numbers from file. - - Parameters - ---------- - path : string - input file path - - Returns - ------- - image_number_list : list of string - image numbers - """ - - image_number_list = [] - with open(path) as f: - for line in f: - image_number_list.append(line.strip()) - - return image_number_list - - -@module_runner(version='1.0', - depends=['numpy'], - run_method='serial') -def get_images_runner2(input_file_list, run_dirs, file_number_string, - config, w_log): - - # Input image numbers from all input tile fils - all_image_numbers = [] - for input_file in input_file_list: - numbers_from_tile = read_image_numbers(input_file[0]) - all_image_numbers.append(numbers_from_tile) - flat_list = [item for sublist in all_image_numbers for item in sublist] - w_log.info('{} image numbers read in total'.format(len(flat_list))) - - # Get unique number list - image_number_list = list(set(flat_list)) - w_log.info('{} unique exposures numbers'.format(len(image_number_list))) - - # Read config file section - - # Paths - input_dir = config.getlist('GET_IMAGES_RUNNER2', 'INPUT_PATH') - nitem = len(input_dir) - input_file_pattern = config.getlist('GET_IMAGES_RUNNER2', 'INPUT_FILE_PATTERN') - input_file_ext = config.getlist('GET_IMAGES_RUNNER2', 'INPUT_FILE_EXT') - output_file_pattern = config.getlist('GET_IMAGES_RUNNER2', 'OUTPUT_FILE_PATTERN') - - input_numbering = config.get('GET_IMAGES_RUNNER2', 'INPUT_NUMBERING') - - if config.has_option('GET_IMAGES_RUNNER2', 'OUTPUT_PATH'): - output_dir = config.getexpanded('GET_IMAGES_RUNNER2', 'OUTPUT_PATH') - else: - output_dir = run_dirs['output'] - output_dir = [output_dir] * nitem - - if any(len(lst) != nitem for lst in [input_file_pattern, input_file_ext, - output_dir, output_file_pattern]): - raise ValueError('Lists INPUT_DIR, INPUT_FILE_PATTERN, INPUT_FILE_EXT, ' - 'OUTPUT_DIR, OUTPUT_FILE_PATTERN need to ' - 'have equal length') - - # Method to retrieve images - retrieve = config.get('GET_IMAGES_RUNNER2', 'RETRIEVE') - retrieve_ok = ['vos', 'symlink'] - if retrieve not in retrieve_ok: - raise ValueError('key RETRIEVE={} is invalid, must be in {}'.format(retrieve, retrieve_ok)) - - if config.has_option('GET_IMAGES_RUNNER2', 'RETRIEVE_OPTIONS'): - options = config.getexpanded('GET_IMAGES_RUNNER2', 'RETRIEVE_OPTIONS') - else: - options = None - - # Check for already retrieved files - if config.has_option('GET_IMAGES_RUNNER2', 'CHECK_EXISTING_DIR'): - check_existing_dir = config.getexpanded('GET_IMAGES_RUNNER2', 'CHECK_EXISTING_DIR') - else: - check_existing_dir = None - - inst = GetImages(retrieve, options, image_number_list, input_numbering, - input_file_pattern, input_file_ext, w_log, - check_existing_dir=check_existing_dir) - - # Assemble input and output file lists - all_inputs = inst.get_file_list(input_dir) - all_outputs = inst.get_file_list(output_dir, output_file_pattern=output_file_pattern) - inst.retrieve(all_inputs, all_outputs) - - return None, None From 3cba3a0fa3266875104eadff5588e731200f1bdc Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Tue, 20 Jul 2021 19:50:04 +0200 Subject: [PATCH 07/15] fixed uncomp config --- example/cfis/config_unfz_w.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/cfis/config_unfz_w.ini b/example/cfis/config_unfz_w.ini index 479df61eb..67b6a4e10 100644 --- a/example/cfis/config_unfz_w.ini +++ b/example/cfis/config_unfz_w.ini @@ -34,7 +34,7 @@ LOG_NAME = log_sp RUN_LOG_NAME = log_run_sp # Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner +INPUT_DIR = $SP_RUN/output/run_sp_Git:get_images_runner # Output directory OUTPUT_DIR = $SP_RUN/output From fbfaec363656884e62778098ca0df9e06e059d34 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Tue, 20 Jul 2021 19:53:11 +0200 Subject: [PATCH 08/15] job_sp script back to original --- scripts/sh/job_sp.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sh/job_sp.bash b/scripts/sh/job_sp.bash index 06dffdbc3..2740a7363 100755 --- a/scripts/sh/job_sp.bash +++ b/scripts/sh/job_sp.bash @@ -311,7 +311,7 @@ if [[ $do_job != 0 ]]; then done ### Download config files - #command_sp "$VCP vos:cfis/cosmostat/kilbinger/cfis ." "Get shapepipe config files" + command_sp "$VCP vos:cfis/cosmostat/kilbinger/cfis ." "Get shapepipe config files" ### Get tiles command_sp "shapepipe_run -c $SP_CONFIG/config_get_tiles_$retrieve.ini" "Run shapepipe (get tiles)" From dc93d5d9458ea568afc41223044318f60dff3a91 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Tue, 20 Jul 2021 19:59:31 +0200 Subject: [PATCH 09/15] passing tests --- .../modules/get_images_package/get_images.py | 30 ++++++++++------- shapepipe/modules/get_images_runner.py | 32 +++++++++++++++---- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/shapepipe/modules/get_images_package/get_images.py b/shapepipe/modules/get_images_package/get_images.py index 76846f77b..d0e60784e 100644 --- a/shapepipe/modules/get_images_package/get_images.py +++ b/shapepipe/modules/get_images_package/get_images.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - + """GET IMAGES This module copies all images required for processing @@ -107,9 +107,11 @@ def __init__( w_log : log file check_existing_dir : string, optional, default=None - if not None, only retrieve image if not existing at this path (recursively) + if not None, only retrieve image if not existing at this + path (recursively) n_expected : int, optional, default=None - number of expected files per type and ID to download/check for existence + number of expected files per type and ID to download/check for + existence """ self._retrieve_method = retrieve_method @@ -156,9 +158,8 @@ def process(self, input_dir, output_dir): nitem = len(input_dir) # Make sure output_dir is list and compatible to input lists - #if len(output_dir) == 1: output_dir = [output_dir] * nitem - + # Check consistency of list lengths if any( len(lst) != nitem for lst in [ @@ -177,14 +178,20 @@ def process(self, input_dir, output_dir): ) # Assemble input and output file lists - all_inputs = self.get_file_list(image_number_list, input_dir, use_output_file_pattern=False) - all_outputs = self.get_file_list(image_number_list, output_dir, use_output_file_pattern=True) + all_inputs = self.get_file_list( + image_number_list, + input_dir, + use_output_file_pattern=False + ) + all_outputs = self.get_file_list( + image_number_list, + output_dir, + use_output_file_pattern=True + ) # Retrieve files self.retrieve(all_inputs, all_outputs) - - def get_file_list(self, image_number_list, dest_dir, use_output_file_pattern=False): """Get File List Return lists of file paths to retrieve. @@ -223,7 +230,9 @@ def get_file_list(self, image_number_list, dest_dir, use_output_file_pattern=Fal x = in_ext[1:] x2 = re.sub(r'\.', '', x) ext_final = in_ext[0] + x2 - fbase = '{}{}'.format(self._output_file_pattern[i], number_final) + fbase = ( + f'{self._output_file_pattern[i],}{number_final}' + ) else: fbase = re.sub(self._input_numbering, number, in_pattern) ext_final = in_ext @@ -315,4 +324,3 @@ def retrieve_one(self, in_path, out_path): # dst is regular file dst_name = dst os.symlink(src, dst_name) - diff --git a/shapepipe/modules/get_images_runner.py b/shapepipe/modules/get_images_runner.py index 28865103a..2fb6d813b 100644 --- a/shapepipe/modules/get_images_runner.py +++ b/shapepipe/modules/get_images_runner.py @@ -27,7 +27,8 @@ def get_images_runner( run_dirs, file_number_string, config, - w_log): + w_log +): # Read config file section @@ -35,18 +36,32 @@ def get_images_runner( retrieve_method = config.get('GET_IMAGES_RUNNER', 'RETRIEVE') retrieve_ok = ['vos', 'symlink'] if retrieve_method not in retrieve_ok: - raise ValueError(f'key RETRIEVE={retrieve_method} is invalid, must be in {retrieve_ok}') + raise ValueError( + f'key RETRIEVE={retrieve_method} is invalid, must be in {retrieve_ok}' + ) if config.has_option('GET_IMAGES_RUNNER', 'RETRIEVE_OPTIONS'): - retrieve_options = config.getexpanded('GET_IMAGES_RUNNER', 'RETRIEVE_OPTIONS') + retrieve_options = config.getexpanded( + 'GET_IMAGES_RUNNER', + 'RETRIEVE_OPTIONS' + ) else: retrieve_options = None # Paths input_dir = config.getlist('GET_IMAGES_RUNNER', 'INPUT_PATH') - input_file_pattern = config.getlist('GET_IMAGES_RUNNER', 'INPUT_FILE_PATTERN') - input_file_ext = config.getlist('GET_IMAGES_RUNNER', 'INPUT_FILE_EXT') - output_file_pattern = config.getlist('GET_IMAGES_RUNNER', 'OUTPUT_FILE_PATTERN') + input_file_pattern = config.getlist( + 'GET_IMAGES_RUNNER', + 'INPUT_FILE_PATTERN' + ) + input_file_ext = config.getlist( + 'GET_IMAGES_RUNNER', + 'INPUT_FILE_EXT' + ) + output_file_pattern = config.getlist( + 'GET_IMAGES_RUNNER', + 'OUTPUT_FILE_PATTERN' + ) input_numbering = config.get('GET_IMAGES_RUNNER', 'INPUT_NUMBERING') @@ -57,7 +72,10 @@ def get_images_runner( # Flags to check for already retrieved files if config.has_option('GET_IMAGES_RUNNER', 'CHECK_EXISTING_DIR'): - check_existing_dir = config.getexpanded('GET_IMAGES_RUNNER', 'CHECK_EXISTING_DIR') + check_existing_dir = config.getexpanded( + 'GET_IMAGES_RUNNER', + 'CHECK_EXISTING_DIR' + ) if config.has_option('GET_IMAGES_RUNNER', 'N_EXPECTED'): n_expected = config.getint('GET_IMAGES_RUNNER', 'N_EXPECTED') else: From 94bfea965c493fc1d58eb0e20da93c0aa91c0244 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Wed, 21 Jul 2021 13:57:23 +0200 Subject: [PATCH 10/15] typo fixed in merge_final_cat python script --- scripts/python/merge_final_cat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/python/merge_final_cat.py b/scripts/python/merge_final_cat.py index 42ba95f99..be89824bb 100755 --- a/scripts/python/merge_final_cat.py +++ b/scripts/python/merge_final_cat.py @@ -305,7 +305,7 @@ def main(argv=None): print('Saving final catalogue') np.save('final_cat.npy', d) - mgs = '{} catalog files merged with success'.format(count) + msg = '{} catalog files merged with success'.format(count) if param.verbose: print(msg) print(msg, file=f_log) From cef00625078eac1e777b266df9cbf193e515b1e6 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Fri, 30 Jul 2021 10:15:40 +0200 Subject: [PATCH 11/15] f-string in script --- scripts/python/merge_final_cat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/python/merge_final_cat.py b/scripts/python/merge_final_cat.py index be89824bb..fd4b397a7 100755 --- a/scripts/python/merge_final_cat.py +++ b/scripts/python/merge_final_cat.py @@ -305,7 +305,7 @@ def main(argv=None): print('Saving final catalogue') np.save('final_cat.npy', d) - msg = '{} catalog files merged with success'.format(count) + msg = f'{count} catalog files merged with success' if param.verbose: print(msg) print(msg, file=f_log) From 3875d8105073b15a1928a52c367c4ffe590dc970 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Fri, 30 Jul 2021 10:17:03 +0200 Subject: [PATCH 12/15] comment -> str Co-authored-by: Samuel Farrens --- shapepipe/modules/get_images_package/get_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shapepipe/modules/get_images_package/get_images.py b/shapepipe/modules/get_images_package/get_images.py index d0e60784e..e37267760 100644 --- a/shapepipe/modules/get_images_package/get_images.py +++ b/shapepipe/modules/get_images_package/get_images.py @@ -90,7 +90,7 @@ def __init__( Parameters ---------- - retrieve_method : string + retrieve_method : str copy/download method retrieve_option : string retrieve options From 845c4c0eddf0ca5215b5d74d38085bdd825a11f0 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Fri, 30 Jul 2021 10:18:20 +0200 Subject: [PATCH 13/15] comment -> str Co-authored-by: Samuel Farrens --- shapepipe/modules/get_images_package/get_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shapepipe/modules/get_images_package/get_images.py b/shapepipe/modules/get_images_package/get_images.py index e37267760..df1970c7d 100644 --- a/shapepipe/modules/get_images_package/get_images.py +++ b/shapepipe/modules/get_images_package/get_images.py @@ -92,7 +92,7 @@ def __init__( ---------- retrieve_method : str copy/download method - retrieve_option : string + retrieve_option : str retrieve options input_file_list : list of string input files From 73afab7a4fea2108f88847ac7c4defa7f7b5ff0c Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Fri, 30 Jul 2021 10:19:58 +0200 Subject: [PATCH 14/15] comments: string -> str --- scripts/python/merge_final_cat.py | 12 +++--- .../modules/get_images_package/get_images.py | 42 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/scripts/python/merge_final_cat.py b/scripts/python/merge_final_cat.py index fd4b397a7..dbdf1f575 100755 --- a/scripts/python/merge_final_cat.py +++ b/scripts/python/merge_final_cat.py @@ -74,8 +74,8 @@ def parse_options(p_def): ------- options: tuple Command line options - args: string - Command line string + args: str + Command line str """ usage = "%prog [OPTIONS]" @@ -150,14 +150,14 @@ def read_param_file(path, verbose=False): Parameters ---------- - path: string + path: str input file name verbose: bool, optional, default=False verbose output if True Returns ------- - param_list: list of strings + param_list: list of str parameter names """ @@ -203,11 +203,11 @@ def get_data(path, hdu_num, param_list): Parameters ---------- - path: string + path: str input file name hdu_num: int HDU number - param_list: list of strings + param_list: list of str parameters to be extracted. If none, copy all columns diff --git a/shapepipe/modules/get_images_package/get_images.py b/shapepipe/modules/get_images_package/get_images.py index d0e60784e..a6ec55186 100644 --- a/shapepipe/modules/get_images_package/get_images.py +++ b/shapepipe/modules/get_images_package/get_images.py @@ -27,12 +27,12 @@ def read_image_numbers(path): Parameters ---------- - path : string + path : str input file path Returns ------- - image_number_list : list of string + image_number_list : list of str image numbers """ @@ -50,12 +50,12 @@ def in2out_pattern(number): Parameters ---------- - number : string + number : str input number Returns ------- - number_final : string + number_final : str output number """ @@ -90,23 +90,23 @@ def __init__( Parameters ---------- - retrieve_method : string + retrieve_method : str copy/download method - retrieve_option : string + retrieve_option : str retrieve options - input_file_list : list of string + input_file_list : list of str input files - input_numbering : string + input_numbering : str numbering scheme, python regexp - input_file_pattern : list of strings + input_file_pattern : list of str file pattern including input number template of input files - input_file_ext : list of strings + input_file_ext : list of str input file extensions - output_file_pattern : list of strings + output_file_pattern : list of str output file patterns w_log : log file - check_existing_dir : string, optional, default=None + check_existing_dir : str, optional, default=None if not None, only retrieve image if not existing at this path (recursively) n_expected : int, optional, default=None @@ -132,9 +132,9 @@ def process(self, input_dir, output_dir): Parameters ---------- - input_dir : string + input_dir : str input directory - output_dir : string + output_dir : str output directory """ @@ -198,9 +198,9 @@ def get_file_list(self, image_number_list, dest_dir, use_output_file_pattern=Fal Parameters ---------- - image_number_list : list of string + image_number_list : list of str image numbers - dest_dir : list of string + dest_dir : list of str input directory or url use_output_file_pattern : bool, optional, default=False if True, use output file base patterns excluding numbering scheme; @@ -208,7 +208,7 @@ def get_file_list(self, image_number_list, dest_dir, use_output_file_pattern=Fal Returns ------- - list_all_files : list of list of strings + list_all_files : list of list of str complete file paths, one list for each input file type """ @@ -256,9 +256,9 @@ def retrieve(self, all_inputs, all_outputs): Parameters ---------- - all_inputs: list of list of strings + all_inputs: list of list of str input file paths, one list for each input file type - all_outputs: list of list of strings + all_outputs: list of list of str output file paths, one list for each input file type """ @@ -283,9 +283,9 @@ def retrieve_one(self, in_path, out_path): Parameters ---------- - in_path : string + in_path : str input path - out_path : string + out_path : str output path """ From 0aca5cfdf01db2f7069dfa031d750020dc41501b Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Fri, 30 Jul 2021 10:22:30 +0200 Subject: [PATCH 15/15] i -> idx in loops --- .../modules/get_images_package/get_images.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/shapepipe/modules/get_images_package/get_images.py b/shapepipe/modules/get_images_package/get_images.py index a6ec55186..c02b08276 100644 --- a/shapepipe/modules/get_images_package/get_images.py +++ b/shapepipe/modules/get_images_package/get_images.py @@ -213,10 +213,10 @@ def get_file_list(self, image_number_list, dest_dir, use_output_file_pattern=Fal """ list_all_files = [] - for i in range(len(dest_dir)): - in_path = dest_dir[i] - in_pattern = self._input_file_pattern[i] - in_ext = self._input_file_ext[i] + for idx in range(len(dest_dir)): + in_path = dest_dir[idx] + in_pattern = self._input_file_pattern[idx] + in_ext = self._input_file_ext[idx] list_files_per_type = [] for number in image_number_list: @@ -263,9 +263,9 @@ def retrieve(self, all_inputs, all_outputs): """ for in_per_type, out_per_type in zip(all_inputs, all_outputs): - for i in range(len(in_per_type)): + for idx in range(len(in_per_type)): if self._check_existing_dir: - out_base = os.path.basename(in_per_type[i]) + out_base = os.path.basename(in_per_type[idx]) path = glob.glob('{}/**/{}' ''.format(self._check_existing_dir, out_base), @@ -274,8 +274,8 @@ def retrieve(self, all_inputs, all_outputs): self._w_log.info('{} found, skipping' ''.format(path[0])) continue - self._w_log.info('Retrieving {}'.format(in_per_type[i])) - self.retrieve_one(in_per_type[i], out_per_type[i]) + self._w_log.info('Retrieving {}'.format(in_per_type[idx])) + self.retrieve_one(in_per_type[i], out_per_type[idx]) def retrieve_one(self, in_path, out_path): """Retrieve One