user-friendlyer job runs

example/cfis/config_GitFeGie_symlink.ini

@@ -0,0 +1,144 @@
+# ShapePipe configuration file for: get images
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = False
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_GitFeGie
+
+# Add date and time to RUN_NAME, optional, default: False
+RUN_DATETIME = True
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = get_images_runner, find_exposures_runner, get_images_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = $SP_RUN
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 1
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+# Get tiles
+[GET_IMAGES_RUNNER_RUN_1]
+
+FILE_PATTERN = tile_numbers
+
+FILE_EXT = .txt
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = 
+
+# Paths
+
+# Input path where original images are stored. Can be local path or vos url.
+# Single string or list of strings
+INPUT_PATH = $HOME/astro/data/CFIS/tiles_DR2, $HOME/astro/data/CFIS/tiles_DR2
+
+# Input file pattern including tile number as dummy template
+INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight
+
+# Input file extensions
+INPUT_FILE_EXT = .fits, .fits.fz
+
+# Input numbering scheme, python regexp
+INPUT_NUMBERING = \d{3}\.\d{3}
+
+# Output file pattern without number
+OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight-
+
+# Copy/download method, one in 'vos', 'symlink'
+RETRIEVE = symlink
+
+# Copy command options, optional
+RETRIEVE_OPTIONS = -L
+
+
+[FIND_EXPOSURES_RUNNER]
+
+INPUT_MODULE = get_images_runner_run_1
+
+FILE_PATTERN = CFIS_image
+
+FILE_EXT = .fits
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+
+# Get exposures
+[GET_IMAGES_RUNNER_RUN_2]
+
+INPUT_MODULE = find_exposures_runner
+
+FILE_PATTERN = exp_numbers
+
+FILE_EXT = .txt
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+
+# Paths
+
+# Output path (optional, default is [FILE]:OUTPUT_DIR
+# OUTPUT_PATH = input_images
+
+# Input path where original images are stored. Can be local path or vos url.
+# Single string or list of strings
+INPUT_PATH = $HOME/astro/data/CFIS/pitcairn_DR2, $HOME/astro/data/CFIS/weights_DR2, $HOME/astro/data/CFIS/flags_DR2
+
+# Input file pattern including tile number as dummy template
+INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag
+
+# Input file extensions
+INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz
+
+# Input numbering scheme, python regexp
+INPUT_NUMBERING = \d{6}
+
+# Output file pattern without number
+OUTPUT_FILE_PATTERN = image-, weight-, flag-
+
+# Method to retrieve images, one in 'vos', 'symlink'
+RETRIEVE = symlink
+
+# If RETRIEVE=vos, number of attempts to download
+# Optional, default=3
+N_TRY = 3
+
+# Retrieve command options, optional
+RETRIEVE_OPTIONS = -L

example/cfis/config_GitFeGie_vos.ini

@@ -0,0 +1,149 @@
+# ShapePipe configuration file for: get images
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = False
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_GitFeGie
+
+# Add date and time to RUN_NAME, optional, default: False
+RUN_DATETIME = True
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = get_images_runner, find_exposures_runner, get_images_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = $SP_RUN
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 1
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+# Get tiles
+[GET_IMAGES_RUNNER_RUN_1]
+
+FILE_PATTERN = tile_numbers
+
+FILE_EXT = .txt
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = 
+
+# Paths
+
+# Input path where original images are stored. Can be local path or vos url.
+# Single string or list of strings
+INPUT_PATH = vos:cfis/tiles_DR3, vos:cfis/tiles_DR3
+
+# Input file pattern including tile number as dummy template
+INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight
+
+# Input file extensions
+INPUT_FILE_EXT = .fits, .fits.fz
+
+# Input numbering scheme, python regexp
+INPUT_NUMBERING = \d{3}\.\d{3}
+
+# Output file pattern without number
+OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight-
+
+# Copy/download method, one in 'vos', 'symlink'
+RETRIEVE = vos
+
+# Copy command options, optional
+RETRIEVE_OPTIONS = --certfile=$VM_HOME/.ssl/cadcproxy.pem
+
+
+[FIND_EXPOSURES_RUNNER]
+
+INPUT_MODULE = get_images_runner_run_1
+
+FILE_PATTERN = CFIS_image
+
+FILE_EXT = .fits
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+
+# Get exposures
+[GET_IMAGES_RUNNER_RUN_2]
+
+INPUT_DIR = last:find_exposures_runner
+
+FILE_PATTERN = exp_numbers
+
+FILE_EXT = .txt
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+
+# Paths
+
+# Output path (optional, default is [FILE]:OUTPUT_DIR
+# OUTPUT_PATH = input_images
+
+# Input path where original images are stored. Can be local path or vos url.
+# Single string or list of strings
+INPUT_PATH = vos:cfis/pitcairn, vos:cfis/weights, vos:cfis/flags
+# LSB images:
+#INPUT_PATH = vos:cfis/lsb_individual, vos:cfis/weights, vos:cfis/flags
+
+# Input file pattern including tile number as dummy template
+INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag
+# LSB images
+#INPUT_FILE_PATTERN = 000000s, 000000p.weight, 000000p.flag
+
+# Input file extensions
+INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz
+
+# Input numbering scheme, python regexp
+INPUT_NUMBERING = \d{6}
+
+# Output file pattern without number
+OUTPUT_FILE_PATTERN = image-, weight-, flag-
+
+# Method to retrieve images, one in 'vos', 'symlink'
+RETRIEVE = vos
+
+# If RETRIEVE=vos, number of attempts to download
+# Optional, default=3
+N_TRY = 3
+
+# Retrieve command options, optional
+RETRIEVE_OPTIONS = --certfile=$VM_HOME/.ssl/cadcproxy.pem
+