From c16fd9946f6ebcab6bd4571fac7dbbefa97bfbce Mon Sep 17 00:00:00 2001 From: Samuel Farrens Date: Fri, 7 May 2021 09:45:31 +0200 Subject: [PATCH 1/4] fixed style errors --- shapepipe/modules/galsim_shapes_v2_runner.py | 6 ++++-- shapepipe/utilities/canfar.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/shapepipe/modules/galsim_shapes_v2_runner.py b/shapepipe/modules/galsim_shapes_v2_runner.py index a09864673..503178e1e 100644 --- a/shapepipe/modules/galsim_shapes_v2_runner.py +++ b/shapepipe/modules/galsim_shapes_v2_runner.py @@ -712,8 +712,10 @@ def process(tile_cat_path, tile_weight_path, gal_vignet_path, bkg_vignet_path, if skip_col: # Skip this object if marked, also if mark is nan - if tile_skip[i_tile] == skip_val \ - or (np.isnan(tile_skip[i_tile]) and np.isnan(skip_val)): + if ( + tile_skip[i_tile] == skip_val + or (np.isnan(tile_skip[i_tile]) and np.isnan(skip_val)) + ): continue count = count + 1 diff --git a/shapepipe/utilities/canfar.py b/shapepipe/utilities/canfar.py index d5dcf712b..a5fca1849 100644 --- a/shapepipe/utilities/canfar.py +++ b/shapepipe/utilities/canfar.py @@ -9,7 +9,7 @@ """ import os -import sys +import sys try: import vos.commands as vosc @@ -92,6 +92,7 @@ def __call__(self, *args, **kwargs): raise vosError('Error in VOs command: {}' ''.format(self._command.__name__)) + def download(source, target, verbose=False): """Download file from vos. From 82b0e0b57d91cca2d372e3af9db0b13d6ca3ce88 Mon Sep 17 00:00:00 2001 From: Samuel Farrens Date: Fri, 7 May 2021 09:50:47 +0200 Subject: [PATCH 2/4] added test run of pipeline --- .github/workflows/test.yml | 1 + example/config.ini | 2 +- shapepipe/modules/python_example.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38d10373b..95d9f21a6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,3 +34,4 @@ jobs: run: | conda activate shapepipe python setup.py test + shapepipe_run -c example/config.ini diff --git a/example/config.ini b/example/config.ini index 915143b23..c03798ed2 100644 --- a/example/config.ini +++ b/example/config.ini @@ -42,7 +42,7 @@ NUMBERING_SCHEME = _0 ## ShapePipe job handling options [JOB] # BATCH_SIZE (optional) default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 3 +SMP_BATCH_SIZE = 4 # SMP_BACKEND (opional) options are loky, multiprocessing or threading, default is loky ; SMP_BACKEND = multiprocessing # TIMEOUT (optional) default is None, i.e. no timeout limit applied diff --git a/shapepipe/modules/python_example.py b/shapepipe/modules/python_example.py index 39e63d1d0..283c77dca 100644 --- a/shapepipe/modules/python_example.py +++ b/shapepipe/modules/python_example.py @@ -12,6 +12,20 @@ from numpy.random import randint from shapepipe.modules.module_decorator import module_runner +# Test import of key dependencies +import astropy +import galsim +import joblib +import mccd +import ngmix +import pandas +import pysap +import scipy +import sf_tools +import sip_tpv +import sqlitedict +import treecorr + class Dummy(object): From b18c257876f6bb2c8795ea24f03d4e9bbc40cf66 Mon Sep 17 00:00:00 2001 From: Samuel Farrens Date: Fri, 7 May 2021 09:51:44 +0200 Subject: [PATCH 3/4] trigger CI on PRs to develop --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 95d9f21a6..d3e7e90df 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - master + - develop jobs: From c105a9a97941edf624179ab718e5666dd51f9d1c Mon Sep 17 00:00:00 2001 From: Samuel Farrens Date: Fri, 7 May 2021 10:11:47 +0200 Subject: [PATCH 4/4] set default example batch size to 2 --- example/config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/config.ini b/example/config.ini index c03798ed2..b6dc05e3d 100644 --- a/example/config.ini +++ b/example/config.ini @@ -42,7 +42,7 @@ NUMBERING_SCHEME = _0 ## ShapePipe job handling options [JOB] # BATCH_SIZE (optional) default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 +SMP_BATCH_SIZE = 2 # SMP_BACKEND (opional) options are loky, multiprocessing or threading, default is loky ; SMP_BACKEND = multiprocessing # TIMEOUT (optional) default is None, i.e. no timeout limit applied