From 1d4b4d19ebeb40ff53592b92744594789495e132 Mon Sep 17 00:00:00 2001 From: Emir Karamehmetoglu Date: Mon, 16 May 2022 22:56:55 +0200 Subject: [PATCH 1/6] Add N tries while increasing theshold Because no way to catch sep bare C exception, we have to do generic exception catching. --- flows/reference_cleaning.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/flows/reference_cleaning.py b/flows/reference_cleaning.py index 839c839..67b74cd 100644 --- a/flows/reference_cleaning.py +++ b/flows/reference_cleaning.py @@ -88,11 +88,22 @@ def add_target(self, target: Target, starid: int = 0) -> None: self.xy = np.vstack(((target.pixel_column, target.pixel_row), self.xy)) -def use_sep(image: FlowsImage): +def use_sep(image: FlowsImage, tries: int = 5, thresh: float = 5.): + # Use sep to for soure extraction sep_background = sep.Background(image.image, mask=image.mask) - objects = sep.extract(image.image - sep_background, thresh=5., err=sep_background.globalrms, mask=image.mask, - deblend_cont=0.1, minarea=9, clean_param=2.0) + try: + objects = sep.extract(image.image - sep_background, thresh=thresh, err=sep_background.globalrms, + mask=image.mask, deblend_cont=0.1, minarea=9, clean_param=2.0) + except KeyboardInterrupt: + raise + except Exception: + logger.warning("SEP failed, trying again...") + if tries > 0: + thresh += 3 + return use_sep(image, tries - 1, thresh * 2) + else: + raise return References(table=objects) From fe91633f809fbaecae03ed9b2bda8d5e79bb6785 Mon Sep 17 00:00:00 2001 From: Emir Karamehmetoglu Date: Mon, 16 May 2022 23:46:15 +0200 Subject: [PATCH 2/6] minor update to readme mostly to bump tests. --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index cba0a52..1cfcd37 100644 --- a/README.rst +++ b/README.rst @@ -20,9 +20,10 @@ Installation instructions >>> git clone https://github.com/SNflows/flows.git . -* All dependencies can be installed using the following command. It is recommended to do this in a dedicated `virtualenv `_ or similar: +* Required dependencies can be installed using the following command. It is recommended to do this in a dedicated `virtualenv `_ or similar: >>> pip install -r requirements.txt + >>> pip install -r requirements_dev.txt # for tests/development * Last step is to create a config-file. Create a file named "config.ini" and place it in the "flows" directory. Make sure that the file can only be read by you (``chmod 0600 config.ini``)! This file can contain all the settings for running the pipeline. A minimal file for working with the pipeline is From dbb3aef89958a11aa26f3e663a0ba573d69c63df Mon Sep 17 00:00:00 2001 From: Emir Karamehmetoglu Date: Mon, 16 May 2022 23:48:55 +0200 Subject: [PATCH 3/6] update tests to ignore G004 It's already optimized enough. --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bfa8018..4276869 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,7 +46,7 @@ jobs: run: | # stop the build if there are Python syntax errors or undefined names # For some reason we have to specifically ignore G001 as well - flake8 --select=E9,F63,F7,F82 --ignore=G001 --show-source + flake8 --select=E9,F63,F7,F82 --ignore=G001 --ignore=G004 --show-source # exit-zero treats all errors as warnings. flake8 --exit-zero From 2feae1d18fa774cb87286fb988bae6dc4215bc59 Mon Sep 17 00:00:00 2001 From: Emir Karamehmetoglu Date: Mon, 16 May 2022 23:57:46 +0200 Subject: [PATCH 4/6] make flake8 config compliant with inline comment rules --- setup.cfg | 48 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/setup.cfg b/setup.cfg index e18cff9..1ca59e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,22 +9,38 @@ extend-ignore = E203 enable-extensions = G ignore = - G004, # logger does not optimizes f strings but __str__ calls are ok for us. - E117, # over-indented (set when using tabs) - E127, # continuation line over-indented for visual indent - E128, # continuation line under-indented for visual indent - E201, # whitespace after '(' - E202, # whitespace before ')' - E265, # block comment should start with '# ' - E231, # missing whitespace after ',' - E226, # missing whitespace around arithmetic operator - E261, # at least two spaces before inline comment - E302, # expected 2 blank lines, found 1 - E305, # expected 2 blank lines after class or function definition, found 1 - E501, # line too long - E701, # multiple statements on one line (colon) - W503, # Line break occurred before a binary operator - ET128, # (flake8-tabs) unexpected number of tabs and spaces at start of XXX + G004, + # logger does not optimizes f strings but __str__ calls are ok for us. + E117, + # over-indented (set when using tabs) + E127, + # continuation line over-indented for visual indent + E128, + # continuation line under-indented for visual indent + E201, + # whitespace after '(' + E202, + # whitespace before ')' + E265, + # block comment should start with '# ' + E231, + # missing whitespace after ',' + E226, + # missing whitespace around arithmetic operator + E261, + # at least two spaces before inline comment + E302, + # expected 2 blank lines, found 1 + E305, + # expected 2 blank lines after class or function definition, found 1 + E501, + # line too long + E701, + # multiple statements on one line (colon) + W503, + # Line break occurred before a binary operator + ET128, + # (flake8-tabs) unexpected number of tabs and spaces at start of XXX [tool:pytest] addopts = --strict-markers --durations=10 From 1897beab0b37a236382961d3c7f77695a8c53887 Mon Sep 17 00:00:00 2001 From: Emir Karamehmetoglu Date: Tue, 17 May 2022 00:01:58 +0200 Subject: [PATCH 5/6] update tests workflow to ignore G004 correctly --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4276869..0861e0a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,7 +46,7 @@ jobs: run: | # stop the build if there are Python syntax errors or undefined names # For some reason we have to specifically ignore G001 as well - flake8 --select=E9,F63,F7,F82 --ignore=G001 --ignore=G004 --show-source + flake8 --select=E9,F63,F7,F82 --show-source --ignore G001,G004 # exit-zero treats all errors as warnings. flake8 --exit-zero From 9efc961e1a0a2f495f395d5d712c3794f526dc8c Mon Sep 17 00:00:00 2001 From: Emir Karamehmetoglu Date: Tue, 17 May 2022 00:08:36 +0200 Subject: [PATCH 6/6] Remove pip caching on tests Creating undefined hard to track behavior. --- .github/workflows/tests.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0861e0a..b710c74 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,15 +27,6 @@ jobs: with: python-version: 3.9 - - name: Cache pip - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-py3.9-${{ hashFiles('requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-py3.9- - ${{ runner.os }}-pip- - - name: Install dependencies run: | python -m pip install --upgrade pip wheel @@ -46,9 +37,9 @@ jobs: run: | # stop the build if there are Python syntax errors or undefined names # For some reason we have to specifically ignore G001 as well - flake8 --select=E9,F63,F7,F82 --show-source --ignore G001,G004 + flake8 --select=E9,F63,F7,F82 --show-source --ignore=G001,G004 # exit-zero treats all errors as warnings. - flake8 --exit-zero + # flake8 --exit-zero # Run unit tests on Linux, OSX and Windows pytest: