Fix CI/CD,Tests ADD MyPy, Black, Flake8, Coverage.

.github/workflows/python-package.yml

@@ -10,13 +10,70 @@ on:
     branches: [ main ]
 
 jobs:
+  linting:
+    name: Lint with Flake8, Black
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        persist-credentials: true
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 black
+    - name: Lint with Flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-ignore=F821 --exclude .git,.idea,.mypy_cache,Notebooks/
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=120 --statistics --extend-ignore=F821 --exclude .git,.idea,.mypy_cache,Notebooks/
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Enforce Black codestyle.
+      uses: psf/black@stable
+      with:
+        options: "--check --verbose"
+        src: "./flows_get_brightest"
+        version: "22.10.0"
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  mypy:
+    name: mypy
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        persist-credentials: true
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install '.[test]'
+        python -m pip install mypy types-requests
+    - name: Run MyPy
+      id: runmypy
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        mypy flows_get_brightest/ typings/tendrils/ --config-file=pyproject.toml --check-untyped-defs
+
   build:
-    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
         python-version: ["3.10"]
+    runs-on: ${{ matrix.os }}
+    name: Pytest on ${{ matrix.os }}, python 3.10
 
     steps:
     - uses: actions/checkout@v3
@@ -27,21 +84,20 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-        pip install '.[test]'
-    - name: Lint with flake8
-      continue-on-error: true
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-ignore=F821 --exclude .git,.idea,.mypy_cache,Notebooks/
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=20 --max-line-length=300 --statistics --extend-ignore=F821 --exclude .git,.idea,.mypy_cache,Notebooks/
-    - name: update tendrils
-      run: |
-
+        python -m pip install '.[test]'
     - name: Test with pytest
       env:
         FLOWS_API_TOKEN: ${{ secrets.FLOWS_API_TOKEN }}
       run: |
-        pytest -v
+        pytest -v --cov-report "xml:coverage.xml" --cov
+    - name: Pytest coverage report
+      uses: MishaKav/pytest-coverage-comment@main
+      if: startsWith(${{matrix.os}},'ubuntu') 
+      with:
+        pytest-xml-coverage-path: ./coverage.xml
+        token: ${{ secrets.GITHUB_TOKEN }}
+        title: Coverage Report
+        badge-title: Tests Coverage
+        hide-badge: false
+        hide-report: false
+

cutouts.py

@@ -1,4 +1,5 @@
 import astropy.io.fits as fits
+
 # noinspection PyProtectedMember
 from astropy.io.fits.hdu.image import _ImageBaseHDU
 from astropy.nddata import Cutout2D
@@ -11,9 +12,9 @@
 import numpy as np
 from abc import ABC, abstractmethod
 
-"""Expects fits compatible images saved with a single file extension (no .fits.gz), 
+"""Expects fits compatible images saved with a single file extension (no .fits.gz),
 Makes a cutout based on values of y and x and recomputes wcs,
-Saves the new fits file in output dir, 
+Saves the new fits file in output dir,
 preserves all extensions and applies the same cut to all image extensions
 optionally plots the primary image in cut and uncut form.
 
@@ -22,7 +23,6 @@
 
 
 class AbstractImage(ABC):
-
     @abstractmethod
     def __init__(self, image: _ImageBaseHDU):
         self.image = image
@@ -41,9 +41,7 @@ def make_cutout(self, y: int, x: int):
 
 # noinspection PyAttributeOutsideInit
 class Image(AbstractImage):
-
-    def __init__(self, image: _ImageBaseHDU,
-                 primary: bool = False):
+    def __init__(self, image: _ImageBaseHDU, primary: bool = False):
         self.image = image
         self.data = image.data
         self.header = image.header
@@ -63,12 +61,7 @@ def __init__(self, image: _ImageBaseHDU,
     #    else: raise(TypeError)
 
     def cutout(self, y, x):
-        self.cut = Cutout2D(
-            self.data,
-            position=self.wcs.wcs.crpix,
-            wcs=self.wcs,
-            size=(y, x)
-        )
+        self.cut = Cutout2D(self.data, position=self.wcs.wcs.crpix, wcs=self.wcs, size=(y, x))
 
     # noinspection PyCallingNonCallable
     def make_fits_from_cutout(self):
@@ -82,7 +75,7 @@ def fill_cut_image_header(self):
             try:
                 self.cut_image.header[key] = self.header[key]
             except ValueError:
-                print(f'Bad fits key: {key}')
+                print(f"Bad fits key: {key}")
 
     def make_cutout(self, y: int, x: int):
         """convenience function for making a cutout image and forward filling header"""
@@ -93,13 +86,9 @@ def make_cutout(self, y: int, x: int):
 
 
 class Plotter:
-
-    def __init__(self,
-                 figsize: tuple = (8, 8),
-                 zs: BaseInterval = ZScaleInterval(1000),
-                 subplots: int = 1,
-                 dpi: int = 125
-                 ):
+    def __init__(
+        self, figsize: tuple = (8, 8), zs: BaseInterval = ZScaleInterval(1000), subplots: int = 1, dpi: int = 125
+    ):
         self.figsize = figsize
         self.interval_finder = zs
         self.fig = plt.figure(figsize=self.figsize, dpi=dpi)
@@ -109,8 +98,8 @@ def __init__(self,
         self.ax = None
 
     # noinspection SpellCheckingInspection
-    def plot_img(self, image: Image, wcs: WCS = 'None'):
-        if wcs == 'None':
+    def plot_img(self, image: Image, wcs: WCS = "None"):
+        if wcs == "None":
             wcs = image.wcs
         self.ax = self.fig.add_subplot(1, self.subplots, self.active_axis, projection=wcs)
         vmin, vmax = self.interval_finder.get_limits(image.data)
@@ -124,18 +113,15 @@ def parse():
     """Parse input
     :rtype: object
     """
-    parser = argparse.ArgumentParser(prog='cut_image',
-                                     description='Cut image and all image extensions')
+    parser = argparse.ArgumentParser(prog="cut_image", description="Cut image and all image extensions")
 
     parser.add_argument("file", help="Specify the original file here")
 
-    parser.add_argument("--output_dir", help="Give output directory for saving", default='./')
-    parser.add_argument("-x", help='output suffix', type=int, default=4096)
-    parser.add_argument("-y", help='output suffix', type=int, default=4030)
-    parser.add_argument("-o", "--overwrite", help='overwrite output if exists',
-                        action='store_true', default=False)
-    parser.add_argument("-p", "--plot", help='overwrite output if exists',
-                        action='store_true', default=False)
+    parser.add_argument("--output_dir", help="Give output directory for saving", default="./")
+    parser.add_argument("-x", help="output suffix", type=int, default=4096)
+    parser.add_argument("-y", help="output suffix", type=int, default=4030)
+    parser.add_argument("-o", "--overwrite", help="overwrite output if exists", action="store_true", default=False)
+    parser.add_argument("-p", "--plot", help="overwrite output if exists", action="store_true", default=False)
 
     return parser.parse_args()
 
@@ -144,8 +130,10 @@ def find_primary(hdul):
     # Find primary HDU and use its wcs
     primary_index = np.argwhere([isinstance(i, fits.PrimaryHDU) for i in hdul]).flatten()
     if primary_index.size <= 0 or primary_index.size > 1:
-        raise ValueError('WCS to use could not be set as there were less than or greater than one Primary HDUs \
-                         This should never be the case, please check your fits image!')
+        raise ValueError(
+            "WCS to use could not be set as there were less than or greater than one Primary HDUs                      "
+            "    This should never be the case, please check your fits image!"
+        )
     return primary_index[0]
 
 
@@ -163,7 +151,7 @@ def make_img_cutout(i, wcs, y, x) -> AbstractImage:
 def main():
     args = parse()
     curr = Path(args.file)
-    save_path = Path.joinpath(Path(args.output_dir), curr.stem + '_cutout' + curr.suffix).resolve()
+    save_path = Path.joinpath(Path(args.output_dir), curr.stem + "_cutout" + curr.suffix).resolve()
 
     hdul = fits.open(curr)
     new_hdul = fits.HDUList()
@@ -184,16 +172,15 @@ def main():
     if args.plot:
         plotter = Plotter(subplots=2, dpi=200)
         plotter.plot_img(Image(new_hdul[0], primary=True))
-        plotter.ax.set_title('cutout')
+        plotter.ax.set_title("cutout")
         plotter.plot_img(Image(hdul[0], primary=True))
-        plotter.ax.set_title('uncut')
+        plotter.ax.set_title("uncut")
         plt.show(block=True)
 
     # Save
-    new_hdul.writeto(str(save_path), output_verify='exception',
-                     overwrite=args.overwrite, checksum=False)
-    print(f'saved to: {save_path}')
+    new_hdul.writeto(str(save_path), output_verify="exception", overwrite=args.overwrite, checksum=False)
+    print(f"saved to: {save_path}")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

fetch_eso_data.py

@@ -1,83 +1,80 @@
-from tendrils import api, utils
-import pandas as pd
-import multiprocessing as mp
+import asyncio
 from typing import Collection, Optional
 import numpy as np
-from astropy.table import Table
-import asyncio
+import pandas as pd
 from requests.exceptions import HTTPError
+from tendrils import api
 
 eso_sites = [2, 12, 15]
-eso_site_names = {2:"VLT-HAWKI", 12: "NTT-SOFI", 15: "NTT-EFOSC"}
-usecols = ['fileid_img','time', 'mag', 'site', 'photfilter']
-tgtcols = ['target_name','redshift','ra','decl']
+eso_site_names = {2: "VLT-HAWKI", 12: "NTT-SOFI", 15: "NTT-EFOSC"}
+usecols = ["fileid_img", "time", "mag", "site", "photfilter"]
+tgtcols = ["target_name", "redshift", "ra", "decl"]
+
 
 def get_targets(sub_targets: Optional[list[int]] = None) -> pd.DataFrame:
     targets = api.get_targets()
     targets = pd.DataFrame(targets)
-    targets = targets[targets['target_status']=='target']
+    targets = targets[targets["target_status"] == "target"]
     if sub_targets is not None:
-        targets = targets[targets['targetid'].isin(sub_targets)]
-    return targets.set_index('targetid')
+        targets = targets[targets["targetid"].isin(sub_targets)]
+    return targets.set_index("targetid")
+
 
 def get_target_data(tgtid: int, targets: pd.DataFrame) -> pd.Series:
     return targets.loc[(tgtid, tgtcols)]
 
+
 def get_sites(fids: Collection[int]) -> pd.DataFrame:
-    keys = ['obstime', 'site', 'photfilter']
-    r = {fid:{k:dat[k] for k in keys}
-         for fid in fids
-         if (dat:=api.get_datafile(fid))['site'] in eso_sites}
+    keys = ["obstime", "site", "photfilter"]
+    r = {fid: {k: dat[k] for k in keys} for fid in fids if (dat := api.get_datafile(fid))["site"] in eso_sites}
     df = pd.DataFrame(r).T
-    df.index.name = 'fileid_img'
-    df.rename({'obstime':'time'}, axis=1, inplace=True)
-    df['mag'] = np.nan
+    df.index.name = "fileid_img"
+    df.rename({"obstime": "time"}, axis=1, inplace=True)
+    df["mag"] = np.nan
     return df
 
-async def get_target_lc(tgtid: int, targets: pd.DataFrame) -> pd.DataFrame:
 
+async def get_target_lc(tgtid: int, targets: pd.DataFrame) -> pd.DataFrame:
     lc = api.get_lightcurve(tgtid)
-    lc['time'] = lc['time'].mjd  # type: ignore
+    lc["time"] = lc["time"].mjd  # type: ignore
     lc = lc.to_pandas()
-    lc['mag'] = lc['mag_sub'].fillna(lc['mag_raw'])
+    lc["mag"] = lc["mag_sub"].fillna(lc["mag_raw"])
     lc = lc[usecols]
-    return lc.set_index('fileid_img')
-
+    return lc.set_index("fileid_img")
+
+
 async def get_missing_eso_lcs(lc: pd.DataFrame, tgtid: int, targets: pd.DataFrame) -> pd.DataFrame:
     # add data with failed/missing photometry
-    dfs = api.get_datafiles(tgtid, filt='all')
+    dfs = api.get_datafiles(tgtid, filt="all")
     missing_fids = set(dfs) - set(lc.index.values)
     lc = pd.concat((lc, get_sites(missing_fids)))
     lc.sort_index(inplace=True)
-    return lc[lc['site'].isin(eso_sites)]
+    return lc[lc["site"].isin(eso_sites)]
+
 
 async def get_eso_lc(tgtid: int, targets: pd.DataFrame) -> pd.DataFrame:
     try:
         lc = await get_target_lc(tgtid, targets)
     except HTTPError:
         lc = pd.DataFrame(index=[], columns=usecols)
-        lc.set_index('fileid_img', inplace=True)
+        lc.set_index("fileid_img", inplace=True)
     lc = await get_missing_eso_lcs(lc, tgtid, targets)
     if len(lc) == 0:
         return pd.DataFrame()
     lc[tgtcols] = get_target_data(tgtid, targets)
     return lc
 
 
-
 async def main():
     targets = get_targets()
     coroutines = [get_eso_lc(tgtid, targets) for tgtid in targets.index]
     eso_lcs_with_excp = await asyncio.gather(*coroutines, return_exceptions=True)
     eso_lcs = pd.concat([lc for lc in eso_lcs_with_excp if isinstance(lc, pd.DataFrame)])
-    eso_lcs['site'] = eso_lcs.site.apply(lambda x: eso_site_names[x])
-    eso_lcs.to_json('eso_lcs.json')
-    eso_lcs.to_csv('eso_lcs.csv')
+    eso_lcs["site"] = eso_lcs.site.apply(lambda x: eso_site_names[x])
+    eso_lcs.to_json("eso_lcs.json")
+    eso_lcs.to_csv("eso_lcs.csv")
     print([lc for lc in eso_lcs_with_excp if not isinstance(lc, pd.DataFrame)])
-
-
-
-if __name__ == '__main__':
-    asyncio.run(main())
 
 
+if __name__ == "__main__":
+    asyncio.run(main())