From 4d217b8eb8667ee8249762d101366b7c09087f53 Mon Sep 17 00:00:00 2001 From: Emir Karamehmetoglu Date: Wed, 6 Jul 2022 15:40:12 +0200 Subject: [PATCH 1/8] add J filter to astronircam --- flows/instruments/instruments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flows/instruments/instruments.py b/flows/instruments/instruments.py index db28055..50b2e86 100644 --- a/flows/instruments/instruments.py +++ b/flows/instruments/instruments.py @@ -367,7 +367,7 @@ def get_obstime(self): def get_photfilter(self): hdr = self.image.header - photfilter = {'H_Open': 'H', 'K_Open': 'K', }.get(hdr['FILTER'], hdr['FILTER']) + photfilter = {'H_Open': 'H', 'K_Open': 'K', 'J_Open': 'J'}.get(hdr['FILTER'], hdr['FILTER']) return photfilter From 7dce8739067a5d375cd6b21a72632554ceeb3568 Mon Sep 17 00:00:00 2001 From: Emir Karamehmetoglu Date: Wed, 6 Jul 2022 15:41:28 +0200 Subject: [PATCH 2/8] remove unnecessarily strict site checking in Schmidt instrument. --- flows/instruments/instruments.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/flows/instruments/instruments.py b/flows/instruments/instruments.py index 50b2e86..18ac962 100644 --- a/flows/instruments/instruments.py +++ b/flows/instruments/instruments.py @@ -488,12 +488,9 @@ def get_photfilter(self): class Schmidt(Instrument): siteid = 26 peakmax = 56_000 - telescope = '67/91 Schmidt Telescope' # Fits Header name of TELESCOP - instrument = 'Moravian G4-16000LC' # Fits Header name of Instrument (can be partial) - origin = '' # Fits Header value of ORIGIN (if relevant) - unique_headers = { - 'SITELAT': 45.8494444 - } # Unique key value pairs from header for identifying instrument. + telescope = '67/91 Schmidt Telescope' + instrument = 'Moravian G4-16000LC' + origin = '' def get_obstime(self): obstime = Time(self.image.header['DATE-OBS'], format='isot', scale='utc', From 457c075bfc33f410157343e3ae2bdd4ce98933b6 Mon Sep 17 00:00:00 2001 From: Emir Karamehmetoglu Date: Wed, 6 Jul 2022 15:53:58 +0200 Subject: [PATCH 3/8] update docstring for instruments.py with instructions --- flows/instruments/instruments.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/flows/instruments/instruments.py b/flows/instruments/instruments.py index 18ac962..85128cd 100644 --- a/flows/instruments/instruments.py +++ b/flows/instruments/instruments.py @@ -1,5 +1,32 @@ """ -Load image code. +Instrument classes that inherit from the base class. +Modify to add your own instrument. + +Identifying relevant image properties: + +`site` = required, add manually. +`peakmax` = optional, add manually or provide header. +The rest of `exptime`, `obstime`, `photfilter` use the +following (overrideable) base functions. Override +if the default one from the baseclass does not fit +your instrument. See: + ``` + self.image.peakmax = self.peakmax + self.image.site = self.get_site() + self.image.exptime = self.get_exptime() + self.image.obstime = self.get_obstime() + self.image.photfilter = self.get_photfilter() + ``` + +Identifying the instrument for an image: + +Each instrument can define (one or many) of `origin`, +`telescope`, `instrument` fields correspinding to the +standard fits headers to help uniquely identify itself. +More advanced logic is possible using `unique_headers` +field as a dict of key,value pairs in the header. Ex: +unique_headers = {'PRODCATG': 'SCIENCE.MEFIMAGE'}. +These are all optional, defaults are set in baseclass. """ # Standard lib from __future__ import annotations From 6e7c9b7b38d3766f93986a998317ace3207b5c77 Mon Sep 17 00:00:00 2001 From: Emir Karamehmetoglu Date: Wed, 6 Jul 2022 16:10:14 +0200 Subject: [PATCH 4/8] fix for Hawk-I images without multiextenstion fitsheaders. --- flows/instruments/instruments.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/flows/instruments/instruments.py b/flows/instruments/instruments.py index 85128cd..cb02a88 100644 --- a/flows/instruments/instruments.py +++ b/flows/instruments/instruments.py @@ -12,7 +12,7 @@ your instrument. See: ``` self.image.peakmax = self.peakmax - self.image.site = self.get_site() + self.image.site = self.get_site() self.image.exptime = self.get_exptime() self.image.obstime = self.get_obstime() self.image.photfilter = self.get_photfilter() @@ -99,7 +99,7 @@ class HAWKI(Instrument): telescope = 'ESO-VLT-U4' # Fits Header name of TELESCOP instrument = 'HAWKI' # Fits Header name of Instrument (can be partial) origin = 'ESO-PARANAL' # Fits Header value of ORIGIN (if relevant) - unique_headers = {'PRODCATG': 'SCIENCE.MEFIMAGE'} + #unique_headers = {'PRODCATG': 'SCIENCE.MEFIMAGE'} def __init__(self, image: FlowsImage = None): super().__init__(image) @@ -128,6 +128,10 @@ def get_ext(hdul: fits.HDUList, target_coords: coords.SkyCoord = None, if target_coord is None: raise ValueError("TARGET_COORD is needed for HAWKI images to find the correct extension") + # Incase this is not a multi-extension imageL + if len(hdul) == 1: + return 0 + # For HAWKI multi-extension images we search the extensions for which one contains # the target, Create Image from that extension. target_radec = [[target_coord.icrs.ra.deg, target_coord.icrs.dec.deg]] From f101531e01bce27fcf44b6796eb7553477cc475a Mon Sep 17 00:00:00 2001 From: Emir Karamehmetoglu Date: Wed, 6 Jul 2022 16:18:17 +0200 Subject: [PATCH 5/8] fix for manually given target coords that are given as floats make sure to verify target coords in load_image --- flows/instruments/__init__.py | 2 +- flows/load_image.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/flows/instruments/__init__.py b/flows/instruments/__init__.py index 0ff55da..dd2eb5b 100644 --- a/flows/instruments/__init__.py +++ b/flows/instruments/__init__.py @@ -1 +1 @@ -from .instruments import INSTRUMENTS, Instrument +from .instruments import INSTRUMENTS, Instrument, verify_coordinates diff --git a/flows/load_image.py b/flows/load_image.py index aff04f9..5151a1a 100644 --- a/flows/load_image.py +++ b/flows/load_image.py @@ -8,7 +8,7 @@ from astropy.io import fits from astropy.time import Time -from .instruments import INSTRUMENTS +from .instruments import INSTRUMENTS, verify_coordinates from .image import FlowsImage from .utilities import create_logger logger = create_logger() @@ -39,6 +39,7 @@ def load_image(filename: str, target_coord: Union[coords.SkyCoord, Tuple[float, for inst_name, inst_cls in INSTRUMENTS: if inst_cls.identifier(telescope, origin, instrument, hdr): logger.info(f"Image is using instrument {inst_name}") + target_coord = verify_coordinates(target_coord) ext = inst_cls.get_ext(hdul, target_coord) mask = inst_cls.get_mask(hdul) # Default = None is to only mask all non-finite values, override here is additive. From 11fb1716ee6431bccfafde3f20c86c3345ead689 Mon Sep 17 00:00:00 2001 From: Emir Karamehmetoglu Date: Fri, 8 Jul 2022 19:03:05 -0700 Subject: [PATCH 6/8] make rescale uncertainty optional --- flows/photometry.py | 13 ++++++++----- run_photometry.py | 14 +++++++++----- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/flows/photometry.py b/flows/photometry.py index eef8367..642228c 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -415,7 +415,7 @@ def create_from_fid(cls, fid: int, directories: Optional[DirectoryProtocol] = No def do_phot(fileid: int, cm_timeout: Optional[float] = None, make_plots: bool = True, directories: Optional[DirectoryProtocol] = None, datafile: Optional[Dict[str, Any]] = None, - rescale_dynamic: bool = True) -> ResultsTable: + rescale: bool = True, rescale_dynamic: bool = True) -> ResultsTable: # Set up photometry runner pm = PhotometryManager.create_from_fid(fileid, directories=directories, datafile=datafile, create_directories=True) @@ -430,8 +430,10 @@ def do_phot(fileid: int, cm_timeout: Optional[float] = None, make_plots: bool = # Do photometry apphot_tbl = pm.apphot() - psfphot_tbl = ResultsTable.verify_uncertainty_column(pm.psfphot()) # Verify uncertainty exists after PSF phot. - psfphot_tbl = pm.rescale_uncertainty(psfphot_tbl, dynamic=rescale_dynamic) # Rescale uncertainties + # Verify uncertainty exists after PSF phot: + psfphot_tbl = ResultsTable.verify_uncertainty_column(pm.psfphot()) + if rescale: # Rescale uncertainties + psfphot_tbl = pm.rescale_uncertainty(psfphot_tbl, dynamic=rescale_dynamic) # Build results table and calculate magnitudes pm.make_result_table(psfphot_tbl, apphot_tbl) @@ -445,10 +447,11 @@ def do_phot(fileid: int, cm_timeout: Optional[float] = None, make_plots: bool = def timed_photometry(fileid: int, cm_timeout: Optional[float] = None, make_plots: bool = True, directories: Optional[DirectoryProtocol] = None, save: bool = True, - datafile: Optional[Dict[str, Any]] = None, rescale_dynamic: bool = True) -> ResultsTable: + datafile: Optional[Dict[str, Any]] = None, rescale: bool = True, + rescale_dynamic: bool = True) -> ResultsTable: # TODO: Timer should be moved out of this function. tic = default_timer() - results_table = do_phot(fileid, cm_timeout, make_plots, directories, datafile, rescale_dynamic) + results_table = do_phot(fileid, cm_timeout, make_plots, directories, datafile, rescale, rescale_dynamic) # Save the results table: if save: diff --git a/run_photometry.py b/run_photometry.py index 7e73256..469358e 100644 --- a/run_photometry.py +++ b/run_photometry.py @@ -11,8 +11,8 @@ from flows.utilities import create_logger, parse_log_level, create_warning_logger, remove_file_handlers -def process_fileid(fid, autoupload=False, cm_timeout=None, no_plots=False, - rescale_dynamic=True) -> result_model.ResultsTable: +def process_fileid(fid, autoupload: bool = False, cm_timeout=None, no_plots: bool = False, + rescale: bool = True, rescale_dynamic: bool = True) -> result_model.ResultsTable: # Create the output directory if it doesn't exist: datafile = api.get_datafile(fid) directories = fileio.Directories.from_fid(fid, datafile=datafile) @@ -25,7 +25,8 @@ def process_fileid(fid, autoupload=False, cm_timeout=None, no_plots=False, api.set_photometry_status(fid, 'running') table = photometry(fileid=fid, cm_timeout=cm_timeout, make_plots=not no_plots, - directories=directories, datafile=datafile, rescale_dynamic=rescale_dynamic) + directories=directories, datafile=datafile, rescale=rescale, + rescale_dynamic=rescale_dynamic) except (SystemExit, KeyboardInterrupt): logger.error("Aborted by user or system.") @@ -79,6 +80,8 @@ def main(): group.add_argument('--fixposdiff', action='store_true', help="Fix SN position during PSF photometry of difference image. " "Useful when difference image is noisy.") + group.add_argument('--rescale-off', action='store_false', + help='Turn off uncertainty rescaling.') group.add_argument('--wcstimeout', type=int, default=None, help="Timeout in Seconds for WCS.") args = parser.parse_args() @@ -118,8 +121,9 @@ def main(): fileids = list(set(fileids)) # Create function wrapper: - process_fileid_wrapper = functools.partial(process_fileid, autoupload=args.autoupload, cm_timeout=args.wcstimeout, - no_plots=args.noplots, rescale_dynamic=not args.rescale_static) + process_fileid_wrapper = functools.partial(process_fileid, autoupload=args.autoupload, + cm_timeout=args.wcstimeout, no_plots=args.noplots, + rescale=args.rescale_off, rescale_dynamic=not args.rescale_static) if threads > 1: # process in parallel: From 266beca09f31c1c74c05455cb9139ce9ae69dd77 Mon Sep 17 00:00:00 2001 From: Emir Karamehmetoglu Date: Sat, 9 Jul 2022 18:10:32 -0700 Subject: [PATCH 7/8] add ensure_greater to rescale uncertainty --- flows/photometry.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/flows/photometry.py b/flows/photometry.py index 642228c..98330af 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -316,10 +316,11 @@ def diff_psf_phot(self) -> Table: init_table=self.init_guesses_diff.init_guess_diff) return psf_tbl - def rescale_uncertainty(self, psfphot_tbl: Table, dynamic: bool = True, static_fwhm: float = 2.5): + def rescale_uncertainty(self, psfphot_tbl: Table, dynamic: bool = True, + static_fwhm: float = 2.5, epsilon_mag: float = 0.004, + ensure_greater: bool = True): """ - Rescale the uncertainty of the PSF photometry to match the uncertainty of the - photometry. + Rescale the uncertainty of the PSF photometry using a variable fitsize. Parameters ---------- @@ -329,6 +330,9 @@ def rescale_uncertainty(self, psfphot_tbl: Table, dynamic: bool = True, static_f Dynamically decide FWHM multiple for rescaling. static_fwhm : float FWHM multiple to use incase dynamic fails or don't want to use it. Default 2.5 determined empirically. + epsilon_mag : float + Small magnitude change within which new and old uncertainties are considered the same. + Should be smaller than ~1/2 the expected uncertainty. """ # Rescale psf errors from fit iteratively fit_shapes = self.photometry.get_fit_shapes(self.fwhm, self.psf_builder.star_size) @@ -361,6 +365,12 @@ def rescale_uncertainty(self, psfphot_tbl: Table, dynamic: bool = True, static_f logger.info(f"Recalculating all reference uncertainties using new fitsize:" f" {fit_shape} pixels, ({fit_shape/self.fwhm if dynamic else static_fwhm :.2} * FWHM).") psfphot_tbl_rescaled = self.psfphot(fit_shape) + if psfphot_tbl['flux_unc'][0] > psfphot_tbl_rescaled['flux_unc'][0] + epsilon_mag and ensure_greater: + logger.info("Recalculated uncertainties were smaller than original and ``ensure_greater`` was True:" + "Not using rescaled uncertainties for the SN.") + psfphot_tbl['flux_unc'][1:] = psfphot_tbl_rescaled['flux_unc'][1:] + return psfphot_tbl + psfphot_tbl['flux_unc'] = psfphot_tbl_rescaled['flux_unc'] return psfphot_tbl From d4c1b796bb40cd266c45181258017cde09d5f1b3 Mon Sep 17 00:00:00 2001 From: Emir Karamehmetoglu Date: Fri, 15 Jul 2022 12:56:48 +0200 Subject: [PATCH 8/8] bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d21f8ae..8eb90bd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -master-v0.10.7 \ No newline at end of file +master-v0.10.8 \ No newline at end of file