From c1e5e97fbbb3d0083215920474fdee9e7056b3d2 Mon Sep 17 00:00:00 2001 From: Emir Date: Fri, 8 Jan 2021 17:07:10 +0100 Subject: [PATCH 01/43] implement wcs checking with sep and astropy --- flows/photometry.py | 99 ++++++++++++++++++++++++++++++++++++++------- run_photometry.py | 38 ++++++++++++++++- 2 files changed, 121 insertions(+), 16 deletions(-) diff --git a/flows/photometry.py b/flows/photometry.py index 537b722..0f37ebe 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -8,11 +8,13 @@ import os import numpy as np +import sep from bottleneck import nansum, nanmedian, allnan from timeit import default_timer import logging import warnings from copy import deepcopy +import pandas as pd from astropy.utils.exceptions import AstropyDeprecationWarning import astropy.units as u @@ -22,6 +24,7 @@ from astropy.nddata import NDData from astropy.modeling import models, fitting from astropy.wcs.utils import proj_plane_pixel_area +from astropy.time import Time warnings.simplefilter('ignore', category=AstropyDeprecationWarning) from photutils import DAOStarFinder, CircularAperture, CircularAnnulus, aperture_photometry @@ -123,6 +126,13 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): references = catalog['references'] references.sort(ref_filter) + + + #time_delta = image.obstime - Time(2015.5, format='jyear') + #references['decl_obs'] = references['decl'] * u.deg + references['pm_dec'] * u.marcsec / u.yr * time_delta + #cosdecavg = np.cos((references['decl_obs']-references['decl'])/2)*u.deg + #references['ra_obs'] = references['ra']*u.deg + references['pm_ra']*u.marcsec/u.yr*time_delta * cosdecavg + # Check that there actually are reference stars in that filter: if allnan(references[ref_filter]): raise ValueError("No reference stars found in current photfilter.") @@ -130,8 +140,16 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # Load the image from the FITS file: image = load_image(filepath) + # Account for proper motion + mycoords = coords.SkyCoord(references['ra'], references['decl'], obstime=Time(2015.5, format='decimalyear'), + pm_ra_cosdec=references['pm_ra'], pm_dec=references['pm_dec'], distance=1 * u.kpc, + radial_velocity=1000 * u.km / u.s) # Dummy velocity and distance needed for procession calc. + mycoords = mycoords.apply_space_motion(image.obstime) + references['ra_obs'] = mycoords.ra + references['decl_obs'] = mycoords.dec + # Calculate pixel-coordinates of references: - row_col_coords = image.wcs.all_world2pix(np.array([[ref['ra'], ref['decl']] for ref in references]), 0) + row_col_coords = image.wcs.all_world2pix(np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) references['pixel_column'] = row_col_coords[:,0] references['pixel_row'] = row_col_coords[:,1] @@ -142,12 +160,12 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): hsize = 10 x = references['pixel_column'] y = references['pixel_row'] - refs_coord = coords.SkyCoord(ra=references['ra'], dec=references['decl'], unit='deg', frame='icrs') + refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', frame='icrs') references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] # & (references[ref_filter] < ref_mag_limit) - + debug_references = references.copy() #============================================================================================== # BARYCENTRIC CORRECTION OF TIME #============================================================================================== @@ -156,7 +174,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): image.obstime = image.obstime.tdb + ltt_bary #============================================================================================== - # BACKGROUND ESITMATION + # BACKGROUND ESTIMATION #============================================================================================== fig, ax = plt.subplots(1, 2, figsize=(20, 18)) @@ -188,6 +206,16 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # TODO: Is this correct?! image.error = calc_total_error(image.clean, bkg.background_rms, 1.0) + # Use sep to for soure extraction + image.sepdata = image.image.byteswap().newbyteorder() + image.sepbkg = sep.Background(image.sepdata,mask=image.mask) + image.sepsub = image.sepdata - image.sepbkg + logger.debug('sub: {} bkg_rms: {} mask: {}'.format(np.shape(image.sepsub),np.shape(image.sepbkg.globalrms), + np.shape(image.mask))) + objects = sep.extract(image.sepsub, thresh=5., err=image.sepbkg.globalrms, mask=image.mask, + deblend_cont=0.1, minarea=9, clean_param=2.0) + + #============================================================================================== # DETECTION OF STARS AND MATCHING WITH CATALOG #============================================================================================== @@ -199,13 +227,6 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): fwhm_min = 3.5 fwhm_max = 18.0 - # Extract stars sub-images: - #stars = extract_stars( - # NDData(data=image.subclean, mask=image.mask), - # stars_for_epsf, - # size=size - #) - # Set up 2D Gaussian model for fitting to reference stars: g2d = models.Gaussian2D(amplitude=1.0, x_mean=radius, y_mean=radius, x_stddev=fwhm_guess*gaussian_fwhm_to_sigma) g2d.amplitude.bounds = (0.1, 2.0) @@ -217,6 +238,39 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): gfitter = fitting.LevMarLSQFitter() + # SEP references reject + sep_xy = np.full((len(objects),2), np.NaN) + sep_rsqs = np.full(len(objects), np.NaN) + for i, (x, y) in enumerate(zip(objects['x'], objects['y'])): + x = int(np.round(x)) + y = int(np.round(y)) + xmin = max(x - radius, 0) + xmax = min(x + radius + 1, image.shape[1]) + ymin = max(y - radius, 0) + ymax = min(y + radius + 1, image.shape[0]) + + curr_star = deepcopy(image.subclean[ymin:ymax, xmin:xmax]) + + edge = np.zeros_like(curr_star, dtype='bool') + edge[(0,-1),:] = True + edge[:,(0,-1)] = True + curr_star -= nanmedian(curr_star[edge]) + curr_star /= np.max(curr_star) + + ypos, xpos = np.mgrid[:curr_star.shape[0], :curr_star.shape[1]] + gfit = gfitter(g2d, x=xpos, y=ypos, z=curr_star) + + sep_xy[i] = np.array([gfit.x_mean+x-radius,gfit.y_mean+y-radius],dtype=np.float64) + # Calculate rsq + sstot = ((curr_star - curr_star.mean()) ** 2).sum() + sserr = (gfitter.fit_info['fvec']**2).sum() + sep_rsqs[i]=1.-(sserr/sstot) + + masked_sep_xy = np.ma.masked_array(sep_xy, ~np.isfinite(sep_xy)) + masked_sep_rsqs = np.ma.masked_array(sep_rsqs, ~np.isfinite(sep_rsqs)) + sep_mask = (masked_sep_rsqs >= 0.5) & (masked_sep_rsqs < 1.0) # Reject Rsq<0.5 + masked_sep_xy = masked_sep_xy[sep_mask] # Clean extracted array. + fwhms = np.full(len(references), np.NaN) gfits = [] #gfit_err = [] @@ -261,15 +315,17 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # Create plot of target and reference star positions from 2D Gaussian fits. # Get data into pandas DF - import pandas as pd gfitsdf = pd.DataFrame(gfits) gfitsdf['pixel_column'] = gfitsdf.y_mean + references['pixel_column'] - 10. gfitsdf['pixel_row'] = gfitsdf.x_mean + references['pixel_row'] - 10. # Make the plot fig, ax = plt.subplots(1, 1, figsize=(20, 18)) plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) + #ax.scatter(debug_references['pixel_column'], debug_references['pixel_row'], c='orange', marker='o', alpha=0.6) ax.scatter(references['pixel_column'], references['pixel_row'], c='r', marker='o', alpha=0.6) - ax.scatter(gfitsdf['pixel_column'], gfitsdf['pixel_row'], c='g', marker='o', alpha=0.6) + #ax.scatter(gfitsdf['pixel_column'], gfitsdf['pixel_row'], c='g', marker='o', alpha=0.6) + ax.scatter(masked_sep_xy[:,0],masked_sep_xy[:,1],marker='s',alpha=0.6, edgecolors='cyan' ,facecolors='none') + #ax.scatter(objects['x'],objects['y'],marker='d',alpha=0.6, edgecolors='cyan' ,facecolors='none') ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') fig.savefig(os.path.join(output_folder, 'positions_g2d.png'), bbox_inches='tight') plt.close(fig) @@ -323,6 +379,19 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # star was actually detected close to the references-star coordinate: cleanout_references = (len(references) > 6) logger.debug("Number of references before cleaning: %d", len(references)) + daofind_args = [{ + 'threshold': 7 * image.std, + 'fwhm': fwhm, + 'exclude_border': True, + 'sharphi': 0.8, + 'sigma_radius': 1.1, + 'peakmax': image.peakmax + }, { + 'threshold': 3 * image.std, + 'fwhm': fwhm, + 'roundlo': -0.5, + 'roundhi': 0.5 + }] if cleanout_references: # Arguments for DAOStarFind, starting with the strictest and ending with the # least strict settings to try: @@ -358,7 +427,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): if np.sum(indx_good) >= min_references: references = references[indx_good] break - + daofind_tbl = DAOStarFinder(**daofind_args[0]).find_stars(image.subclean, mask=image.mask) logger.debug("Number of references after cleaning: %d", len(references)) # Further clean references based on 2D gaussian fits @@ -737,4 +806,4 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): logger.info("Main target: %f +/- %f", tab[0]['mag'], tab[0]['mag_error']) logger.info("Photometry took: %f seconds", toc-tic) - return photometry_output + return photometry_output,objects,masked_sep_xy,debug_references,masked_sep_rsqs,sep_mask,image.wcs diff --git a/run_photometry.py b/run_photometry.py index d2daf93..bb1ec6c 100644 --- a/run_photometry.py +++ b/run_photometry.py @@ -146,4 +146,40 @@ def process_fileid(fid, output_folder_root=None, attempt_imagematch=True, autoup print("="*72) print(fid) print("="*72) - process_fileid_wrapper(fid) + _,objects,masked_sep_xy,debug_references,masked_sep_rsqs,sep_mask,wcs = process_fileid_wrapper(fid) + + #from astropy.table import Table + import numpy as np + import astroalign as aa + import matplotlib + matplotlib.use('qt5agg') + import matplotlib.pyplot as plt + + def mkposxy(posx, posy): + img_posxy = np.array([[x, y] for x, y in zip(posx, posy)], dtype="float64") + return img_posxy + + + aa.NUM_NEAREST_NEIGHBORS = 5 + aa.MIN_MATCHES_FRACTION = 0.8 + aa.PIXEL_TOL = 5 + + #import pandas as pd + from astropy.table import Table + objects=Table(objects) + #objects.sort('flux',reverse=True) + #daofind_tbl.sort('flux', reverse=True) + debug_references.sort('g_mag') + at = Table({'xy': masked_sep_xy, 'flux': objects['flux'][sep_mask]}) + at.sort('flux',reverse=True) + masked_sep_xy = at['xy'].data.data + + source = mkposxy(debug_references['pixel_column'].data,debug_references['pixel_row'].data) + target = masked_sep_xy + + plt.scatter(source[:,0],source[:,1],label='references',alpha=0.5,marker='d') + plt.scatter(masked_sep_xy[:,0],masked_sep_xy[:,1],label='clean sep',alpha=0.5) + plt.legend() + plt.show(block=True) + + out = aa.find_transform(source,target,max_control_points=3) \ No newline at end of file From 22e10ae2dd627b12986d5fbf7829e583df3efd04 Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 13 Jan 2021 15:13:34 +0100 Subject: [PATCH 02/43] wcs_checking_fix --- flows/photometry.py | 380 +++++++++++++++++++++----------------------- flows/wcs.py | 254 +++++++++++++++++++++++++++++ requirements.txt | 7 +- run_photometry.py | 38 +---- 4 files changed, 437 insertions(+), 242 deletions(-) create mode 100644 flows/wcs.py diff --git a/flows/photometry.py b/flows/photometry.py index 0f37ebe..8968337 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -14,7 +14,7 @@ import logging import warnings from copy import deepcopy -import pandas as pd +import astroalign as aa from astropy.utils.exceptions import AstropyDeprecationWarning import astropy.units as u @@ -23,7 +23,7 @@ from astropy.table import Table, vstack from astropy.nddata import NDData from astropy.modeling import models, fitting -from astropy.wcs.utils import proj_plane_pixel_area +from astropy.wcs.utils import proj_plane_pixel_area, fit_wcs_from_points from astropy.time import Time warnings.simplefilter('ignore', category=AstropyDeprecationWarning) @@ -41,6 +41,8 @@ from .load_image import load_image from .run_imagematch import run_imagematch from .zeropoint import bootstrap_outlier, sigma_from_Chauvenet +from .wcs import force_reject_g2d, mkposxy, clean_with_rsq_and_get_fwhm, \ + min_to_max_astroalign, kdtree, get_new_wcs, get_clean_references __version__ = get_version(pep440=False) @@ -126,13 +128,6 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): references = catalog['references'] references.sort(ref_filter) - - - #time_delta = image.obstime - Time(2015.5, format='jyear') - #references['decl_obs'] = references['decl'] * u.deg + references['pm_dec'] * u.marcsec / u.yr * time_delta - #cosdecavg = np.cos((references['decl_obs']-references['decl'])/2)*u.deg - #references['ra_obs'] = references['ra']*u.deg + references['pm_ra']*u.marcsec/u.yr*time_delta * cosdecavg - # Check that there actually are reference stars in that filter: if allnan(references[ref_filter]): raise ValueError("No reference stars found in current photfilter.") @@ -165,7 +160,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] # & (references[ref_filter] < ref_mag_limit) - debug_references = references.copy() + #============================================================================================== # BARYCENTRIC CORRECTION OF TIME #============================================================================================== @@ -227,216 +222,177 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): fwhm_min = 3.5 fwhm_max = 18.0 - # Set up 2D Gaussian model for fitting to reference stars: - g2d = models.Gaussian2D(amplitude=1.0, x_mean=radius, y_mean=radius, x_stddev=fwhm_guess*gaussian_fwhm_to_sigma) - g2d.amplitude.bounds = (0.1, 2.0) - g2d.x_mean.bounds = (0.5*radius, 1.5*radius) - g2d.y_mean.bounds = (0.5*radius, 1.5*radius) - g2d.x_stddev.bounds = (fwhm_min * gaussian_fwhm_to_sigma, fwhm_max * gaussian_fwhm_to_sigma) - g2d.y_stddev.tied = lambda model: model.x_stddev - g2d.theta.fixed = True - - gfitter = fitting.LevMarLSQFitter() - - # SEP references reject - sep_xy = np.full((len(objects),2), np.NaN) - sep_rsqs = np.full(len(objects), np.NaN) - for i, (x, y) in enumerate(zip(objects['x'], objects['y'])): - x = int(np.round(x)) - y = int(np.round(y)) - xmin = max(x - radius, 0) - xmax = min(x + radius + 1, image.shape[1]) - ymin = max(y - radius, 0) - ymax = min(y + radius + 1, image.shape[0]) - - curr_star = deepcopy(image.subclean[ymin:ymax, xmin:xmax]) - - edge = np.zeros_like(curr_star, dtype='bool') - edge[(0,-1),:] = True - edge[:,(0,-1)] = True - curr_star -= nanmedian(curr_star[edge]) - curr_star /= np.max(curr_star) - - ypos, xpos = np.mgrid[:curr_star.shape[0], :curr_star.shape[1]] - gfit = gfitter(g2d, x=xpos, y=ypos, z=curr_star) - - sep_xy[i] = np.array([gfit.x_mean+x-radius,gfit.y_mean+y-radius],dtype=np.float64) - # Calculate rsq - sstot = ((curr_star - curr_star.mean()) ** 2).sum() - sserr = (gfitter.fit_info['fvec']**2).sum() - sep_rsqs[i]=1.-(sserr/sstot) - - masked_sep_xy = np.ma.masked_array(sep_xy, ~np.isfinite(sep_xy)) - masked_sep_rsqs = np.ma.masked_array(sep_rsqs, ~np.isfinite(sep_rsqs)) - sep_mask = (masked_sep_rsqs >= 0.5) & (masked_sep_rsqs < 1.0) # Reject Rsq<0.5 - masked_sep_xy = masked_sep_xy[sep_mask] # Clean extracted array. - - fwhms = np.full(len(references), np.NaN) - gfits = [] - #gfit_err = [] - rsqs = np.full(len(references), np.NaN) - for i, (x, y) in enumerate(zip(references['pixel_column'], references['pixel_row'])): - x = int(np.round(x)) - y = int(np.round(y)) - xmin = max(x - radius, 0) - xmax = min(x + radius + 1, image.shape[1]) - ymin = max(y - radius, 0) - ymax = min(y + radius + 1, image.shape[0]) - - curr_star = deepcopy(image.subclean[ymin:ymax, xmin:xmax]) - - edge = np.zeros_like(curr_star, dtype='bool') - edge[(0,-1),:] = True - edge[:,(0,-1)] = True - curr_star -= nanmedian(curr_star[edge]) - curr_star /= np.max(curr_star) - - ypos, xpos = np.mgrid[:curr_star.shape[0], :curr_star.shape[1]] - gfit = gfitter(g2d, x=xpos, y=ypos, z=curr_star) - - fwhms[i] = gfit.x_fwhm - gfits.append(dict(zip(gfit.param_names,gfit.parameters))) - # Calculate rsq - sstot = ((curr_star - curr_star.mean()) ** 2).sum() - sserr = (gfitter.fit_info['fvec']**2).sum() - rsqs[i]=1.-(sserr/sstot) - - masked_fwhms = np.ma.masked_array(fwhms, ~np.isfinite(fwhms)) - masked_rsqs = np.ma.masked_array(rsqs, ~np.isfinite(rsqs)) + # Clean extracted stars + masked_sep_xy,sep_mask,masked_sep_rsqs = force_reject_g2d(objects['x'], objects['y'], image, get_fwhm=False, + radius=radius, fwhm_guess=fwhm_guess, rsq_min=0.3, fwhm_max=fwhm_max, fwhm_min=fwhm_min) + + # Clean reference star locations + masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(references['pixel_column'], + references['pixel_row'], + image, + get_fwhm=True, + radius=radius, + fwhm_guess=fwhm_guess, + fwhm_max=fwhm_max, + fwhm_min=fwhm_min, + rsq_min=0.15) + + # Use R^2 to more robustly determine initial FWHM guess. # This cleaning is good when we have FEW references. - min_fwhm_references = 2 - min_references = 6 - min_references_now = min_references - rsq_min = 0.15 - rsqvals = np.arange(rsq_min,0.95,0.15)[::-1] - fwhm_found = False - min_references_achieved = False + fwhm, clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, references, + min_fwhm_references=2, min_references=6, rsq_min=0.15) # Create plot of target and reference star positions from 2D Gaussian fits. - # Get data into pandas DF - gfitsdf = pd.DataFrame(gfits) - gfitsdf['pixel_column'] = gfitsdf.y_mean + references['pixel_column'] - 10. - gfitsdf['pixel_row'] = gfitsdf.x_mean + references['pixel_row'] - 10. - # Make the plot fig, ax = plt.subplots(1, 1, figsize=(20, 18)) plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) - #ax.scatter(debug_references['pixel_column'], debug_references['pixel_row'], c='orange', marker='o', alpha=0.6) - ax.scatter(references['pixel_column'], references['pixel_row'], c='r', marker='o', alpha=0.6) - #ax.scatter(gfitsdf['pixel_column'], gfitsdf['pixel_row'], c='g', marker='o', alpha=0.6) - ax.scatter(masked_sep_xy[:,0],masked_sep_xy[:,1],marker='s',alpha=0.6, edgecolors='cyan' ,facecolors='none') - #ax.scatter(objects['x'],objects['y'],marker='d',alpha=0.6, edgecolors='cyan' ,facecolors='none') + ax.scatter(references['pixel_column'], references['pixel_row'], c='r', marker='o', alpha=0.3) + ax.scatter(clean_references['pixel_column'], clean_references['pixel_row'], c='yellow', marker='o', alpha=0.3) + #ax.scatter(masked_ref_xys[:,0], masked_ref_xys[:,0], marker='o', alpha=0.6, edgecolors='green', facecolors='none') + ax.scatter(masked_sep_xy[:,0],masked_sep_xy[:,1],marker='s',alpha=1.0, edgecolors='green' ,facecolors='none') ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') fig.savefig(os.path.join(output_folder, 'positions_g2d.png'), bbox_inches='tight') plt.close(fig) - # Clean based on R^2 Value - while not min_references_achieved: - for rsqval in rsqvals: - mask = (masked_rsqs >= rsqval) & (masked_rsqs<1.0) - nreferences = len(np.isfinite(masked_fwhms[mask])) - if nreferences >= min_fwhm_references: - _fwhms_cut_ = np.mean(sigma_clip(masked_fwhms[mask], maxiters=100, sigma=2.0)) - logger.info('R^2 >= '+str(rsqval)+': '+str(len(np.isfinite(masked_fwhms[mask])))+' stars w/ mean FWHM = '+str(np.round(_fwhms_cut_,1))) - if not fwhm_found: - fwhm = _fwhms_cut_ - fwhm_found = True - if nreferences >= min_references_now: - references = references[mask] - min_references_achieved = True + # Sort by brightness + clean_references.sort('g_mag') # Sorted by g mag + _at = Table({'xy': masked_sep_xy, 'flux': objects['flux'][sep_mask]}) + _at.sort('flux', reverse=True) # Sorted by flux + masked_sep_xy = _at['xy'].data.data + + # Check WCS + wcs_rotation = 0 + wcs_rota_max = 3 + nreferences = len(clean_references['pixel_column']) + + while wcs_rotation < wcs_rota_max: + nreferences_old = nreferences + ref_xys = mkposxy(clean_references['pixel_column'], clean_references['pixel_row']) + + clean_coords = coords.SkyCoord(clean_references['ra'], clean_references['decl'],obstime=Time(2015.5, format='decimalyear'), + pm_ra_cosdec = clean_references['pm_ra'], pm_dec = clean_references['pm_dec'], + distance = 1 * u.kpc, radial_velocity = 1000 * u.km / u.s) + + # Find matches using astroalign + #ref_ind, sep_ind, success_aa = min_to_max_astroalign(ref_xys, masked_sep_xy, fwhm=fwhm, fwhm_min=2, + # fwhm_max=4, knn_min=5, knn_max=25, max_stars=100, + # min_matches=3) + # Basically if aa is failing us by not finding more than 3 matches, use KDtree for the last iteration. + try_kd = True + success_aa = False + + #if success_aa: + # astroalign_nmatches = len(ref_ind) + # if wcs_rotation > 1 and astroalign_nmatches <= 3: + # try_kd = True + + # Find matches using nearest neighbor + #if not success_aa or try_kd: + #fwhm_max = wcs_rotation + ref_ind_kd, sep_ind_kd, success_kd = kdtree(ref_xys, masked_sep_xy, fwhm, fwhm_max=4) + if success_kd: + kdtree_nmatches = len(ref_ind_kd) + if try_kd and kdtree_nmatches > 3: + ref_ind = ref_ind_kd + sep_ind = sep_ind_kd + else: + success_kd = False + + if success_aa or success_kd: + # Fit for new WCS + image.new_wcs = get_new_wcs(sep_ind, masked_sep_xy, clean_references, ref_ind, image.obstime) + wcs_rotation += 1 + + # Calculate pixel-coordinates of references: + row_col_coords = image.new_wcs.all_world2pix(np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]),0) + references['pixel_column'] = row_col_coords[:, 0] + references['pixel_row'] = row_col_coords[:, 1] + + masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(references['pixel_column'], + references['pixel_row'], + image, + get_fwhm=True, + radius=radius, + fwhm_guess=fwhm, + fwhm_max=fwhm_max, + fwhm_min=fwhm_min, + rsq_min=0.15) + # Clean with R^2 + fwhm, clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, references, + min_fwhm_references=2, min_references=6, rsq_min=0.15) + image.fwhm = fwhm + + nreferences_new = len(clean_references) + logging.info('{} References were found after new wcs compared to {} references before'.format(nreferences_old,nreferences_new)) + nreferences = nreferences_new + wcs_success = True + + # Break early if no improvement after 2nd pass. + if wcs_rotation > 1 and nreferences_new <= nreferences_old: break - if min_references_achieved: break - min_references_now = min_references_now - 2 - if (min_references_now < 2) and fwhm_found: break - elif not fwhm_found: raise Exception("Could not estimate FWHM") - logger.debug('{} {} {}'.format(min_references_now,min_fwhm_references,nreferences)) - - - logger.info("FWHM: %f", fwhm) - if np.isnan(fwhm): - raise Exception("Could not estimate FWHM") - - # if minimum references not found, then take what we can get with even a weaker cut. - # TODO: Is this right, or should we grab rsq_min (or even weaker?) - min_references_now = min_references - 2 - while not min_references_achieved: - mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) - nreferences = len(np.isfinite(masked_fwhms[mask])) - if nreferences >= min_references_now: - references = references[mask] - min_references_achieved = True - rsq_min = rsq_min - 0.07 - min_references_now = min_references_now - 1 - - # Check len of references as this is a destructive cleaning. - if len(references)==2: - logger.info('2 reference stars remaining, check WCS and image quality') - elif len(references)<2: - raise Exception(str(len(references))+"References remaining; could not clean.") - - # This cleaning is good when we have MANY references. - # Use DAOStarFinder to search the image for stars, and only use reference-stars where a - # star was actually detected close to the references-star coordinate: - cleanout_references = (len(references) > 6) + + else: + logging.info('New WCS could not be computed due to lack of matches.') + wcs_success = False + break + + if wcs_success: + image.wcs = image.new_wcs + + # Final cleanout of references logger.debug("Number of references before cleaning: %d", len(references)) - daofind_args = [{ - 'threshold': 7 * image.std, - 'fwhm': fwhm, - 'exclude_border': True, - 'sharphi': 0.8, - 'sigma_radius': 1.1, - 'peakmax': image.peakmax - }, { - 'threshold': 3 * image.std, - 'fwhm': fwhm, - 'roundlo': -0.5, - 'roundhi': 0.5 - }] - if cleanout_references: - # Arguments for DAOStarFind, starting with the strictest and ending with the - # least strict settings to try: - # We will stop with the first set that yield more than the minimum number - # of reference stars. - daofind_args = [{ - 'threshold': 7 * image.std, - 'fwhm': fwhm, - 'exclude_border': True, - 'sharphi': 0.8, - 'sigma_radius': 1.1, - 'peakmax': image.peakmax - }, { - 'threshold': 3 * image.std, - 'fwhm': fwhm, - 'roundlo': -0.5, - 'roundhi': 0.5 - }] - - # Loop through argument sets for DAOStarFind: - for kwargs in daofind_args: - # Run DAOStarFind with the given arguments: - daofind_tbl = DAOStarFinder(**kwargs).find_stars(image.subclean, mask=image.mask) - - # Match the found stars with the catalog references: - indx_good = np.zeros(len(references), dtype='bool') - for k, ref in enumerate(references): - dist = np.sqrt( (daofind_tbl['xcentroid'] - ref['pixel_column'])**2 + (daofind_tbl['ycentroid'] - ref['pixel_row'])**2 ) - if np.any(dist <= fwhm/4): # Cutoff set somewhat arbitrary - indx_good[k] = True - - logger.debug("Number of references after cleaning: %d", np.sum(indx_good)) - if np.sum(indx_good) >= min_references: - references = references[indx_good] - break - daofind_tbl = DAOStarFinder(**daofind_args[0]).find_stars(image.subclean, mask=image.mask) + references = get_clean_references(references, masked_rsqs) logger.debug("Number of references after cleaning: %d", len(references)) - # Further clean references based on 2D gaussian fits + + # + # # This cleaning is good when we have MANY references. + # # Use DAOStarFinder to search the image for stars, and only use reference-stars where a + # # star was actually detected close to the references-star coordinate: + # cleanout_references = (len(references) > 6) + # logger.debug("Number of references before cleaning: %d", len(references)) + # if cleanout_references: + # # Arguments for DAOStarFind, starting with the strictest and ending with the + # # least strict settings to try: + # # We will stop with the first set that yield more than the minimum number + # # of reference stars. + # daofind_args = [{ + # 'threshold': 7 * image.std, + # 'fwhm': fwhm, + # 'exclude_border': True, + # 'sharphi': 0.8, + # 'sigma_radius': 1.1, + # 'peakmax': image.peakmax + # }, { + # 'threshold': 3 * image.std, + # 'fwhm': fwhm, + # 'roundlo': -0.5, + # 'roundhi': 0.5 + # }] + # + # # Loop through argument sets for DAOStarFind: + # for kwargs in daofind_args: + # # Run DAOStarFind with the given arguments: + # daofind_tbl = DAOStarFinder(**kwargs).find_stars(image.subclean, mask=image.mask) + # + # # Match the found stars with the catalog references: + # indx_good = np.zeros(len(references), dtype='bool') + # for k, ref in enumerate(references): + # dist = np.sqrt( (daofind_tbl['xcentroid'] - ref['pixel_column'])**2 + (daofind_tbl['ycentroid'] - ref['pixel_row'])**2 ) + # if np.any(dist <= fwhm/4): # Cutoff set somewhat arbitrary + # indx_good[k] = True + # + # logger.debug("Number of references after cleaning: %d", np.sum(indx_good)) + # if np.sum(indx_good) >= min_references: + # references = references[indx_good] + # break + # + # logger.debug("Number of references after cleaning: %d", len(references)) # Create plot of target and reference star positions: fig, ax = plt.subplots(1, 1, figsize=(20, 18)) plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) ax.scatter(references['pixel_column'], references['pixel_row'], c='r', marker='o', alpha=0.6) - if cleanout_references: - ax.scatter(daofind_tbl['xcentroid'], daofind_tbl['ycentroid'], c='g', marker='o', alpha=0.6) + ax.scatter(masked_sep_xy[:,0],masked_sep_xy[:,1], marker='s' , alpha=0.6, edgecolors='green' ,facecolors='none') ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') fig.savefig(os.path.join(output_folder, 'positions.png'), bbox_inches='tight') plt.close(fig) @@ -490,6 +446,24 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): fig.savefig(os.path.join(output_folder, 'epsf_stars%02d.png' % (k+1)), bbox_inches='tight') plt.close(fig) + # # Test epsf builder: + # fig, axs = plt.subplots(3,2) + # for maxiters,ax in zip([1,10,20,30,40,50],axs.flat): + # epsf = EPSFBuilder( + # oversampling=1.0, + # maxiters=maxiters, + # fitter=EPSFFitter(fit_boxsize=2 * fwhm), + # progress_bar=True + # )(stars)[0] + # + # ax.imshow(epsf.data, cmap='viridis') + # ax.set_title(maxiters) + # fig.savefig(os.path.join(output_folder, 'epsftest.png'), bbox_inches='tight') + # plt.close(fig) + # + # logging.info('FWHM = {}'.format(fwhm)) + # return stars + # Build the ePSF: epsf = EPSFBuilder( oversampling=1.0, @@ -806,4 +780,4 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): logger.info("Main target: %f +/- %f", tab[0]['mag'], tab[0]['mag_error']) logger.info("Photometry took: %f seconds", toc-tic) - return photometry_output,objects,masked_sep_xy,debug_references,masked_sep_rsqs,sep_mask,image.wcs + return photometry_output diff --git a/flows/wcs.py b/flows/wcs.py new file mode 100644 index 0000000..691ed29 --- /dev/null +++ b/flows/wcs.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Clean bad source extraction, find and correct wcs + +.. codeauthor:: Emir Karamehmetoglu +""" + +import numpy as np +from astropy.table import Table +import astroalign as aa +import astropy.units as u +import astropy.coordinates as coords +import astropy.wcs as wcs +import astropy.io.fits as fits +from astropy.stats import sigma_clip, SigmaClip, gaussian_fwhm_to_sigma +from astropy.modeling import models, fitting +from copy import deepcopy +from bottleneck import nanmedian + +class MinStarError(RuntimeError): + pass + +def force_reject_g2d(xarray, yarray, image, get_fwhm=True, rsq_min=0.5, radius=10, fwhm_guess=6.0, fwhm_min=3.5, + fwhm_max=18.0): + '''xarray, yarray, image, get_fwhm=True, rsq_min=0.5, radius=10, fwhm_guess=6.0, fwhm_min=3.5, + fwhm_max=18.0''' + # Set up 2D Gaussian model for fitting to reference stars: + g2d = models.Gaussian2D(amplitude=1.0, x_mean=radius, y_mean=radius, x_stddev=fwhm_guess * gaussian_fwhm_to_sigma) + g2d.amplitude.bounds = (0.1, 2.0) + g2d.x_mean.bounds = (0.5 * radius, 1.5 * radius) + g2d.y_mean.bounds = (0.5 * radius, 1.5 * radius) + g2d.x_stddev.bounds = (fwhm_min * gaussian_fwhm_to_sigma, fwhm_max * gaussian_fwhm_to_sigma) + g2d.y_stddev.tied = lambda model: model.x_stddev + g2d.theta.fixed = True + + gfitter = fitting.LevMarLSQFitter() + + # Stars reject + N = len(xarray) + fwhms = np.full((N, 2), np.NaN) + xys = np.full((N, 2), np.NaN) + rsqs = np.full(N, np.NaN) + for i, (x, y) in enumerate(zip(xarray, yarray)): + x = int(np.round(x)) + y = int(np.round(y)) + xmin = max(x - radius, 0) + xmax = min(x + radius + 1, image.shape[1]) + ymin = max(y - radius, 0) + ymax = min(y + radius + 1, image.shape[0]) + + curr_star = deepcopy(image.subclean[ymin:ymax, xmin:xmax]) + + edge = np.zeros_like(curr_star, dtype='bool') + edge[(0, -1), :] = True + edge[:, (0, -1)] = True + curr_star -= nanmedian(curr_star[edge]) + curr_star /= np.max(curr_star) + + ypos, xpos = np.mgrid[:curr_star.shape[0], :curr_star.shape[1]] + gfit = gfitter(g2d, x=xpos, y=ypos, z=curr_star) + + # Center + xys[i] = np.array([gfit.x_mean + x - radius, gfit.y_mean + y - radius], dtype=np.float64) + # Calculate rsq + sstot = ((curr_star - curr_star.mean()) ** 2).sum() + sserr = (gfitter.fit_info['fvec'] ** 2).sum() + rsqs[i] = 1. - (sserr / sstot) + # FWHM + fwhms[i] = gfit.x_fwhm + + masked_xys = np.ma.masked_array(xys, ~np.isfinite(xys)) + masked_rsqs = np.ma.masked_array(rsqs, ~np.isfinite(rsqs)) + mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) # Reject Rsq<0.5 + masked_xys = masked_xys[mask] # Clean extracted array. + masked_fwhms = np.ma.masked_array(fwhms, ~np.isfinite(fwhms)) + + if get_fwhm: return masked_fwhms,masked_xys,mask,masked_rsqs + return masked_xys,mask,masked_rsqs + + + +def clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, references, + min_fwhm_references = 2, min_references = 6, rsq_min = 0.15): + """ + Clean references and obtain fwhm using RSQ values. + Args: + masked_fwhms (np.ma.maskedarray): array of fwhms + masked_rsqs (np.ma.maskedarray): array of rsq values + references (astropy.table.Table): table or reference stars + min_fwhm_references: (Default 2) min stars to get a fwhm + min_references: (Default 6) min stars to aim for when cutting by R2 + rsq_min: (Default 0.15) min rsq value + """ + min_references_now = min_references + rsqvals = np.arange(rsq_min, 0.95, 0.15)[::-1] + fwhm_found = False + min_references_achieved = False + + # Clean based on R^2 Value + while not min_references_achieved: + for rsqval in rsqvals: + mask = (masked_rsqs >= rsqval) & (masked_rsqs < 1.0) + nreferences = np.sum(np.isfinite(masked_fwhms[mask])) + if nreferences >= min_fwhm_references: + _fwhms_cut_ = np.nanmean(sigma_clip(masked_fwhms[mask], maxiters=100, sigma=2.0)) + #logger.info('R^2 >= ' + str(rsqval) + ': ' + str( + # np.sum(np.isfinite(masked_fwhms[mask]))) + ' stars w/ mean FWHM = ' + str(np.round(_fwhms_cut_, 1))) + if not fwhm_found: + fwhm = _fwhms_cut_ + fwhm_found = True + if nreferences >= min_references_now: + references = references[mask] + min_references_achieved = True + break + if min_references_achieved: break + min_references_now = min_references_now - 2 + if (min_references_now < 2) and fwhm_found: + break + elif not fwhm_found: + raise Exception("Could not estimate FWHM") + #logger.debug('{} {} {}'.format(min_references_now, min_fwhm_references, nreferences)) + + #logger.info("FWHM: %f", fwhm) + if np.isnan(fwhm): + raise Exception("Could not estimate FWHM") + + # if minimum references not found, then take what we can get with even a weaker cut. + # TODO: Is this right, or should we grab rsq_min (or even weaker?) + min_references_now = min_references - 2 + while not min_references_achieved: + mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) + nreferences = np.sum(np.isfinite(masked_fwhms[mask])) + if nreferences >= min_references_now: + references = references[mask] + min_references_achieved = True + rsq_min = rsq_min - 0.07 + min_references_now = min_references_now - 1 + + # Check len of references as this is a destructive cleaning. + # if len(references) == 2: logger.info('2 reference stars remaining, check WCS and image quality') + if len(references) < 2: + raise Exception("{} References remaining; could not clean.".format(len(references))) + return fwhm, references + + +def mkposxy(posx, posy): + '''Make 2D np array for astroalign''' + img_posxy = np.array([[x, y] for x, y in zip(posx, posy)], dtype="float64") + return img_posxy + +def try_transform(source, target, pixeltol=2, nnearest=5, max_stars=50): + aa.NUM_NEAREST_NEIGHBORS = nnearest + aa.PIXEL_TOL = pixeltol + transform,(sourcestars, targetstars) = aa.find_transform(source, target, max_control_points=max_stars) + return sourcestars, targetstars + +def try_astroalign(source, target, pixeltol=2, nnearest=5, max_stars_n=50): + # Get indexes of matched stars + success = False + try: + source_stars, target_stars = try_transform(source, target, + pixeltol=pixeltol, nnearest=nnearest, max_stars=max_stars_n) + source_ind = np.argwhere(np.in1d(source, source_stars)[::2]).flatten() + target_ind = np.argwhere(np.in1d(target, target_stars)[::2]).flatten() + success = True + except aa.MaxIterError: + source_ind, target_ind = 'None', 'None' + return source_ind,target_ind,success + + +def min_to_max_astroalign(source, target, fwhm=5, fwhm_min=1, fwhm_max=4, knn_min=5, knn_max=20, + max_stars=100, min_matches=3): + '''Try to find matches using astroalign asterisms by stepping through some parameters.''' + # Set max_control_points par based on number of stars and max_stars. + nstars = max(len(source), len(source)) + if max_stars >= nstars : max_stars_list = 'None' + else: + if max_stars > 60: max_stars_list = (max_stars,50,4,3) + else: max_stars_list = (max_stars,6,4,3) + + # Create max_stars step-through list if not given + if max_stars_list == 'None': + if nstars > 6: + max_stars_list = (nstars, 5, 3) + elif nstars > 3: + max_stars_list = (nstars, 3) + + pixeltols = np.linspace(int(fwhm*fwhm_min), int(fwhm*fwhm_max), 4, dtype=int) + nearest_neighbors = np.linspace(knn_min, min(knn_max,nstars), 4, dtype=int) + + for max_stars_n in max_stars_list: + for pixeltol in pixeltols: + for nnearest in nearest_neighbors: + source_ind,target_ind,success = try_astroalign(source, + target, + pixeltol=pixeltol, + nnearest=nnearest, + max_stars_n=max_stars_n) + if success: + if len(source_ind) >= min_matches: + return source_ind, target_ind, success + else: success = False + return 'None', 'None', success + +def kdtree(source, target, fwhm=5, fwhm_max=4, min_matches=3): + '''Use KDTree to get nearest neighbor matches within fwhm_max*fwhm distance''' + + # Use KDTree to rapidly efficiently query nearest neighbors + from scipy.spatial import KDTree + tt = KDTree(target) + st = KDTree(source) + matches_list = st.query_ball_tree(tt, r=fwhm*fwhm_max) + + #indx = [] + targets = [] + sources = [] + for j, (sstar, match) in enumerate(zip(source, matches_list)): + if np.array(target[match]).size != 0: + targets.append(match[0]) + sources.append(j) + sources = np.array(sources, dtype=int) + targets = np.array(targets, dtype=int) + # Return indexes of matches + return sources, targets, len(sources)>= min_matches + +def get_new_wcs(extracted_ind,extracted_stars,clean_references,ref_ind,obstime): + targets = (extracted_stars[extracted_ind][:,0],extracted_stars[extracted_ind][:,1]) + c = coords.SkyCoord(clean_references['ra_obs'][ref_ind],clean_references['decl_obs'][ref_ind],obstime=obstime) + return wcs.utils.fit_wcs_from_points(targets,c) + +def get_clean_references(references, masked_rsqs, min_references_ideal=6, + min_references_abs=3, rsq_min=0.15, rsq_ideal=0.5, + rescue_bad: bool = True): + mask = (masked_rsqs >= rsq_ideal) & (masked_rsqs < 1.0) + if np.sum(np.isfinite(masked_rsqs[mask])) >= min_references_ideal: + return references[mask] + mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) + nmasked_rsqs = np.argsort(masked_rsqs[mask])[::-1] + nmasked_rsqs = nmasked_rsqs[:min(min_references_ideal, len(nmasked_rsqs))] + if len(nmasked_rsqs>=min_references_abs): + return references[nmasked_rsqs] + if not rescue_bad: + raise MinStarError('Less than {} clean stars and rescue_bad = False'.format(min_references_abs)) + elif rescue_bad: + mask = (masked_rsqs >= 0.02) & (masked_rsqs < 1.0) + nmasked_rsqs = np.argsort(masked_rsqs[mask])[::-1] + nmasked_rsqs = nmasked_rsqs[:min(min_references_ideal, len(nmasked_rsqs))] + if len(nmasked_rsqs) < 2 : + raise MinStarError('Less than 2 clean stars.') + return references[nmasked_rsqs] + raise ValueError('input parameters were wrong, you should not reach here. Check that rescue_bad is True or False.') + + diff --git a/requirements.txt b/requirements.txt index db9c492..d4ebc04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,8 +9,8 @@ matplotlib == 3.3.1 mplcursors == 0.3 seaborn requests -astropy < 4.0 -photutils < 0.7 +astropy >=4.2 +photutils > 1.0 PyYAML psycopg2-binary jplephem @@ -20,3 +20,6 @@ tqdm pytz beautifulsoup4 git+https://github.com/obscode/imagematch.git@photutils#egg=imagematch +pandas +sep +astroalign > 2.3 \ No newline at end of file diff --git a/run_photometry.py b/run_photometry.py index bb1ec6c..1667317 100644 --- a/run_photometry.py +++ b/run_photometry.py @@ -146,40 +146,4 @@ def process_fileid(fid, output_folder_root=None, attempt_imagematch=True, autoup print("="*72) print(fid) print("="*72) - _,objects,masked_sep_xy,debug_references,masked_sep_rsqs,sep_mask,wcs = process_fileid_wrapper(fid) - - #from astropy.table import Table - import numpy as np - import astroalign as aa - import matplotlib - matplotlib.use('qt5agg') - import matplotlib.pyplot as plt - - def mkposxy(posx, posy): - img_posxy = np.array([[x, y] for x, y in zip(posx, posy)], dtype="float64") - return img_posxy - - - aa.NUM_NEAREST_NEIGHBORS = 5 - aa.MIN_MATCHES_FRACTION = 0.8 - aa.PIXEL_TOL = 5 - - #import pandas as pd - from astropy.table import Table - objects=Table(objects) - #objects.sort('flux',reverse=True) - #daofind_tbl.sort('flux', reverse=True) - debug_references.sort('g_mag') - at = Table({'xy': masked_sep_xy, 'flux': objects['flux'][sep_mask]}) - at.sort('flux',reverse=True) - masked_sep_xy = at['xy'].data.data - - source = mkposxy(debug_references['pixel_column'].data,debug_references['pixel_row'].data) - target = masked_sep_xy - - plt.scatter(source[:,0],source[:,1],label='references',alpha=0.5,marker='d') - plt.scatter(masked_sep_xy[:,0],masked_sep_xy[:,1],label='clean sep',alpha=0.5) - plt.legend() - plt.show(block=True) - - out = aa.find_transform(source,target,max_control_points=3) \ No newline at end of file + process_fileid_wrapper(fid) \ No newline at end of file From 3e435ada1ed0713dcfd4f7d36784a5d6eecc2f25 Mon Sep 17 00:00:00 2001 From: Simon Holmbo Date: Tue, 12 Jan 2021 01:14:00 +0100 Subject: [PATCH 03/43] Adjust WCS with identified stars --- flows/photometry.py | 91 ++++++++++++++------ flows/wcs.py | 198 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 263 insertions(+), 26 deletions(-) create mode 100644 flows/wcs.py diff --git a/flows/photometry.py b/flows/photometry.py index c8c48af..dcb0291 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -39,6 +39,8 @@ from .run_imagematch import run_imagematch from .zeropoint import bootstrap_outlier, sigma_from_Chauvenet +from .wcs import WCS + __version__ = get_version(pep440=False) warnings.simplefilter('ignore', category=AstropyDeprecationWarning) @@ -120,34 +122,18 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): logger.warning("Could not find filter '%s' in catalogs. Using default gp filter.", photfilter) ref_filter = 'g_mag' - references = catalog['references'] - references.sort(ref_filter) - # Check that there actually are reference stars in that filter: - if allnan(references[ref_filter]): + if allnan(catalog['references'][ref_filter]): raise ValueError("No reference stars found in current photfilter.") # Load the image from the FITS file: image = load_image(filepath) - # Calculate pixel-coordinates of references: - row_col_coords = image.wcs.all_world2pix(np.array([[ref['ra'], ref['decl']] for ref in references]), 0) - references['pixel_column'] = row_col_coords[:,0] - references['pixel_row'] = row_col_coords[:,1] + references = extract_references(catalog, image.wcs, ref_filter, hsize, ref_target_dist_limit) # Calculate the targets position in the image: target_pixel_pos = image.wcs.all_world2pix([[target['ra'], target['decl']]], 0)[0] - # Clean out the references: - hsize = 10 - x = references['pixel_column'] - y = references['pixel_row'] - refs_coord = coords.SkyCoord(ra=references['ra'], dec=references['decl'], unit='deg', frame='icrs') - references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) - & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) - & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] - # & (references[ref_filter] < ref_mag_limit) - #============================================================================================== # BARYCENTRIC CORRECTION OF TIME #============================================================================================== @@ -271,19 +257,51 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # Loop through argument sets for DAOStarFind: for kwargs in daofind_args: + # Run DAOStarFind with the given arguments: daofind_tbl = DAOStarFinder(**kwargs).find_stars(image.subclean, mask=image.mask) - # Match the found stars with the catalog references: - indx_good = np.zeros(len(references), dtype='bool') - for k, ref in enumerate(references): - dist = np.sqrt( (daofind_tbl['xcentroid'] - ref['pixel_column'])**2 + (daofind_tbl['ycentroid'] - ref['pixel_row'])**2 ) - if np.any(dist <= fwhm/4): # Cutoff set somewhat arbitrary - indx_good[k] = True + def match_catalog_to_image_with_wcs(wcs): + + references = extract_references(catalog, wcs, ref_filter, hsize, ref_target_dist_limit) + + xy_stars_grid = np.tile(daofind_tbl['xcentroid'], (len(references), 1)), np.tile(daofind_tbl['ycentroid'], (len(references), 1)) + xy_references = np.array([references['pixel_column'], references['pixel_row']]).reshape([2, len(references), 1]) + distances = np.sqrt(np.power(xy_stars_grid - xy_references, 2).sum(axis=0)) + idx_refs, idx_stars = np.any(distances <= fwhm/4, axis=1) + + return references, (idx_refs, idx_stars) + + # indices of matching reference stars and image stars + idx_refs, idx_stars = match_catalog_to_image_with_wcs(image.wcs)[1] + + while True: + + wcs_new = WCS.from_astropy_wcs(image.wcs) + wcs_new.adjust_with_points( + xy = list(zip(daofind_tbl['xcentroid'][idx_stars], daofind_tbl['ycentroid'][idx_stars])), + rd = list(zip(references['ra'][idx_refs], references['decl'][idx_refs])) + ) + wcs_new = wcs_new.astropy_wcs + + references_new, (idx_refs_new, idx_stars_new) = match_catalog_to_image_with_wcs(wcs_new) + + # break out if wcs_new did not locate more stars + if len(idx_refs_new) <= len(idx_refs): + break + + references, image.wcs = references_new, wcs_new + idx_refs, idx_stars = idx_refs_new, idx_stars_new + + indx_good = idx_refs + + logger.debug("Number of references after cleaning: %d", len(indx_good)) + + if len(indx_good) >= min_references: - logger.debug("Number of references after cleaning: %d", np.sum(indx_good)) - if np.sum(indx_good) >= min_references: references = references[indx_good] + target_pixel_pos = image.wcs.all_world2pix([[target['ra'], target['decl']]], 0)[0] + break logger.debug("Number of references after cleaning: %d", len(references)) @@ -664,3 +682,24 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): logger.info("Photometry took: %f seconds", toc-tic) return photometry_output + +#-------------------------------------------------------------------------------------------------- + +def extract_references(catalog, wcs, ref_filter='g_mag', hsize=10, ref_target_dist_limit=10*u.arcsec): + + references, target = catalog['references'], catalog['target'][0] + references.sort(ref_filter) + + references['pixel_column'], references['pixel_row'] = x, y = \ + list(zip(*wcs.all_world2pix(list(zip(references['ra'], references['decl'])), 0))) + + targ_coord = coords.SkyCoord(ra=target['ra'], dec=target['decl'], unit='deg', frame='icrs') + refs_coord = coords.SkyCoord(ra=references['ra'], dec=references['decl'], unit='deg', frame='icrs') + references = references[ + (targ_coord.separation(refs_coord) > ref_target_dist_limit) & + (x > hsize) & (x < (image.shape[1] - 1 - hsize)) & + (y > hsize) & (y < (image.shape[0] - 1 - hsize)) + ] + # & (references[ref_filter] < ref_mag_limit) + + return references diff --git a/flows/wcs.py b/flows/wcs.py new file mode 100644 index 0000000..e835cab --- /dev/null +++ b/flows/wcs.py @@ -0,0 +1,198 @@ +from copy import deepcopy + +import numpy as np +import astropy.wcs + +from scipy.optimize import minimize +from scipy.spatial.transform import Rotation + +class WCS () : + '''Manipulate WCS solution. + + Initialize + ---------- + wcs = WCS(x, y, ra, dec, scale, mirror, angle) + wcs = WCS.from_matrix(x, y, ra, dec, matrix) + wcs = WCS.from_points(list(zip(x, y)), list(zip(ra, dec))) + wcs = WCS.from_astropy_wcs(astropy.wcs.WCS()) + + ra, dec and angle should be in degrees + scale should be in arcsec/pixel + matrix should be the PC or CD matrix + + Examples + -------- + Adjust x, y offset: + wcs.x += delta_x + wcs.y += delta_y + + Get scale and angle: + print(wcs.scale, wcs.angle) + + Change an astropy.wcs.WCS (wcs) angle + wcs = WCS(wcs)(angle=new_angle).astropy_wcs + + Adjust solution with points + wcs.adjust_with_points(list(zip(x, y)), list(zip(ra, dec))) + ''' + + def __init__(self, x, y, ra, dec, scale, mirror, angle): + + self.x, self.y = x, y + self.ra, self.dec = ra, dec + self.scale = scale + self.mirror = mirror + self.angle = angle + + @classmethod + def from_matrix(cls, x, y, ra, dec, matrix): + + assert np.shape(matrix) == (2, 2), \ + 'Matrix must be 2x2' + + scale, mirror, angle = cls._decompose_matrix(matrix) + + return cls(x, y, ra, dec, scale, mirror, angle) + + @classmethod + def from_points(cls, xy, rd): + + assert np.shape(xy) == np.shape(rd) == (len(xy), 2) and len(xy) > 2, \ + 'Arguments must be lists of at least 3 sets of coordinates' + + xy, rd = np.array(xy), np.array(rd) + + x, y, ra, dec, matrix = cls._solve_from_points(xy, rd) + scale, mirror, angle = cls._decompose_matrix(matrix) + + return cls(x, y, ra, dec, scale, mirror, angle) + + @classmethod + def from_astropy_wcs(cls, astropy_wcs): + + assert type(astropy_wcs) is astropy.wcs.WCS, \ + 'Must be astropy.wcs.WCS' + + (x, y), (ra, dec) = astropy_wcs.wcs.crpix, astropy_wcs.wcs.crval + scale, mirror, angle = cls._decompose_matrix(astropy_wcs.pixel_scale_matrix) + + return cls(x, y, ra, dec, scale, mirror, angle) + + def adjust_with_points(self, xy, rd): + + assert np.shape(xy) == np.shape(rd) == (len(xy), 2), \ + 'Arguments must be lists of sets of coordinates' + + xy, rd = np.array(xy), np.array(rd) + + self.x, self.y = xy.mean(axis=0) + self.ra, self.dec = rd.mean(axis=0) + + A, b = xy - xy.mean(axis=0), rd - rd.mean(axis=0) + b[:,0] *= np.cos(np.deg2rad(rd[:,1])) + + if len(xy) == 2: + + M = np.diag([[-1, 1][self.mirror], 1]) + R = lambda t: np.array([[np.cos(t), -np.sin(t)], [np.sin(t), np.cos(t)]]) + + chi2 = lambda x: np.power(A.dot(x[1]/60/60*R(x[0]).dot(M).T) - b, 2).sum() + self.angle, self.scale = minimize(chi2, [self.angle, self.scale]).x + + elif len(xy) > 2: + + matrix = np.linalg.lstsq(A, b, rcond=None)[0].T + self.scale, self.mirror, self.angle = self._decompose_matrix(matrix) + + @property + def matrix(self): + + scale = self.scale / 60 / 60 + mirror = np.diag([[-1, 1][self.mirror], 1]) + angle = np.deg2rad(self.angle) + + matrix = np.array([ + [np.cos(angle), -np.sin(angle)], + [np.sin(angle), np.cos(angle)] + ]) + + return scale * matrix @ mirror + + @property + def astropy_wcs(self): + + wcs = astropy.wcs.WCS() + wcs.wcs.crpix = self.x, self.y + wcs.wcs.crval = self.ra, self.dec + wcs.wcs.pc = self.matrix + + return wcs + + @staticmethod + def _solve_from_points(xy, rd): + + (x, y), (ra, dec) = xy.mean(axis=0), rd.mean(axis=0) + + A, b = xy - xy.mean(axis=0), rd - rd.mean(axis=0) + b[:,0] *= np.cos(np.deg2rad(rd[:,1])) + + matrix = np.linalg.lstsq(A, b, rcond=None)[0].T + + return x, y, ra, dec, matrix + + @staticmethod + def _decompose_matrix(matrix): + + scale = np.sqrt(np.power(matrix, 2).sum() / 2) * 60 * 60 + + if np.argmax(np.power(matrix[0], 2)): + mirror = True if np.sign(matrix[0,1]) != np.sign(matrix[1,0]) else False + else: + mirror = True if np.sign(matrix[0,0]) == np.sign(matrix[1,1]) else False + + matrix = matrix if mirror else matrix.dot(np.diag([-1, 1])) + + matrix3d = np.eye(3); matrix3d[:2,:2] = matrix / (scale / 60 / 60) + angle = Rotation.from_matrix(matrix3d).as_euler('xyz', degrees=True)[2] + + return scale, mirror, angle + + def __setattr__(self, name, value): + + if name == 'ra': + + assert 0 <= value < 360, '0 <= R.A. < 360' + + elif name == 'dec': + + assert -180 <= value <= 180, '-180 <= Dec. <= 180' + + elif name == 'scale': + + assert value > 0, 'Scale > 0' + + elif name == 'mirror': + + assert type(value) is bool, 'Mirror = True | False' + + elif name == 'angle': + + assert -180 < value <= 180, '-180 < Angle <= 180' + + super().__setattr__(name, value) + + def __call__(self, **kwargs): + + keys = ('x', 'y', 'ra', 'dec', 'scale', 'mirror', 'angle') + + if not all(k in keys for k in kwargs): + + raise Exception('unknown argument(s)') + + obj = deepcopy(self) + + for k, v in kwargs.items(): + + obj.__setattr__(k, v) + + return obj From fffc31a4a94506ee2651a7562d3a3eb8e12b5e64 Mon Sep 17 00:00:00 2001 From: Emir Date: Fri, 15 Jan 2021 12:13:15 +0100 Subject: [PATCH 04/43] Remove astroalign & improve wcs checking loop astroalign is sometimes returning false matches so we remove it for now until we can fix this. Also some changes to the wcs_checking loop to catch whether references left the frame or got near the edge after the new wcs. --- flows/photometry.py | 248 ++++++++++++++++++++++---------------------- 1 file changed, 123 insertions(+), 125 deletions(-) diff --git a/flows/photometry.py b/flows/photometry.py index 8968337..a8eb12a 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -13,8 +13,6 @@ from timeit import default_timer import logging import warnings -from copy import deepcopy -import astroalign as aa from astropy.utils.exceptions import AstropyDeprecationWarning import astropy.units as u @@ -31,6 +29,7 @@ from photutils.psf import EPSFBuilder, EPSFFitter, BasicPSFPhotometry, DAOGroup, extract_stars from photutils import Background2D, SExtractorBackground from photutils.utils import calc_total_error +from photutils.centroids import centroid_com from scipy.interpolate import UnivariateSpline @@ -42,7 +41,7 @@ from .run_imagematch import run_imagematch from .zeropoint import bootstrap_outlier, sigma_from_Chauvenet from .wcs import force_reject_g2d, mkposxy, clean_with_rsq_and_get_fwhm, \ - min_to_max_astroalign, kdtree, get_new_wcs, get_clean_references + try_astroalign, kdtree, get_new_wcs, get_clean_references __version__ = get_version(pep440=False) @@ -156,7 +155,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): x = references['pixel_column'] y = references['pixel_row'] refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', frame='icrs') - references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) + clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] # & (references[ref_filter] < ref_mag_limit) @@ -215,7 +214,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # DETECTION OF STARS AND MATCHING WITH CATALOG #============================================================================================== - logger.info("References:\n%s", references) + logger.info("References:\n%s", clean_references) radius = 10 fwhm_guess = 6.0 @@ -227,8 +226,8 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): radius=radius, fwhm_guess=fwhm_guess, rsq_min=0.3, fwhm_max=fwhm_max, fwhm_min=fwhm_min) # Clean reference star locations - masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(references['pixel_column'], - references['pixel_row'], + masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], + clean_references['pixel_row'], image, get_fwhm=True, radius=radius, @@ -240,14 +239,14 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # Use R^2 to more robustly determine initial FWHM guess. # This cleaning is good when we have FEW references. - fwhm, clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, references, + fwhm, clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, min_fwhm_references=2, min_references=6, rsq_min=0.15) # Create plot of target and reference star positions from 2D Gaussian fits. fig, ax = plt.subplots(1, 1, figsize=(20, 18)) plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) - ax.scatter(references['pixel_column'], references['pixel_row'], c='r', marker='o', alpha=0.3) - ax.scatter(clean_references['pixel_column'], clean_references['pixel_row'], c='yellow', marker='o', alpha=0.3) + #ax.scatter(references['pixel_column'], references['pixel_row'], c='r', marker='o', alpha=0.3) + ax.scatter(clean_references['pixel_column'], clean_references['pixel_row'], c='r', marker='o', alpha=0.3) #ax.scatter(masked_ref_xys[:,0], masked_ref_xys[:,0], marker='o', alpha=0.6, edgecolors='green', facecolors='none') ax.scatter(masked_sep_xy[:,0],masked_sep_xy[:,1],marker='s',alpha=1.0, edgecolors='green' ,facecolors='none') ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') @@ -264,72 +263,106 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): wcs_rotation = 0 wcs_rota_max = 3 nreferences = len(clean_references['pixel_column']) - + try_aa = True while wcs_rotation < wcs_rota_max: nreferences_old = nreferences ref_xys = mkposxy(clean_references['pixel_column'], clean_references['pixel_row']) - clean_coords = coords.SkyCoord(clean_references['ra'], clean_references['decl'],obstime=Time(2015.5, format='decimalyear'), - pm_ra_cosdec = clean_references['pm_ra'], pm_dec = clean_references['pm_dec'], - distance = 1 * u.kpc, radial_velocity = 1000 * u.km / u.s) - # Find matches using astroalign - #ref_ind, sep_ind, success_aa = min_to_max_astroalign(ref_xys, masked_sep_xy, fwhm=fwhm, fwhm_min=2, - # fwhm_max=4, knn_min=5, knn_max=25, max_stars=100, - # min_matches=3) - # Basically if aa is failing us by not finding more than 3 matches, use KDtree for the last iteration. - try_kd = True - success_aa = False + # try_kd = True + # if try_aa: + # for maxstars in [80,4]: + # ref_ind, sep_ind, success_aa = try_astroalign(ref_xys, masked_sep_xy, + # pixeltol=4*fwhm, + # nnearest=min(20,len(ref_xys)), + # max_stars_n=max(maxstars,len(ref_xys))) + # # Break if successful + # if success_aa: + # astroalign_nmatches = len(ref_ind) + # try_kd = False + # if wcs_rotation > 1 and astroalign_nmatches <= 4: + # try_kd = True + # success_aa = False + # break - #if success_aa: - # astroalign_nmatches = len(ref_ind) - # if wcs_rotation > 1 and astroalign_nmatches <= 3: - # try_kd = True + try_kd = True + success_aa, try_aa = False, False # Don't use astroalign for now; it's giving false matches! # Find matches using nearest neighbor - #if not success_aa or try_kd: - #fwhm_max = wcs_rotation - ref_ind_kd, sep_ind_kd, success_kd = kdtree(ref_xys, masked_sep_xy, fwhm, fwhm_max=4) - if success_kd: - kdtree_nmatches = len(ref_ind_kd) - if try_kd and kdtree_nmatches > 3: - ref_ind = ref_ind_kd - sep_ind = sep_ind_kd - else: - success_kd = False + if try_kd: + ref_ind_kd, sep_ind_kd, success_kd = kdtree(ref_xys, masked_sep_xy, fwhm, fwhm_max=4) + if success_kd: + kdtree_nmatches = len(ref_ind_kd) + if try_kd and kdtree_nmatches > 3: + ref_ind = ref_ind_kd + sep_ind = sep_ind_kd + else: + success_kd = False if success_aa or success_kd: # Fit for new WCS - image.new_wcs = get_new_wcs(sep_ind, masked_sep_xy, clean_references, ref_ind, image.obstime) wcs_rotation += 1 + image.new_wcs = get_new_wcs(sep_ind, masked_sep_xy, clean_references, ref_ind, image.obstime) # Calculate pixel-coordinates of references: - row_col_coords = image.new_wcs.all_world2pix(np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]),0) + row_col_coords = image.new_wcs.all_world2pix( + np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) references['pixel_column'] = row_col_coords[:, 0] references['pixel_row'] = row_col_coords[:, 1] - masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(references['pixel_column'], - references['pixel_row'], - image, - get_fwhm=True, - radius=radius, - fwhm_guess=fwhm, - fwhm_max=fwhm_max, - fwhm_min=fwhm_min, - rsq_min=0.15) - # Clean with R^2 - fwhm, clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, references, - min_fwhm_references=2, min_references=6, rsq_min=0.15) - image.fwhm = fwhm - - nreferences_new = len(clean_references) - logging.info('{} References were found after new wcs compared to {} references before'.format(nreferences_old,nreferences_new)) - nreferences = nreferences_new - wcs_success = True - - # Break early if no improvement after 2nd pass. - if wcs_rotation > 1 and nreferences_new <= nreferences_old: - break + # Clean out the references: + hsize = 10 + x = references['pixel_column'] + y = references['pixel_row'] + refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', + frame='icrs') + clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) + & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) + & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] + try: + masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], + clean_references['pixel_row'], + image, + get_fwhm=True, + radius=radius, + fwhm_guess=fwhm, + fwhm_max=fwhm_max, + fwhm_min=fwhm_min, + rsq_min=0.15) + # Clean with R^2 + fwhm, clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, + min_fwhm_references=2, min_references=6, rsq_min=0.15) + image.fwhm = fwhm + nreferences_new = len(clean_references) + logging.info('{} References were found after new wcs compared to {} references before'.format(nreferences_old,nreferences_new)) + nreferences = nreferences_new + wcs_success = True + + # Break early if no improvement after 2nd pass! + # Note: New references can actually be less in a better WCS + # if the actual stars were within radius (10) pixels of the edge. + # @TODO: Adjust nreferences new and old based on whether extracted stars are within radius pix of edge. + if wcs_rotation > 1 and nreferences_new <= nreferences_old: + break + except: + # Calculate pixel-coordinates of references using old wcs: + row_col_coords = image.wcs.all_world2pix( + np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) + references['pixel_column'] = row_col_coords[:, 0] + references['pixel_row'] = row_col_coords[:, 1] + + # Clean out the references: + hsize = 10 + x = references['pixel_column'] + y = references['pixel_row'] + refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', + frame='icrs') + clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) + & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) + & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] + wcs_success = False + if try_aa: try_aa = False + elif try_kd: break else: logging.info('New WCS could not be computed due to lack of matches.') @@ -339,55 +372,37 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): if wcs_success: image.wcs = image.new_wcs + # @Todo: Is the below block needed? # Final cleanout of references + # Calculate pixel-coordinates of references using old wcs: + row_col_coords = image.wcs.all_world2pix( + np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) + references['pixel_column'] = row_col_coords[:, 0] + references['pixel_row'] = row_col_coords[:, 1] + + # Clean out the references: + hsize = 10 + x = references['pixel_column'] + y = references['pixel_row'] + refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', + frame='icrs') + references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) + & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) + & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] + masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(references['pixel_column'], + references['pixel_row'], + image, + get_fwhm=True, + radius=radius, + fwhm_guess=fwhm, + fwhm_max=fwhm_max, + fwhm_min=fwhm_min, + rsq_min=0.15) + logger.debug("Number of references before cleaning: %d", len(references)) references = get_clean_references(references, masked_rsqs) logger.debug("Number of references after cleaning: %d", len(references)) - # - # # This cleaning is good when we have MANY references. - # # Use DAOStarFinder to search the image for stars, and only use reference-stars where a - # # star was actually detected close to the references-star coordinate: - # cleanout_references = (len(references) > 6) - # logger.debug("Number of references before cleaning: %d", len(references)) - # if cleanout_references: - # # Arguments for DAOStarFind, starting with the strictest and ending with the - # # least strict settings to try: - # # We will stop with the first set that yield more than the minimum number - # # of reference stars. - # daofind_args = [{ - # 'threshold': 7 * image.std, - # 'fwhm': fwhm, - # 'exclude_border': True, - # 'sharphi': 0.8, - # 'sigma_radius': 1.1, - # 'peakmax': image.peakmax - # }, { - # 'threshold': 3 * image.std, - # 'fwhm': fwhm, - # 'roundlo': -0.5, - # 'roundhi': 0.5 - # }] - # - # # Loop through argument sets for DAOStarFind: - # for kwargs in daofind_args: - # # Run DAOStarFind with the given arguments: - # daofind_tbl = DAOStarFinder(**kwargs).find_stars(image.subclean, mask=image.mask) - # - # # Match the found stars with the catalog references: - # indx_good = np.zeros(len(references), dtype='bool') - # for k, ref in enumerate(references): - # dist = np.sqrt( (daofind_tbl['xcentroid'] - ref['pixel_column'])**2 + (daofind_tbl['ycentroid'] - ref['pixel_row'])**2 ) - # if np.any(dist <= fwhm/4): # Cutoff set somewhat arbitrary - # indx_good[k] = True - # - # logger.debug("Number of references after cleaning: %d", np.sum(indx_good)) - # if np.sum(indx_good) >= min_references: - # references = references[indx_good] - # break - # - # logger.debug("Number of references after cleaning: %d", len(references)) - # Create plot of target and reference star positions: fig, ax = plt.subplots(1, 1, figsize=(20, 18)) plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) @@ -446,30 +461,13 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): fig.savefig(os.path.join(output_folder, 'epsf_stars%02d.png' % (k+1)), bbox_inches='tight') plt.close(fig) - # # Test epsf builder: - # fig, axs = plt.subplots(3,2) - # for maxiters,ax in zip([1,10,20,30,40,50],axs.flat): - # epsf = EPSFBuilder( - # oversampling=1.0, - # maxiters=maxiters, - # fitter=EPSFFitter(fit_boxsize=2 * fwhm), - # progress_bar=True - # )(stars)[0] - # - # ax.imshow(epsf.data, cmap='viridis') - # ax.set_title(maxiters) - # fig.savefig(os.path.join(output_folder, 'epsftest.png'), bbox_inches='tight') - # plt.close(fig) - # - # logging.info('FWHM = {}'.format(fwhm)) - # return stars - # Build the ePSF: epsf = EPSFBuilder( oversampling=1.0, maxiters=500, - fitter=EPSFFitter(fit_boxsize=2*fwhm), - progress_bar=True + fitter=EPSFFitter(fit_boxsize=np.round(2*fwhm,0).astype(int)), + progress_bar=True, + recentering_func=centroid_com )(stars)[0] logger.info('Successfully built PSF model') @@ -637,8 +635,8 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): tab[i][key] = np.NaN # Subtract background estimated from annuli: - flux_aperture = apphot_tbl['aperture_sum_0'] - (apphot_tbl['aperture_sum_1'] / annuli.area()) * apertures.area() - flux_aperture_error = np.sqrt(apphot_tbl['aperture_sum_err_0']**2 + (apphot_tbl['aperture_sum_err_1']/annuli.area() * apertures.area())**2) + flux_aperture = apphot_tbl['aperture_sum_0'] - (apphot_tbl['aperture_sum_1'] / annuli.area) * apertures.area + flux_aperture_error = np.sqrt(apphot_tbl['aperture_sum_err_0']**2 + (apphot_tbl['aperture_sum_err_1']/annuli.area * apertures.area)**2) # Add table columns with results: tab['flux_aperture'] = flux_aperture/image.exptime From eb8e0282e592f6a3c5b9172d78cfa1fce29b0577 Mon Sep 17 00:00:00 2001 From: Emir Date: Fri, 15 Jan 2021 19:08:45 +0100 Subject: [PATCH 05/43] Auto stash before merge of "wcs_fix" and "emirkmo/wcs_fix" --- flows/photometry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flows/photometry.py b/flows/photometry.py index a8eb12a..42eab31 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -27,6 +27,7 @@ warnings.simplefilter('ignore', category=AstropyDeprecationWarning) from photutils import DAOStarFinder, CircularAperture, CircularAnnulus, aperture_photometry from photutils.psf import EPSFBuilder, EPSFFitter, BasicPSFPhotometry, DAOGroup, extract_stars +from photutils.centroids import centroid_com from photutils import Background2D, SExtractorBackground from photutils.utils import calc_total_error from photutils.centroids import centroid_com From 145c6e9275e5b730bca52288d3700ac482bd1c57 Mon Sep 17 00:00:00 2001 From: Simon Holmbo Date: Mon, 25 Jan 2021 15:27:32 +0100 Subject: [PATCH 06/43] Revert "Adjust WCS with identified stars" This reverts commit 3e435ada1ed0713dcfd4f7d36784a5d6eecc2f25. --- flows/photometry.py | 91 ++++++-------------- flows/wcs.py | 198 -------------------------------------------- 2 files changed, 26 insertions(+), 263 deletions(-) delete mode 100644 flows/wcs.py diff --git a/flows/photometry.py b/flows/photometry.py index dcb0291..c8c48af 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -39,8 +39,6 @@ from .run_imagematch import run_imagematch from .zeropoint import bootstrap_outlier, sigma_from_Chauvenet -from .wcs import WCS - __version__ = get_version(pep440=False) warnings.simplefilter('ignore', category=AstropyDeprecationWarning) @@ -122,18 +120,34 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): logger.warning("Could not find filter '%s' in catalogs. Using default gp filter.", photfilter) ref_filter = 'g_mag' + references = catalog['references'] + references.sort(ref_filter) + # Check that there actually are reference stars in that filter: - if allnan(catalog['references'][ref_filter]): + if allnan(references[ref_filter]): raise ValueError("No reference stars found in current photfilter.") # Load the image from the FITS file: image = load_image(filepath) - references = extract_references(catalog, image.wcs, ref_filter, hsize, ref_target_dist_limit) + # Calculate pixel-coordinates of references: + row_col_coords = image.wcs.all_world2pix(np.array([[ref['ra'], ref['decl']] for ref in references]), 0) + references['pixel_column'] = row_col_coords[:,0] + references['pixel_row'] = row_col_coords[:,1] # Calculate the targets position in the image: target_pixel_pos = image.wcs.all_world2pix([[target['ra'], target['decl']]], 0)[0] + # Clean out the references: + hsize = 10 + x = references['pixel_column'] + y = references['pixel_row'] + refs_coord = coords.SkyCoord(ra=references['ra'], dec=references['decl'], unit='deg', frame='icrs') + references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) + & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) + & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] + # & (references[ref_filter] < ref_mag_limit) + #============================================================================================== # BARYCENTRIC CORRECTION OF TIME #============================================================================================== @@ -257,51 +271,19 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # Loop through argument sets for DAOStarFind: for kwargs in daofind_args: - # Run DAOStarFind with the given arguments: daofind_tbl = DAOStarFinder(**kwargs).find_stars(image.subclean, mask=image.mask) - def match_catalog_to_image_with_wcs(wcs): - - references = extract_references(catalog, wcs, ref_filter, hsize, ref_target_dist_limit) - - xy_stars_grid = np.tile(daofind_tbl['xcentroid'], (len(references), 1)), np.tile(daofind_tbl['ycentroid'], (len(references), 1)) - xy_references = np.array([references['pixel_column'], references['pixel_row']]).reshape([2, len(references), 1]) - distances = np.sqrt(np.power(xy_stars_grid - xy_references, 2).sum(axis=0)) - idx_refs, idx_stars = np.any(distances <= fwhm/4, axis=1) - - return references, (idx_refs, idx_stars) - - # indices of matching reference stars and image stars - idx_refs, idx_stars = match_catalog_to_image_with_wcs(image.wcs)[1] - - while True: - - wcs_new = WCS.from_astropy_wcs(image.wcs) - wcs_new.adjust_with_points( - xy = list(zip(daofind_tbl['xcentroid'][idx_stars], daofind_tbl['ycentroid'][idx_stars])), - rd = list(zip(references['ra'][idx_refs], references['decl'][idx_refs])) - ) - wcs_new = wcs_new.astropy_wcs - - references_new, (idx_refs_new, idx_stars_new) = match_catalog_to_image_with_wcs(wcs_new) - - # break out if wcs_new did not locate more stars - if len(idx_refs_new) <= len(idx_refs): - break - - references, image.wcs = references_new, wcs_new - idx_refs, idx_stars = idx_refs_new, idx_stars_new - - indx_good = idx_refs - - logger.debug("Number of references after cleaning: %d", len(indx_good)) - - if len(indx_good) >= min_references: + # Match the found stars with the catalog references: + indx_good = np.zeros(len(references), dtype='bool') + for k, ref in enumerate(references): + dist = np.sqrt( (daofind_tbl['xcentroid'] - ref['pixel_column'])**2 + (daofind_tbl['ycentroid'] - ref['pixel_row'])**2 ) + if np.any(dist <= fwhm/4): # Cutoff set somewhat arbitrary + indx_good[k] = True + logger.debug("Number of references after cleaning: %d", np.sum(indx_good)) + if np.sum(indx_good) >= min_references: references = references[indx_good] - target_pixel_pos = image.wcs.all_world2pix([[target['ra'], target['decl']]], 0)[0] - break logger.debug("Number of references after cleaning: %d", len(references)) @@ -682,24 +664,3 @@ def match_catalog_to_image_with_wcs(wcs): logger.info("Photometry took: %f seconds", toc-tic) return photometry_output - -#-------------------------------------------------------------------------------------------------- - -def extract_references(catalog, wcs, ref_filter='g_mag', hsize=10, ref_target_dist_limit=10*u.arcsec): - - references, target = catalog['references'], catalog['target'][0] - references.sort(ref_filter) - - references['pixel_column'], references['pixel_row'] = x, y = \ - list(zip(*wcs.all_world2pix(list(zip(references['ra'], references['decl'])), 0))) - - targ_coord = coords.SkyCoord(ra=target['ra'], dec=target['decl'], unit='deg', frame='icrs') - refs_coord = coords.SkyCoord(ra=references['ra'], dec=references['decl'], unit='deg', frame='icrs') - references = references[ - (targ_coord.separation(refs_coord) > ref_target_dist_limit) & - (x > hsize) & (x < (image.shape[1] - 1 - hsize)) & - (y > hsize) & (y < (image.shape[0] - 1 - hsize)) - ] - # & (references[ref_filter] < ref_mag_limit) - - return references diff --git a/flows/wcs.py b/flows/wcs.py deleted file mode 100644 index e835cab..0000000 --- a/flows/wcs.py +++ /dev/null @@ -1,198 +0,0 @@ -from copy import deepcopy - -import numpy as np -import astropy.wcs - -from scipy.optimize import minimize -from scipy.spatial.transform import Rotation - -class WCS () : - '''Manipulate WCS solution. - - Initialize - ---------- - wcs = WCS(x, y, ra, dec, scale, mirror, angle) - wcs = WCS.from_matrix(x, y, ra, dec, matrix) - wcs = WCS.from_points(list(zip(x, y)), list(zip(ra, dec))) - wcs = WCS.from_astropy_wcs(astropy.wcs.WCS()) - - ra, dec and angle should be in degrees - scale should be in arcsec/pixel - matrix should be the PC or CD matrix - - Examples - -------- - Adjust x, y offset: - wcs.x += delta_x - wcs.y += delta_y - - Get scale and angle: - print(wcs.scale, wcs.angle) - - Change an astropy.wcs.WCS (wcs) angle - wcs = WCS(wcs)(angle=new_angle).astropy_wcs - - Adjust solution with points - wcs.adjust_with_points(list(zip(x, y)), list(zip(ra, dec))) - ''' - - def __init__(self, x, y, ra, dec, scale, mirror, angle): - - self.x, self.y = x, y - self.ra, self.dec = ra, dec - self.scale = scale - self.mirror = mirror - self.angle = angle - - @classmethod - def from_matrix(cls, x, y, ra, dec, matrix): - - assert np.shape(matrix) == (2, 2), \ - 'Matrix must be 2x2' - - scale, mirror, angle = cls._decompose_matrix(matrix) - - return cls(x, y, ra, dec, scale, mirror, angle) - - @classmethod - def from_points(cls, xy, rd): - - assert np.shape(xy) == np.shape(rd) == (len(xy), 2) and len(xy) > 2, \ - 'Arguments must be lists of at least 3 sets of coordinates' - - xy, rd = np.array(xy), np.array(rd) - - x, y, ra, dec, matrix = cls._solve_from_points(xy, rd) - scale, mirror, angle = cls._decompose_matrix(matrix) - - return cls(x, y, ra, dec, scale, mirror, angle) - - @classmethod - def from_astropy_wcs(cls, astropy_wcs): - - assert type(astropy_wcs) is astropy.wcs.WCS, \ - 'Must be astropy.wcs.WCS' - - (x, y), (ra, dec) = astropy_wcs.wcs.crpix, astropy_wcs.wcs.crval - scale, mirror, angle = cls._decompose_matrix(astropy_wcs.pixel_scale_matrix) - - return cls(x, y, ra, dec, scale, mirror, angle) - - def adjust_with_points(self, xy, rd): - - assert np.shape(xy) == np.shape(rd) == (len(xy), 2), \ - 'Arguments must be lists of sets of coordinates' - - xy, rd = np.array(xy), np.array(rd) - - self.x, self.y = xy.mean(axis=0) - self.ra, self.dec = rd.mean(axis=0) - - A, b = xy - xy.mean(axis=0), rd - rd.mean(axis=0) - b[:,0] *= np.cos(np.deg2rad(rd[:,1])) - - if len(xy) == 2: - - M = np.diag([[-1, 1][self.mirror], 1]) - R = lambda t: np.array([[np.cos(t), -np.sin(t)], [np.sin(t), np.cos(t)]]) - - chi2 = lambda x: np.power(A.dot(x[1]/60/60*R(x[0]).dot(M).T) - b, 2).sum() - self.angle, self.scale = minimize(chi2, [self.angle, self.scale]).x - - elif len(xy) > 2: - - matrix = np.linalg.lstsq(A, b, rcond=None)[0].T - self.scale, self.mirror, self.angle = self._decompose_matrix(matrix) - - @property - def matrix(self): - - scale = self.scale / 60 / 60 - mirror = np.diag([[-1, 1][self.mirror], 1]) - angle = np.deg2rad(self.angle) - - matrix = np.array([ - [np.cos(angle), -np.sin(angle)], - [np.sin(angle), np.cos(angle)] - ]) - - return scale * matrix @ mirror - - @property - def astropy_wcs(self): - - wcs = astropy.wcs.WCS() - wcs.wcs.crpix = self.x, self.y - wcs.wcs.crval = self.ra, self.dec - wcs.wcs.pc = self.matrix - - return wcs - - @staticmethod - def _solve_from_points(xy, rd): - - (x, y), (ra, dec) = xy.mean(axis=0), rd.mean(axis=0) - - A, b = xy - xy.mean(axis=0), rd - rd.mean(axis=0) - b[:,0] *= np.cos(np.deg2rad(rd[:,1])) - - matrix = np.linalg.lstsq(A, b, rcond=None)[0].T - - return x, y, ra, dec, matrix - - @staticmethod - def _decompose_matrix(matrix): - - scale = np.sqrt(np.power(matrix, 2).sum() / 2) * 60 * 60 - - if np.argmax(np.power(matrix[0], 2)): - mirror = True if np.sign(matrix[0,1]) != np.sign(matrix[1,0]) else False - else: - mirror = True if np.sign(matrix[0,0]) == np.sign(matrix[1,1]) else False - - matrix = matrix if mirror else matrix.dot(np.diag([-1, 1])) - - matrix3d = np.eye(3); matrix3d[:2,:2] = matrix / (scale / 60 / 60) - angle = Rotation.from_matrix(matrix3d).as_euler('xyz', degrees=True)[2] - - return scale, mirror, angle - - def __setattr__(self, name, value): - - if name == 'ra': - - assert 0 <= value < 360, '0 <= R.A. < 360' - - elif name == 'dec': - - assert -180 <= value <= 180, '-180 <= Dec. <= 180' - - elif name == 'scale': - - assert value > 0, 'Scale > 0' - - elif name == 'mirror': - - assert type(value) is bool, 'Mirror = True | False' - - elif name == 'angle': - - assert -180 < value <= 180, '-180 < Angle <= 180' - - super().__setattr__(name, value) - - def __call__(self, **kwargs): - - keys = ('x', 'y', 'ra', 'dec', 'scale', 'mirror', 'angle') - - if not all(k in keys for k in kwargs): - - raise Exception('unknown argument(s)') - - obj = deepcopy(self) - - for k, v in kwargs.items(): - - obj.__setattr__(k, v) - - return obj From fb4a0190ea1cb731f33b0fdeb9ab9d0e17957f5a Mon Sep 17 00:00:00 2001 From: Simon Holmbo Date: Mon, 25 Jan 2021 16:23:35 +0100 Subject: [PATCH 07/43] CoordinateMatch for WCS --- flows/coordinatematch/__init__.py | 1 + flows/coordinatematch/coordinatematch.py | 323 +++++++++++++++++++++++ flows/coordinatematch/wcs.py | 198 ++++++++++++++ flows/photometry.py | 21 ++ requirements.txt | 3 +- 5 files changed, 545 insertions(+), 1 deletion(-) create mode 100644 flows/coordinatematch/__init__.py create mode 100644 flows/coordinatematch/coordinatematch.py create mode 100644 flows/coordinatematch/wcs.py diff --git a/flows/coordinatematch/__init__.py b/flows/coordinatematch/__init__.py new file mode 100644 index 0000000..8326efe --- /dev/null +++ b/flows/coordinatematch/__init__.py @@ -0,0 +1 @@ +from .coordinatematch import CoordinateMatch diff --git a/flows/coordinatematch/coordinatematch.py b/flows/coordinatematch/coordinatematch.py new file mode 100644 index 0000000..d7c1a9f --- /dev/null +++ b/flows/coordinatematch/coordinatematch.py @@ -0,0 +1,323 @@ +import time + +from itertools import count, islice, chain, product, zip_longest + +import numpy as np + +from astropy.coordinates.angle_utilities import angular_separation +from scipy.spatial import cKDTree as KDTree +from networkx import Graph, connected_components + +from .wcs import WCS + +class CoordinateMatch () : + + def __init__(self, xy, rd, xy_mag=None, rd_mag=None, + n_triangle_packages = 10, + triangle_package_size = 10000, + maximum_angle_distance = 0.001 + ): + + self.xy, self.rd = np.array(xy), np.array(rd) + + self._xy = xy - np.mean(xy, axis=0) + self._rd = rd - np.mean(rd, axis=0) + self._rd[:,0] *= np.cos(np.deg2rad(self.rd[:,1])) + + self.i_xy = np.argsort(xy_mag) if not xy_mag is None else np.arange(len(xy)) + self.i_rd = np.argsort(rd_mag) if not rd_mag is None else np.arange(len(rd)) + + self.n_triangle_packages = n_triangle_packages + self.triangle_package_size = triangle_package_size + + self.maximum_angle_distance = maximum_angle_distance + + self.triangle_package_generator = self._sorted_triangle_packages() + + self.i_xy_triangles = list() + self.i_rd_triangles = list() + self.parameters = None + self.neighbours = Graph() + + self.normalizations = type('Normalizations', (object,), dict( + ra = 0.002, dec = 0.002, scale = 0.002, angle = 0.002 + )) + + self.bounds = type('Bounds', (object,), dict( + xy = self.xy.mean(axis=0), rd = None, radius = None, + scale = None, angle = None + )) + + def set_normalizations(self, ra=None, dec=None, scale=None, angle=None): + + if not self.parameters is None: + + raise Exception('can\'t change normalization after matching is started') + + assert ra == None or 0 < ra + assert dec == None or 0 < dec + assert scale == None or 0 < scale + assert angle == None or 0 < angle + + self.normalizations.ra = ra if not ra is None else self.normalizations.ra + self.normalizations.dec = dec if not dec is None else self.normalizations.dec + self.normalizations.scale = scale if not scale is None else self.normalizations.scale + self.normalizations.angle = angle if not ra is None else self.normalizations.angle + + def set_bounds(self, x=None, y=None, ra=None, dec=None, radius=None, scale=None, angle=None): + + if not self.parameters is None: + + raise Exception('can\'t change bounds after matching is started') + + if [x, y, ra, dec, radius].count(None) == 5: + + assert 0 <= ra < 360 + assert -180 <= dec <= 180 + assert 0 < radius + + self.bounds.xy = x, y + self.bounds.rd = ra, dec + self.bounds.radius = radius + + elif [x, y, ra, dec, radius].count(None) > 0: + + raise Exception('x, y, ra, dec and radius must all be specified') + + assert scale == None or 0 < scale[0] < scale[1] + assert angle == None or -np.pi <= angle[0] < angle[1] <= np.pi + + self.bounds.scale = scale if not scale is None else self.bounds.scale + self.bounds.angle = angle if not angle is None else self.bounds.angle + + def _sorted_triangles(self, pool): + + for i, c in enumerate(pool): + for i, b in enumerate(pool[:i]): + for a in pool[:i]: + + yield a, b, c + + def _sorted_product_pairs(self, p, q): + + i_p = np.argsort(np.arange(len(p))) + i_q = np.argsort(np.arange(len(q))) + + for _i_p, _i_q in sorted(product(i_p, i_q), key=lambda idxs: sum(idxs)): + + yield p[_i_p], q[_i_q] + + def _sorted_triangle_packages(self): + + i_xy_triangle_generator = self._sorted_triangles(self.i_xy) + i_rd_triangle_generator = self._sorted_triangles(self.i_rd) + + i_xy_triangle_slice_generator = ( + tuple(islice(i_xy_triangle_generator, self.triangle_package_size)) + for _ in count() + ) + i_rd_triangle_slice_generator = ( + list(islice(i_rd_triangle_generator, self.triangle_package_size)) + for _ in count() + ) + + for n in count(step=self.n_triangle_packages): + + i_xy_triangle_slice = tuple(filter(None, + islice(i_xy_triangle_slice_generator, self.n_triangle_packages) + )) + i_rd_triangle_slice = tuple(filter(None, + islice(i_rd_triangle_slice_generator, self.n_triangle_packages) + )) + + if not len(i_xy_triangle_slice) and not len(i_rd_triangle_slice): + return + + i_xy_triangle_generator2 = self._sorted_triangles(self.i_xy) + i_rd_triangle_generator2 = self._sorted_triangles(self.i_rd) + + i_xy_triangle_cum = filter(None, ( + tuple(islice(i_xy_triangle_generator2, self.triangle_package_size)) + for _ in range(n) + )) + i_rd_triangle_cum = filter(None, ( + tuple(islice(i_rd_triangle_generator2, self.triangle_package_size)) + for _ in range(n) + )) + + for i_xy_triangles, i_rd_triangles in chain( + filter(None, chain(*zip_longest( # alternating chain + product(i_xy_triangle_slice, i_rd_triangle_cum), + product(i_xy_triangle_cum, i_rd_triangle_slice) + ))), + self._sorted_product_pairs(i_xy_triangle_slice, i_rd_triangle_slice) + ): + yield np.array(i_xy_triangles), np.array(i_rd_triangles) + + def _get_triangle_angles(self, triangles): + + sidelengths = np.sqrt(np.power(triangles[:,(1,0,0)] - triangles[:,(2,2,1)], 2).sum(axis=2)) + + # law of cosines + angles = np.power(sidelengths[:,((1,2),(0,2),(0,1))], 2).sum(axis=2) + angles -= np.power(sidelengths[:,(0,1,2)], 2) + angles /= 2 * sidelengths[:,((1,2),(0,2),(0,1))].prod(axis=2) + + return np.arccos(angles) + + def _solve_for_matrices(self, xy_triangles, rd_triangles): + + n = len(xy_triangles) + + A = xy_triangles - np.mean(xy_triangles, axis=1).reshape(n,1,2) + b = rd_triangles - np.mean(rd_triangles, axis=1).reshape(n,1,2) + + matrices = [np.linalg.lstsq(Ai, bi, rcond=None)[0].T for Ai, bi in zip(A, b)] + + return np.array(matrices) + + def _extract_parameters(self, xy_triangles, rd_triangles, matrices): + + parameters = [] + + for xy_com, rd_com, matrix in zip( # com -> center-of-mass + xy_triangles.mean(axis=1), + rd_triangles.mean(axis=1), + matrices + ): + + cos_dec = np.cos(np.deg2rad(rd_com[1])) + coordinates = (self.bounds.xy - xy_com).dot(matrix) + coordinates = coordinates / np.array([cos_dec, 1]) + rd_com + + wcs = WCS.from_matrix(*xy_com, *rd_com, matrix) + + parameters.append( + (*coordinates, np.log(wcs.scale), np.deg2rad(wcs.angle)) + ) + + return parameters + + def _get_bounds_mask(self, parameters): + + i = np.ones(len(parameters), dtype=bool) + parameters = np.array(parameters) + + if not self.bounds.radius is None: + + i *= angular_separation( + *np.deg2rad(self.bounds.rd), + *zip(*np.deg2rad(parameters[:,(0,1)])) + ) <= np.deg2rad(self.bounds.radius) + + if not self.bounds.scale is None: + + i *= self.bounds.scale[0] <= parameters[:,2] + i *= parameters[:,2] <= self.bounds.scale[1] + + if not self.bounds.angle is None: + + i *= self.bounds.angle[0] <= parameters[:,3] + i *= parameters[:,3] <= self.bounds.angle[1] + + return i + + def __call__(self, + minimum_matches = 4, + distance_factor = 1, + timeout = 60 + ): + + self.parameters = list() if self.parameters is None else self.parameters + + t0 = time.time() + + while time.time() - t0 < timeout: + + # get triangles and derive angles + + i_xy_triangles, i_rd_triangles = next(self.triangle_package_generator) + + xy_angles = self._get_triangle_angles(self._xy[i_xy_triangles]) + rd_angles = self._get_triangle_angles(self._rd[i_rd_triangles]) + + # sort triangle vertices based on angles + + i = np.argsort(xy_angles, axis=1) + i_xy_triangles = np.take_along_axis(i_xy_triangles, i, axis=1) + xy_angles = np.take_along_axis(xy_angles, i, axis=1) + + i = np.argsort(rd_angles, axis=1) + i_rd_triangles = np.take_along_axis(i_rd_triangles, i, axis=1) + rd_angles = np.take_along_axis(rd_angles, i, axis=1) + + # match triangles + + matches = KDTree(xy_angles).query_ball_tree(KDTree(rd_angles), r=self.maximum_angle_distance) + matches = np.array([(_i_xy, _i_rd) for _i_xy, _li_rd in enumerate(matches) for _i_rd in _li_rd]) + + if not len(matches): + continue + + i_xy_triangles = list(i_xy_triangles[matches[:,0]]) + i_rd_triangles = list(i_rd_triangles[matches[:,1]]) + + # get parameters of wcs solutions + + matrices = self._solve_for_matrices( + self._xy[i_xy_triangles], + self._rd[i_rd_triangles] + ) + + parameters = self._extract_parameters( + self.xy[i_xy_triangles], + self.rd[i_rd_triangles], + matrices + ) + + # apply bounds if any + + if any([self.bounds.radius, self.bounds.scale, self.bounds.angle]): + + mask = self._get_bounds_mask(parameters) + + i_xy_triangles = np.array(i_xy_triangles)[mask].tolist() + i_rd_triangles = np.array(i_rd_triangles)[mask].tolist() + parameters = np.array(parameters)[mask].tolist() + + # normalize parameters + + normalization = [getattr(self.normalizations, v) for v in ('ra', 'dec', 'scale', 'angle')] + normalization[0] *= np.cos(np.deg2rad(self.rd[:,1].mean(axis=0))) + parameters = list(parameters / np.array(normalization)) + + # match parameters + + neighbours = KDTree(parameters).query_ball_tree(KDTree(self.parameters + parameters), r=distance_factor) + neighbours = np.array([(i, j) for i, lj in enumerate(neighbours, len(self.parameters)) for j in lj]) + neighbours = list(neighbours[(np.diff(neighbours, axis=1) < 0).flatten()]) + + if not len(neighbours): + continue + + self.i_xy_triangles += i_xy_triangles + self.i_rd_triangles += i_rd_triangles + self.parameters += parameters + self.neighbours.add_edges_from(neighbours) + + # get largest neighborhood + + l = list(max(connected_components(self.neighbours), key=len)) + i = np.unique(np.array(self.i_xy_triangles)[l].flatten(), return_index=True)[1] + + if len(i) >= minimum_matches: + break + + else: + + raise TimeoutError + + i_xy = np.array(self.i_xy_triangles)[l].flatten()[i] + i_rd = np.array(self.i_rd_triangles)[l].flatten()[i] + + return list(zip(i_xy, i_rd)) diff --git a/flows/coordinatematch/wcs.py b/flows/coordinatematch/wcs.py new file mode 100644 index 0000000..e835cab --- /dev/null +++ b/flows/coordinatematch/wcs.py @@ -0,0 +1,198 @@ +from copy import deepcopy + +import numpy as np +import astropy.wcs + +from scipy.optimize import minimize +from scipy.spatial.transform import Rotation + +class WCS () : + '''Manipulate WCS solution. + + Initialize + ---------- + wcs = WCS(x, y, ra, dec, scale, mirror, angle) + wcs = WCS.from_matrix(x, y, ra, dec, matrix) + wcs = WCS.from_points(list(zip(x, y)), list(zip(ra, dec))) + wcs = WCS.from_astropy_wcs(astropy.wcs.WCS()) + + ra, dec and angle should be in degrees + scale should be in arcsec/pixel + matrix should be the PC or CD matrix + + Examples + -------- + Adjust x, y offset: + wcs.x += delta_x + wcs.y += delta_y + + Get scale and angle: + print(wcs.scale, wcs.angle) + + Change an astropy.wcs.WCS (wcs) angle + wcs = WCS(wcs)(angle=new_angle).astropy_wcs + + Adjust solution with points + wcs.adjust_with_points(list(zip(x, y)), list(zip(ra, dec))) + ''' + + def __init__(self, x, y, ra, dec, scale, mirror, angle): + + self.x, self.y = x, y + self.ra, self.dec = ra, dec + self.scale = scale + self.mirror = mirror + self.angle = angle + + @classmethod + def from_matrix(cls, x, y, ra, dec, matrix): + + assert np.shape(matrix) == (2, 2), \ + 'Matrix must be 2x2' + + scale, mirror, angle = cls._decompose_matrix(matrix) + + return cls(x, y, ra, dec, scale, mirror, angle) + + @classmethod + def from_points(cls, xy, rd): + + assert np.shape(xy) == np.shape(rd) == (len(xy), 2) and len(xy) > 2, \ + 'Arguments must be lists of at least 3 sets of coordinates' + + xy, rd = np.array(xy), np.array(rd) + + x, y, ra, dec, matrix = cls._solve_from_points(xy, rd) + scale, mirror, angle = cls._decompose_matrix(matrix) + + return cls(x, y, ra, dec, scale, mirror, angle) + + @classmethod + def from_astropy_wcs(cls, astropy_wcs): + + assert type(astropy_wcs) is astropy.wcs.WCS, \ + 'Must be astropy.wcs.WCS' + + (x, y), (ra, dec) = astropy_wcs.wcs.crpix, astropy_wcs.wcs.crval + scale, mirror, angle = cls._decompose_matrix(astropy_wcs.pixel_scale_matrix) + + return cls(x, y, ra, dec, scale, mirror, angle) + + def adjust_with_points(self, xy, rd): + + assert np.shape(xy) == np.shape(rd) == (len(xy), 2), \ + 'Arguments must be lists of sets of coordinates' + + xy, rd = np.array(xy), np.array(rd) + + self.x, self.y = xy.mean(axis=0) + self.ra, self.dec = rd.mean(axis=0) + + A, b = xy - xy.mean(axis=0), rd - rd.mean(axis=0) + b[:,0] *= np.cos(np.deg2rad(rd[:,1])) + + if len(xy) == 2: + + M = np.diag([[-1, 1][self.mirror], 1]) + R = lambda t: np.array([[np.cos(t), -np.sin(t)], [np.sin(t), np.cos(t)]]) + + chi2 = lambda x: np.power(A.dot(x[1]/60/60*R(x[0]).dot(M).T) - b, 2).sum() + self.angle, self.scale = minimize(chi2, [self.angle, self.scale]).x + + elif len(xy) > 2: + + matrix = np.linalg.lstsq(A, b, rcond=None)[0].T + self.scale, self.mirror, self.angle = self._decompose_matrix(matrix) + + @property + def matrix(self): + + scale = self.scale / 60 / 60 + mirror = np.diag([[-1, 1][self.mirror], 1]) + angle = np.deg2rad(self.angle) + + matrix = np.array([ + [np.cos(angle), -np.sin(angle)], + [np.sin(angle), np.cos(angle)] + ]) + + return scale * matrix @ mirror + + @property + def astropy_wcs(self): + + wcs = astropy.wcs.WCS() + wcs.wcs.crpix = self.x, self.y + wcs.wcs.crval = self.ra, self.dec + wcs.wcs.pc = self.matrix + + return wcs + + @staticmethod + def _solve_from_points(xy, rd): + + (x, y), (ra, dec) = xy.mean(axis=0), rd.mean(axis=0) + + A, b = xy - xy.mean(axis=0), rd - rd.mean(axis=0) + b[:,0] *= np.cos(np.deg2rad(rd[:,1])) + + matrix = np.linalg.lstsq(A, b, rcond=None)[0].T + + return x, y, ra, dec, matrix + + @staticmethod + def _decompose_matrix(matrix): + + scale = np.sqrt(np.power(matrix, 2).sum() / 2) * 60 * 60 + + if np.argmax(np.power(matrix[0], 2)): + mirror = True if np.sign(matrix[0,1]) != np.sign(matrix[1,0]) else False + else: + mirror = True if np.sign(matrix[0,0]) == np.sign(matrix[1,1]) else False + + matrix = matrix if mirror else matrix.dot(np.diag([-1, 1])) + + matrix3d = np.eye(3); matrix3d[:2,:2] = matrix / (scale / 60 / 60) + angle = Rotation.from_matrix(matrix3d).as_euler('xyz', degrees=True)[2] + + return scale, mirror, angle + + def __setattr__(self, name, value): + + if name == 'ra': + + assert 0 <= value < 360, '0 <= R.A. < 360' + + elif name == 'dec': + + assert -180 <= value <= 180, '-180 <= Dec. <= 180' + + elif name == 'scale': + + assert value > 0, 'Scale > 0' + + elif name == 'mirror': + + assert type(value) is bool, 'Mirror = True | False' + + elif name == 'angle': + + assert -180 < value <= 180, '-180 < Angle <= 180' + + super().__setattr__(name, value) + + def __call__(self, **kwargs): + + keys = ('x', 'y', 'ra', 'dec', 'scale', 'mirror', 'angle') + + if not all(k in keys for k in kwargs): + + raise Exception('unknown argument(s)') + + obj = deepcopy(self) + + for k, v in kwargs.items(): + + obj.__setattr__(k, v) + + return obj diff --git a/flows/photometry.py b/flows/photometry.py index 42eab31..f8da5c9 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -43,6 +43,7 @@ from .zeropoint import bootstrap_outlier, sigma_from_Chauvenet from .wcs import force_reject_g2d, mkposxy, clean_with_rsq_and_get_fwhm, \ try_astroalign, kdtree, get_new_wcs, get_clean_references +from .coordinatematch import CoordinateMatch __version__ = get_version(pep440=False) @@ -254,6 +255,26 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): fig.savefig(os.path.join(output_folder, 'positions_g2d.png'), bbox_inches='tight') plt.close(fig) +################################################################################ + + _xy = list(zip(objects['x'], objects['y'])) + _rd = list(zip(references['ra_obs'].deg, references['decl_obs'].deg)) + _xy_mag = -2.5*np.log10(objects['flux']) + _rd_mag = references[ref_filter] + + try: + cm = CoordinateMatch(_xy, _rd, _xy_mag, _rd_mag) + _i_xy, _i_rd = list(zip(*cm())) + except (TimeoutError, StopIteration): + new_wcs = None + else: + new_wcs = fit_wcs_from_points( + np.array(list(zip(*_xy[np.array(i_xy)]))), + SkyCoord(*map(list, zip(*_rd[np.array(i_rd)])), unit='deg') + ) + +################################################################################ + # Sort by brightness clean_references.sort('g_mag') # Sorted by g mag _at = Table({'xy': masked_sep_xy, 'flux': objects['flux'][sep_mask]}) diff --git a/requirements.txt b/requirements.txt index d4ebc04..5eaf4aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,5 @@ beautifulsoup4 git+https://github.com/obscode/imagematch.git@photutils#egg=imagematch pandas sep -astroalign > 2.3 \ No newline at end of file +astroalign > 2.3 +networkx From 71c06403c51ca08a422b2d5b371ee546ee8fa335 Mon Sep 17 00:00:00 2001 From: Simon Holmbo Date: Sat, 30 Jan 2021 16:49:18 +0100 Subject: [PATCH 08/43] Coordinate Match --- flows/coordinatematch/coordinatematch.py | 42 +++++++++++------- flows/photometry.py | 56 +++++++++++++++--------- 2 files changed, 63 insertions(+), 35 deletions(-) diff --git a/flows/coordinatematch/coordinatematch.py b/flows/coordinatematch/coordinatematch.py index d7c1a9f..cb52fc4 100644 --- a/flows/coordinatematch/coordinatematch.py +++ b/flows/coordinatematch/coordinatematch.py @@ -12,10 +12,11 @@ class CoordinateMatch () : - def __init__(self, xy, rd, xy_mag=None, rd_mag=None, + def __init__(self, xy, rd, xy_order=None, rd_order=None, n_triangle_packages = 10, triangle_package_size = 10000, - maximum_angle_distance = 0.001 + maximum_angle_distance = 0.001, + distance_factor = 1 ): self.xy, self.rd = np.array(xy), np.array(rd) @@ -24,13 +25,14 @@ def __init__(self, xy, rd, xy_mag=None, rd_mag=None, self._rd = rd - np.mean(rd, axis=0) self._rd[:,0] *= np.cos(np.deg2rad(self.rd[:,1])) - self.i_xy = np.argsort(xy_mag) if not xy_mag is None else np.arange(len(xy)) - self.i_rd = np.argsort(rd_mag) if not rd_mag is None else np.arange(len(rd)) + self.i_xy = xy_order if not xy_order is None else np.arange(len(xy)) + self.i_rd = rd_order if not rd_order is None else np.arange(len(rd)) self.n_triangle_packages = n_triangle_packages self.triangle_package_size = triangle_package_size self.maximum_angle_distance = maximum_angle_distance + self.distance_factor = distance_factor self.triangle_package_generator = self._sorted_triangle_packages() @@ -40,7 +42,7 @@ def __init__(self, xy, rd, xy_mag=None, rd_mag=None, self.neighbours = Graph() self.normalizations = type('Normalizations', (object,), dict( - ra = 0.002, dec = 0.002, scale = 0.002, angle = 0.002 + ra = 0.0001, dec = 0.0001, scale = 0.002, angle = 0.002 )) self.bounds = type('Bounds', (object,), dict( @@ -224,7 +226,7 @@ def _get_bounds_mask(self, parameters): def __call__(self, minimum_matches = 4, - distance_factor = 1, + ratio_superiority = 1, timeout = 60 ): @@ -265,13 +267,13 @@ def __call__(self, # get parameters of wcs solutions matrices = self._solve_for_matrices( - self._xy[i_xy_triangles], - self._rd[i_rd_triangles] + self._xy[np.array(i_xy_triangles)], + self._rd[np.array(i_rd_triangles)] ) parameters = self._extract_parameters( - self.xy[i_xy_triangles], - self.rd[i_rd_triangles], + self.xy[np.array(i_xy_triangles)], + self.rd[np.array(i_rd_triangles)], matrices ) @@ -293,7 +295,7 @@ def __call__(self, # match parameters - neighbours = KDTree(parameters).query_ball_tree(KDTree(self.parameters + parameters), r=distance_factor) + neighbours = KDTree(parameters).query_ball_tree(KDTree(self.parameters + parameters), r=self.distance_factor) neighbours = np.array([(i, j) for i, lj in enumerate(neighbours, len(self.parameters)) for j in lj]) neighbours = list(neighbours[(np.diff(neighbours, axis=1) < 0).flatten()]) @@ -307,8 +309,18 @@ def __call__(self, # get largest neighborhood - l = list(max(connected_components(self.neighbours), key=len)) - i = np.unique(np.array(self.i_xy_triangles)[l].flatten(), return_index=True)[1] + communities = list(connected_components(self.neighbours)) + c1 = np.array(list(max(communities, key=len))) + i = np.unique(np.array(self.i_xy_triangles)[c1].flatten(), return_index=True)[1] + + if ratio_superiority > 1 and len(communities) > 1: + + communities.remove(set(c1)) + c2 = np.array(list(max(communities, key=len))) + _i = np.unique(np.array(self.i_xy_triangles)[c2].flatten()) + + if len(i) / len(_i) < ratio_superiority: + continue if len(i) >= minimum_matches: break @@ -317,7 +329,7 @@ def __call__(self, raise TimeoutError - i_xy = np.array(self.i_xy_triangles)[l].flatten()[i] - i_rd = np.array(self.i_rd_triangles)[l].flatten()[i] + i_xy = np.array(self.i_xy_triangles)[c1].flatten()[i] + i_rd = np.array(self.i_rd_triangles)[c1].flatten()[i] return list(zip(i_xy, i_rd)) diff --git a/flows/photometry.py b/flows/photometry.py index 9f40c8b..b85aa7a 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -201,6 +201,42 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): refs_coord = refs_coord.apply_space_motion(image.obstime) + # Solve for new WCS + cm = CoordinateMatch( + xy = list(zip(objects['x'], objects['y'])), + rd = list(zip(refs_coord.ra.deg, refs_coord.dec.deg)), + xy_order = np.argsort(-2.5*np.log10(objects['flux'])), + rd_order = np.argsort(target_coord.separation(refs_coord)), + maximum_angle_distance = 0.002, + ) + + try: + i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=10))) + except (TimeoutError, StopIteration): + logging.warning('No new WCS solution found') + else: + image.wcs = fit_wcs_from_points( + np.array(list(zip(*cm.xy[i_xy]))), + coords.SkyCoord(*map(list, zip(*cm.rd[i_rd])), unit='deg') + ) + +# XXX # TESTING ################################################################ +# import matplotlib +# _backend = matplotlib.get_backend() +# matplotlib.pyplot.switch_backend('TkAgg') +# matplotlib.pyplot.subplot(projection=image.wcs) +# matplotlib.pyplot.imshow(image.subclean, origin='lower', cmap='gray_r', clim=np.nanquantile(image.subclean[image.subclean>0], (.01, .99))) +# matplotlib.pyplot.scatter(*zip(*image.wcs.wcs_world2pix(cm.rd, 0)), c=[], edgecolor='C0', s=200, label='References') +# for i, ((_x, _y), (_r, _d)) in enumerate(zip(cm.xy[i_xy], image.wcs.wcs_world2pix(cm.rd[i_rd], 0))): +# matplotlib.pyplot.plot([_x, _r], [_y, _d], color='C1', marker='o', ms=9, mfc='none', mec='C1', label='Solution' if not i else None) +# matplotlib.pyplot.scatter(*zip(*image.wcs.wcs_world2pix([(target['ra'], target['decl'])], 0)), c=[], edgecolor='C2', s=100, label='Target') +# matplotlib.pyplot.xlim(0, image.subclean.shape[1]) +# matplotlib.pyplot.ylim(0, image.subclean.shape[0]) +# matplotlib.pyplot.legend() +# matplotlib.pyplot.show() +# matplotlib.pyplot.switch_backend(_backend) +# XXX # TESTING ################################################################ + # Calculate pixel-coordinates of references: row_col_coords = image.wcs.all_world2pix(np.array([[ref.ra.deg, ref.dec.deg] for ref in refs_coord]), 0) references['pixel_column'] = row_col_coords[:,0] @@ -257,26 +293,6 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): fig.savefig(os.path.join(output_folder, 'positions_g2d.png'), bbox_inches='tight') plt.close(fig) -################################################################################ - - _xy = list(zip(objects['x'], objects['y'])) - _rd = list(zip(references['ra_obs'].deg, references['decl_obs'].deg)) - _xy_mag = -2.5*np.log10(objects['flux']) - _rd_mag = references[ref_filter] - - try: - cm = CoordinateMatch(_xy, _rd, _xy_mag, _rd_mag) - _i_xy, _i_rd = list(zip(*cm())) - except (TimeoutError, StopIteration): - new_wcs = None - else: - new_wcs = fit_wcs_from_points( - np.array(list(zip(*_xy[np.array(i_xy)]))), - SkyCoord(*map(list, zip(*_rd[np.array(i_rd)])), unit='deg') - ) - -################################################################################ - # Sort by brightness clean_references.sort('g_mag') # Sorted by g mag _at = Table({'xy': masked_sep_xy, 'flux': objects['flux'][sep_mask]}) From eb9bc01a962a7be8038bf03942dc20eb8bb3d547 Mon Sep 17 00:00:00 2001 From: Simon Holmbo Date: Sun, 31 Jan 2021 21:07:18 +0100 Subject: [PATCH 09/43] Added docstrings --- flows/coordinatematch/coordinatematch.py | 61 ++++++++++++++++++++---- flows/coordinatematch/wcs.py | 14 +++++- 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/flows/coordinatematch/coordinatematch.py b/flows/coordinatematch/coordinatematch.py index cb52fc4..df5aef7 100644 --- a/flows/coordinatematch/coordinatematch.py +++ b/flows/coordinatematch/coordinatematch.py @@ -51,15 +51,23 @@ def __init__(self, xy, rd, xy_order=None, rd_order=None, )) def set_normalizations(self, ra=None, dec=None, scale=None, angle=None): + '''Set normalization factors in the (ra, dec, scale, angle) space. + + Defaults are: + ra = 0.0001 degrees + dec = 0.0001 degrees + scale = 0.002 log(arcsec/pixel) + angle = 0.002 radians + ''' if not self.parameters is None: raise Exception('can\'t change normalization after matching is started') - assert ra == None or 0 < ra - assert dec == None or 0 < dec - assert scale == None or 0 < scale - assert angle == None or 0 < angle + assert ra is None or 0 < ra + assert dec is None or 0 < dec + assert scale is None or 0 < scale + assert angle is None or 0 < angle self.normalizations.ra = ra if not ra is None else self.normalizations.ra self.normalizations.dec = dec if not dec is None else self.normalizations.dec @@ -67,6 +75,14 @@ def set_normalizations(self, ra=None, dec=None, scale=None, angle=None): self.normalizations.angle = angle if not ra is None else self.normalizations.angle def set_bounds(self, x=None, y=None, ra=None, dec=None, radius=None, scale=None, angle=None): + '''Set bounds for what are valid results. + + Set x, y, ra, dec and radius to specify that the x, y coordinates must be no + further that the radius [degrees] away from the ra, dec coordinates. + Set upper and lower bounds on the scale [log(arcsec/pixel)] and/or the angle + [radians] if those are known, possibly from previous observations with the + same system. + ''' if not self.parameters is None: @@ -78,7 +94,7 @@ def set_bounds(self, x=None, y=None, ra=None, dec=None, radius=None, scale=None, assert -180 <= dec <= 180 assert 0 < radius - self.bounds.xy = x, y + self.bounds.xy = x, y self.bounds.rd = ra, dec self.bounds.radius = radius @@ -86,8 +102,8 @@ def set_bounds(self, x=None, y=None, ra=None, dec=None, radius=None, scale=None, raise Exception('x, y, ra, dec and radius must all be specified') - assert scale == None or 0 < scale[0] < scale[1] - assert angle == None or -np.pi <= angle[0] < angle[1] <= np.pi + assert scale is None or 0 < scale[0] < scale[1] + assert angle is None or -np.pi <= angle[0] < angle[1] <= np.pi self.bounds.scale = scale if not scale is None else self.bounds.scale self.bounds.angle = angle if not angle is None else self.bounds.angle @@ -162,7 +178,7 @@ def _get_triangle_angles(self, triangles): # law of cosines angles = np.power(sidelengths[:,((1,2),(0,2),(0,1))], 2).sum(axis=2) - angles -= np.power(sidelengths[:,(0,1,2)], 2) + angles -= np.power(sidelengths[:,(0,1,2)], 2) angles /= 2 * sidelengths[:,((1,2),(0,2),(0,1))].prod(axis=2) return np.arccos(angles) @@ -229,6 +245,35 @@ def __call__(self, ratio_superiority = 1, timeout = 60 ): + '''Start the alogrithm. + + Can be run multiple times with different arguments to relax the + restrictions. + + Example + -------- + cm = CoordinateMatch(xy, rd) + + lkwargs = [{ + minimum_matches = 20, + ratio_superiority = 5, + timeout = 10 + },{ + timeout = 60 + } + + for i, kwargs in enumerate(lkwargs): + try: + i_xy, i_rd = cm(**kwargs) + except TimeoutError: + continue + except StopIteration: + print('Failed, no more stars.') + else: + print('Success with kwargs[%d].' % i) + else: + print('Failed, timeout.') + ''' self.parameters = list() if self.parameters is None else self.parameters diff --git a/flows/coordinatematch/wcs.py b/flows/coordinatematch/wcs.py index e835cab..ff23230 100644 --- a/flows/coordinatematch/wcs.py +++ b/flows/coordinatematch/wcs.py @@ -25,7 +25,7 @@ class WCS () : Adjust x, y offset: wcs.x += delta_x wcs.y += delta_y - + Get scale and angle: print(wcs.scale, wcs.angle) @@ -46,6 +46,7 @@ def __init__(self, x, y, ra, dec, scale, mirror, angle): @classmethod def from_matrix(cls, x, y, ra, dec, matrix): + '''Initiate the class with a matrix.''' assert np.shape(matrix) == (2, 2), \ 'Matrix must be 2x2' @@ -56,6 +57,7 @@ def from_matrix(cls, x, y, ra, dec, matrix): @classmethod def from_points(cls, xy, rd): + '''Initiate the class with at least pixel + sky coordinates.''' assert np.shape(xy) == np.shape(rd) == (len(xy), 2) and len(xy) > 2, \ 'Arguments must be lists of at least 3 sets of coordinates' @@ -69,6 +71,7 @@ def from_points(cls, xy, rd): @classmethod def from_astropy_wcs(cls, astropy_wcs): + '''Initiate the class with an astropy.wcs.WCS object.''' assert type(astropy_wcs) is astropy.wcs.WCS, \ 'Must be astropy.wcs.WCS' @@ -79,6 +82,12 @@ def from_astropy_wcs(cls, astropy_wcs): return cls(x, y, ra, dec, scale, mirror, angle) def adjust_with_points(self, xy, rd): + '''Adjust the WCS with pixel + sky coordinates. + + If one set is given the change will be a simple offset. + If two sets are given the offset, angle and scale will be derived. + And if more sets are given a completely new solution will be found. + ''' assert np.shape(xy) == np.shape(rd) == (len(xy), 2), \ 'Arguments must be lists of sets of coordinates' @@ -182,9 +191,10 @@ def __setattr__(self, name, value): super().__setattr__(name, value) def __call__(self, **kwargs): + '''Make a copy with, or a copy with changes.''' keys = ('x', 'y', 'ra', 'dec', 'scale', 'mirror', 'angle') - + if not all(k in keys for k in kwargs): raise Exception('unknown argument(s)') From 58187987bdd1b1e27f0c9960e733a59f6aa0ae25 Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 3 Feb 2021 10:58:56 +0100 Subject: [PATCH 10/43] Making the pipeline work with Simon's new changes --- VERSION | 2 +- flows/load_image.py | 41 +++++- flows/photometry.py | 310 ++++++++++++++++++++++---------------------- flows/wcs.py | 8 +- flows/ztf.py | 15 ++- run_download_ztf.py | 4 +- 6 files changed, 211 insertions(+), 169 deletions(-) diff --git a/VERSION b/VERSION index e699978..a98df34 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -master-v0.4.1 +master-v0.4.4 \ No newline at end of file diff --git a/flows/load_image.py b/flows/load_image.py index 98140e1..9ca1169 100644 --- a/flows/load_image.py +++ b/flows/load_image.py @@ -218,7 +218,7 @@ def load_image(FILENAME): 'i': 'ip', }.get(hdr['FILTER'], hdr['FILTER']) - elif telescope == 'DUP' and hdr.get('SITENAME') == 'LCO': + elif telescope == 'DUP' and hdr.get('SITENAME') == 'LCO' and instrument == 'Direct/SITe2K-1': image.site = api.get_site(14) # Hard-coded the siteid for Du Pont, Las Campanas Observatory image.obstime = Time(hdr['JD'], format='jd', scale='utc', location=image.site['EarthLocation']) image.photfilter = { @@ -228,6 +228,15 @@ def load_image(FILENAME): 'i': 'ip', }.get(hdr['FILTER'], hdr['FILTER']) + elif telescope == 'DUP' and instrument == 'RetroCam': + image.site = api.get_site(16) # Hard-coded the siteid for Du Pont, Las Campanas Observatory + image.obstime = Time(hdr['JD'], format='jd', scale='utc', location=image.site['EarthLocation']) + image.photfilter = { + 'Yc': 'Y', + 'Hc': 'H', + 'Jo': 'J', + }.get(hdr['FILTER'], hdr['FILTER']) + elif telescope == 'Baade' and hdr.get('SITENAME') == 'LCO' and instrument == 'FourStar': image.site = api.get_site(11) # Hard-coded the siteid for Swope, Las Campanas Observatory image.obstime = Time(hdr['JD'], format='jd', scale='utc', location=image.site['EarthLocation']) @@ -237,21 +246,43 @@ def load_image(FILENAME): }.get(hdr['FILTER'], hdr['FILTER']) image.exptime *= int(hdr['NCOMBINE']) # EXPTIME is only for a single exposure - elif origin == 'ESO' and telescope == 'ESO-NTT' and instrument == 'SOFI': + elif instrument == 'SOFI' and telescope in ('ESO-NTT', 'other') and (origin == 'ESO' or origin.startswith('NOAO-IRAF')): image.site = api.get_site(12) # Hard-coded the siteid for SOFT, ESO NTT - image.obstime = Time(hdr['TMID'], format='mjd', scale='utc', location=image.site['EarthLocation']) - image.photfilter = hdr['FILTER'] + if 'TMID' in hdr: + image.obstime = Time(hdr['TMID'], format='mjd', scale='utc', location=image.site['EarthLocation']) + else: + image.obstime = Time(hdr['MJD-OBS'], format='mjd', scale='utc', location=image.site['EarthLocation']) + image.obstime += 0.5*image.exptime * u.second # Make time centre of exposure + + # Photometric filter: + photfilter_translate = { + 'Ks': 'K' + } + if 'FILTER' in hdr: + image.photfilter = photfilter_translate.get(hdr['FILTER'], hdr['FILTER']) + else: + filters_used = [] + for check_headers in ('ESO INS FILT1 ID', 'ESO INS FILT2 ID'): + if hdr.get(check_headers) and hdr.get(check_headers).strip().lower() != 'open': + filters_used.append(hdr.get(check_headers).strip()) + if len(filters_used) == 1: + image.photfilter = photfilter_translate.get(filters_used[0], filters_used[0]) + else: + raise Exception("Could not determine filter used.") # Mask out "halo" of pixels with zero value along edge of image: image.mask |= edge_mask(image.image, value=0) - elif origin == 'ESO' and telescope == 'ESO-NTT' and instrument == 'EFOSC': + elif telescope == 'ESO-NTT' and instrument == 'EFOSC' and (origin == 'ESO' or origin.startswith('NOAO-IRAF')): image.site = api.get_site(15) # Hard-coded the siteid for EFOSC, ESO NTT image.obstime = Time(hdr['DATE-OBS'], format='isot', scale='utc', location=image.site['EarthLocation']) image.obstime += 0.5*image.exptime * u.second # Make time centre of exposure image.photfilter = { 'g782': 'gp', + 'r784': 'rp', + 'i705': 'ip', 'B639': 'B', + 'V641': 'V' }.get(hdr['FILTER'], hdr['FILTER']) elif telescope == 'SAI-2.5' and instrument == 'ASTRONIRCAM': diff --git a/flows/photometry.py b/flows/photometry.py index b85aa7a..4a66c39 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -256,6 +256,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): logger.info("References:\n%s", references) + # @TODO: These need to be based on the instrument! radius = 10 fwhm_guess = 6.0 fwhm_min = 3.5 @@ -279,169 +280,171 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # Use R^2 to more robustly determine initial FWHM guess. # This cleaning is good when we have FEW references. - fwhm, clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, + fwhm, fwhm_clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, min_fwhm_references=2, min_references=6, rsq_min=0.15) + logging.info('Initial FWHM guess is {} pixels'.format(fwhm)) + image.fwhm = fwhm + # Create plot of target and reference star positions from 2D Gaussian fits. fig, ax = plt.subplots(1, 1, figsize=(20, 18)) plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) - #ax.scatter(references['pixel_column'], references['pixel_row'], c='r', marker='o', alpha=0.3) - ax.scatter(clean_references['pixel_column'], clean_references['pixel_row'], c='r', marker='o', alpha=0.3) - #ax.scatter(masked_ref_xys[:,0], masked_ref_xys[:,0], marker='o', alpha=0.6, edgecolors='green', facecolors='none') + ax.scatter(fwhm_clean_references['pixel_column'], fwhm_clean_references['pixel_row'], c='r', marker='o', alpha=0.3) ax.scatter(masked_sep_xy[:,0],masked_sep_xy[:,1],marker='s',alpha=1.0, edgecolors='green' ,facecolors='none') ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') fig.savefig(os.path.join(output_folder, 'positions_g2d.png'), bbox_inches='tight') plt.close(fig) - # Sort by brightness - clean_references.sort('g_mag') # Sorted by g mag - _at = Table({'xy': masked_sep_xy, 'flux': objects['flux'][sep_mask]}) - _at.sort('flux', reverse=True) # Sorted by flux - masked_sep_xy = _at['xy'].data.data - - # Check WCS - wcs_rotation = 0 - wcs_rota_max = 3 - nreferences = len(clean_references['pixel_column']) - try_aa = True - while wcs_rotation < wcs_rota_max: - nreferences_old = nreferences - ref_xys = mkposxy(clean_references['pixel_column'], clean_references['pixel_row']) - - # Find matches using astroalign - # try_kd = True - # if try_aa: - # for maxstars in [80,4]: - # ref_ind, sep_ind, success_aa = try_astroalign(ref_xys, masked_sep_xy, - # pixeltol=4*fwhm, - # nnearest=min(20,len(ref_xys)), - # max_stars_n=max(maxstars,len(ref_xys))) - # # Break if successful - # if success_aa: - # astroalign_nmatches = len(ref_ind) - # try_kd = False - # if wcs_rotation > 1 and astroalign_nmatches <= 4: - # try_kd = True - # success_aa = False - # break - - try_kd = True - success_aa, try_aa = False, False # Don't use astroalign for now; it's giving false matches! - - # Find matches using nearest neighbor - if try_kd: - ref_ind_kd, sep_ind_kd, success_kd = kdtree(ref_xys, masked_sep_xy, fwhm, fwhm_max=4) - if success_kd: - kdtree_nmatches = len(ref_ind_kd) - if try_kd and kdtree_nmatches > 3: - ref_ind = ref_ind_kd - sep_ind = sep_ind_kd - else: - success_kd = False - - if success_aa or success_kd: - # Fit for new WCS - wcs_rotation += 1 - image.new_wcs = get_new_wcs(sep_ind, masked_sep_xy, clean_references, ref_ind, image.obstime) - - # Calculate pixel-coordinates of references: - row_col_coords = image.new_wcs.all_world2pix( - np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) - references['pixel_column'] = row_col_coords[:, 0] - references['pixel_row'] = row_col_coords[:, 1] - - # Clean out the references: - hsize = 10 - x = references['pixel_column'] - y = references['pixel_row'] - refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', - frame='icrs') - clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) - & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) - & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] - try: - masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], - clean_references['pixel_row'], - image, - get_fwhm=True, - radius=radius, - fwhm_guess=fwhm, - fwhm_max=fwhm_max, - fwhm_min=fwhm_min, - rsq_min=0.15) - # Clean with R^2 - fwhm, clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, - min_fwhm_references=2, min_references=6, rsq_min=0.15) - image.fwhm = fwhm - nreferences_new = len(clean_references) - logging.info('{} References were found after new wcs compared to {} references before'.format(nreferences_old,nreferences_new)) - nreferences = nreferences_new - wcs_success = True - - # Break early if no improvement after 2nd pass! - # Note: New references can actually be less in a better WCS - # if the actual stars were within radius (10) pixels of the edge. - # @TODO: Adjust nreferences new and old based on whether extracted stars are within radius pix of edge. - if wcs_rotation > 1 and nreferences_new <= nreferences_old: - break - except: - # Calculate pixel-coordinates of references using old wcs: - row_col_coords = image.wcs.all_world2pix( - np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) - references['pixel_column'] = row_col_coords[:, 0] - references['pixel_row'] = row_col_coords[:, 1] - - # Clean out the references: - hsize = 10 - x = references['pixel_column'] - y = references['pixel_row'] - refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', - frame='icrs') - clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) - & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) - & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] - wcs_success = False - if try_aa: try_aa = False - elif try_kd: break - - else: - logging.info('New WCS could not be computed due to lack of matches.') - wcs_success = False - break - - if wcs_success: - image.wcs = image.new_wcs - - # @Todo: Is the below block needed? - # Final cleanout of references - # Calculate pixel-coordinates of references using old wcs: - row_col_coords = image.wcs.all_world2pix( - np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) - references['pixel_column'] = row_col_coords[:, 0] - references['pixel_row'] = row_col_coords[:, 1] - - # Clean out the references: - hsize = 10 - x = references['pixel_column'] - y = references['pixel_row'] - refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', - frame='icrs') - references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) - & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) - & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] - masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(references['pixel_column'], - references['pixel_row'], - image, - get_fwhm=True, - radius=radius, - fwhm_guess=fwhm, - fwhm_max=fwhm_max, - fwhm_min=fwhm_min, - rsq_min=0.15) - - logger.debug("Number of references before cleaning: %d", len(references)) - references = get_clean_references(references, masked_rsqs) - logger.debug("Number of references after cleaning: %d", len(references)) + # # Sort by brightness + # clean_references.sort('g_mag') # Sorted by g mag + # _at = Table({'xy': masked_sep_xy, 'flux': objects['flux'][sep_mask]}) + # _at.sort('flux', reverse=True) # Sorted by flux + # masked_sep_xy = _at['xy'].data.data + + # # Check WCS + # wcs_rotation = 0 + # wcs_rota_max = 3 + # nreferences = len(clean_references['pixel_column']) + # try_aa = True + # while wcs_rotation < wcs_rota_max: + # nreferences_old = nreferences + # ref_xys = mkposxy(clean_references['pixel_column'], clean_references['pixel_row']) + # + # # Find matches using astroalign + # # try_kd = True + # # if try_aa: + # # for maxstars in [80,4]: + # # ref_ind, sep_ind, success_aa = try_astroalign(ref_xys, masked_sep_xy, + # # pixeltol=4*fwhm, + # # nnearest=min(20,len(ref_xys)), + # # max_stars_n=max(maxstars,len(ref_xys))) + # # # Break if successful + # # if success_aa: + # # astroalign_nmatches = len(ref_ind) + # # try_kd = False + # # if wcs_rotation > 1 and astroalign_nmatches <= 4: + # # try_kd = True + # # success_aa = False + # # break + # + # try_kd = True + # success_aa, try_aa = False, False # Don't use astroalign for now; it's giving false matches! + # + # # Find matches using nearest neighbor + # if try_kd: + # ref_ind_kd, sep_ind_kd, success_kd = kdtree(ref_xys, masked_sep_xy, fwhm, fwhm_max=4) + # if success_kd: + # kdtree_nmatches = len(ref_ind_kd) + # if try_kd and kdtree_nmatches > 3: + # ref_ind = ref_ind_kd + # sep_ind = sep_ind_kd + # else: + # success_kd = False + # + # if success_aa or success_kd: + # # Fit for new WCS + # wcs_rotation += 1 + # image.new_wcs = get_new_wcs(sep_ind, masked_sep_xy, clean_references, ref_ind, image.obstime) + # + # # Calculate pixel-coordinates of references: + # row_col_coords = image.new_wcs.all_world2pix( + # np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) + # references['pixel_column'] = row_col_coords[:, 0] + # references['pixel_row'] = row_col_coords[:, 1] + # + # # Clean out the references: + # hsize = 10 + # x = references['pixel_column'] + # y = references['pixel_row'] + # refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', + # frame='icrs') + # clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) + # & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) + # & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] + # try: + # masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], + # clean_references['pixel_row'], + # image, + # get_fwhm=True, + # radius=radius, + # fwhm_guess=fwhm, + # fwhm_max=fwhm_max, + # fwhm_min=fwhm_min, + # rsq_min=0.15) + # # Clean with R^2 + # fwhm, clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, + # min_fwhm_references=2, min_references=6, rsq_min=0.15) + # image.fwhm = fwhm + # nreferences_new = len(clean_references) + # logging.info('{} References were found after new wcs compared to {} references before'.format(nreferences_old,nreferences_new)) + # nreferences = nreferences_new + # wcs_success = True + # + # # Break early if no improvement after 2nd pass! + # # Note: New references can actually be less in a better WCS + # # if the actual stars were within radius (10) pixels of the edge. + # # @TODO: Adjust nreferences new and old based on whether extracted stars are within radius pix of edge. + # if wcs_rotation > 1 and nreferences_new <= nreferences_old: + # break + # except: + # # Calculate pixel-coordinates of references using old wcs: + # row_col_coords = image.wcs.all_world2pix( + # np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) + # references['pixel_column'] = row_col_coords[:, 0] + # references['pixel_row'] = row_col_coords[:, 1] + # + # # Clean out the references: + # hsize = 10 + # x = references['pixel_column'] + # y = references['pixel_row'] + # refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', + # frame='icrs') + # clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) + # & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) + # & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] + # wcs_success = False + # if try_aa: try_aa = False + # elif try_kd: break + # + # else: + # logging.info('New WCS could not be computed due to lack of matches.') + # wcs_success = False + # break + # + # if wcs_success: + # image.wcs = image.new_wcs + # + # # @Todo: Is the below block needed? + # # Final cleanout of references + # # Calculate pixel-coordinates of references using old wcs: + # row_col_coords = image.wcs.all_world2pix( + # np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) + # references['pixel_column'] = row_col_coords[:, 0] + # references['pixel_row'] = row_col_coords[:, 1] + # + # # Clean out the references: + # hsize = 10 + # x = references['pixel_column'] + # y = references['pixel_row'] + # refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', + # frame='icrs') + # references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) + # & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) + # & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] + + # masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], + # clean_references['pixel_row'], + # image, + # get_fwhm=True, + # radius=radius, + # fwhm_guess=fwhm, + # fwhm_max=fwhm_max, + # fwhm_min=fwhm_min, + # rsq_min=0.15) + + logger.debug("Number of references before final cleaning: %d", len(clean_references)) + references = get_clean_references(clean_references, masked_rsqs) + logger.debug("Number of references after final cleaning: %d", len(references)) # Create plot of target and reference star positions: fig, ax = plt.subplots(1, 1, figsize=(20, 18)) @@ -558,6 +561,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # Let's make the final FWHM the largest one we found: fwhm = np.max(fwhms) + iamge.fwhm = fwhm logger.info("Final FWHM based on ePSF: %f", fwhm) #============================================================================================== diff --git a/flows/wcs.py b/flows/wcs.py index 691ed29..07d8935 100644 --- a/flows/wcs.py +++ b/flows/wcs.py @@ -224,9 +224,9 @@ def kdtree(source, target, fwhm=5, fwhm_max=4, min_matches=3): # Return indexes of matches return sources, targets, len(sources)>= min_matches -def get_new_wcs(extracted_ind,extracted_stars,clean_references,ref_ind,obstime): +def get_new_wcs(extracted_ind,extracted_stars,clean_references,ref_ind,obstime,rakey='ra_obs',deckey='decl_obs'): targets = (extracted_stars[extracted_ind][:,0],extracted_stars[extracted_ind][:,1]) - c = coords.SkyCoord(clean_references['ra_obs'][ref_ind],clean_references['decl_obs'][ref_ind],obstime=obstime) + c = coords.SkyCoord(clean_references[rakey][ref_ind],clean_references[deckey][ref_ind],obstime=obstime) return wcs.utils.fit_wcs_from_points(targets,c) def get_clean_references(references, masked_rsqs, min_references_ideal=6, @@ -240,7 +240,7 @@ def get_clean_references(references, masked_rsqs, min_references_ideal=6, nmasked_rsqs = nmasked_rsqs[:min(min_references_ideal, len(nmasked_rsqs))] if len(nmasked_rsqs>=min_references_abs): return references[nmasked_rsqs] - if not rescue_bad: + if not rescue_bad: raise MinStarError('Less than {} clean stars and rescue_bad = False'.format(min_references_abs)) elif rescue_bad: mask = (masked_rsqs >= 0.02) & (masked_rsqs < 1.0) @@ -250,5 +250,3 @@ def get_clean_references(references, masked_rsqs, min_references_ideal=6, raise MinStarError('Less than 2 clean stars.') return references[nmasked_rsqs] raise ValueError('input parameters were wrong, you should not reach here. Check that rescue_bad is True or False.') - - diff --git a/flows/ztf.py b/flows/ztf.py index 3eb530a..14a1578 100644 --- a/flows/ztf.py +++ b/flows/ztf.py @@ -8,6 +8,7 @@ .. codeauthor:: Rasmus Handberg """ +import numpy as np import astropy.units as u from astropy.coordinates import Angle from astropy.table import Table @@ -89,17 +90,25 @@ def download_ztf_photometry(targetid): # Create Astropy table, cut out the needed columns # and rename columns to something better for what we are doing: tab = Table(data=jsn['result']['detections']) - tab = tab[['fid','mjd','magpsf_corr','sigmapsf_corr']] + tab = tab[['fid', 'mjd', 'magpsf_corr', 'sigmapsf_corr']] tab.rename_column('fid', 'photfilter') + tab.rename_column('mjd', 'time') tab.rename_column('magpsf_corr', 'mag') tab.rename_column('sigmapsf_corr', 'mag_err') + # Remove bad values of time and magnitude: + tab['time'] = np.asarray(tab['time'], dtype='float64') + tab['mag'] = np.asarray(tab['mag'], dtype='float64') + tab['mag_err'] = np.asarray(tab['mag_err'], dtype='float64') + indx = np.isfinite(tab['time']) & np.isfinite(tab['mag']) & np.isfinite(tab['mag_err']) + tab = tab[indx] + # Replace photometric filter numbers with keywords used in Flows: photfilter_dict = {1: 'gp', 2: 'rp', 3: 'ip'} tab['photfilter'] = [photfilter_dict[fid] for fid in tab['photfilter']] - # Sort the table on photfilter and mjd: - tab.sort(['photfilter', 'mjd']) + # Sort the table on photfilter and time: + tab.sort(['photfilter', 'time']) # Add meta information to table header: tab.meta['target_name'] = target_name diff --git a/run_download_ztf.py b/run_download_ztf.py index 9030407..facfac3 100644 --- a/run_download_ztf.py +++ b/run_download_ztf.py @@ -77,7 +77,7 @@ def main(): # Find time of maxmimum and 14 days from that: indx_min = np.argmin(tab['mag']) - maximum_mjd = tab['mjd'][indx_min] + maximum_mjd = tab['time'][indx_min] fortnight_mjd = maximum_mjd + 14 # Get LC data out and save as CSV files @@ -86,7 +86,7 @@ def main(): ax.axvline(fortnight_mjd, ls='--', c='0.5', lw=0.5, label='+14 days') for fid in np.unique(tab['photfilter']): band = tab[tab['photfilter'] == fid] - ax.errorbar(band['mjd'], band['mag'], band['mag_err'], + ax.errorbar(band['time'], band['mag'], band['mag_err'], ls='-', lw=0.5, marker='.', label=fid) ax.invert_yaxis() From 10158b473788a926e9bce318cd74cba7e9fec14a Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 3 Feb 2021 12:33:08 +0100 Subject: [PATCH 11/43] Bugfix --- flows/photometry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flows/photometry.py b/flows/photometry.py index 4a66c39..6dcb7d2 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -443,7 +443,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # rsq_min=0.15) logger.debug("Number of references before final cleaning: %d", len(clean_references)) - references = get_clean_references(clean_references, masked_rsqs) + references = get_clean_references(clean_references, masked_rsqs, rsq_ideal=0.8) logger.debug("Number of references after final cleaning: %d", len(references)) # Create plot of target and reference star positions: @@ -561,7 +561,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # Let's make the final FWHM the largest one we found: fwhm = np.max(fwhms) - iamge.fwhm = fwhm + image.fwhm = fwhm logger.info("Final FWHM based on ePSF: %f", fwhm) #============================================================================================== From 3ee000ecb532fdcfd77956182477e544f8fade45 Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 17 Feb 2021 12:27:48 +0100 Subject: [PATCH 12/43] Bug Fix in final clean --- flows/wcs.py | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/flows/wcs.py b/flows/wcs.py index 07d8935..2fab12c 100644 --- a/flows/wcs.py +++ b/flows/wcs.py @@ -71,7 +71,7 @@ def force_reject_g2d(xarray, yarray, image, get_fwhm=True, rsq_min=0.5, radius=1 masked_xys = np.ma.masked_array(xys, ~np.isfinite(xys)) masked_rsqs = np.ma.masked_array(rsqs, ~np.isfinite(rsqs)) - mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) # Reject Rsq<0.5 + mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) # Reject Rsq < rsq_min masked_xys = masked_xys[mask] # Clean extracted array. masked_fwhms = np.ma.masked_array(fwhms, ~np.isfinite(fwhms)) @@ -79,7 +79,6 @@ def force_reject_g2d(xarray, yarray, image, get_fwhm=True, rsq_min=0.5, radius=1 return masked_xys,mask,masked_rsqs - def clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, references, min_fwhm_references = 2, min_references = 6, rsq_min = 0.15): """ @@ -231,22 +230,45 @@ def get_new_wcs(extracted_ind,extracted_stars,clean_references,ref_ind,obstime,r def get_clean_references(references, masked_rsqs, min_references_ideal=6, min_references_abs=3, rsq_min=0.15, rsq_ideal=0.5, + keep_max=100, rescue_bad: bool = True): + + # Greedy first try mask = (masked_rsqs >= rsq_ideal) & (masked_rsqs < 1.0) if np.sum(np.isfinite(masked_rsqs[mask])) >= min_references_ideal: - return references[mask] + if len(references[mask]) <= keep_max: + return references[mask] + else: + import pandas as pd # @TODO: Convert to pure numpy implementation + df = pd.DataFrame(masked_rsqs,columns=['rsq']) + nmasked_rsqs = df.sort_values('rsq',ascending=False).dropna().index._data + references[nmasked_rsqs[:keep_max]] + + # Desperate second try mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) - nmasked_rsqs = np.argsort(masked_rsqs[mask])[::-1] + masked_rsqs.mask = mask + + # Switching to pandas for easier selection + import pandas as pd # @TODO: Convert to pure numpy implementation + df = pd.DataFrame(masked_rsqs,columns=['rsq']) + nmasked_rsqs = df.sort_values('rsq',ascending=False).dropna().index._data nmasked_rsqs = nmasked_rsqs[:min(min_references_ideal, len(nmasked_rsqs))] - if len(nmasked_rsqs>=min_references_abs): + if len(nmasked_rsqs) >= min_references_abs: return references[nmasked_rsqs] if not rescue_bad: raise MinStarError('Less than {} clean stars and rescue_bad = False'.format(min_references_abs)) + + # Extremely desperate last ditch attempt i.e. "rescue bad" elif rescue_bad: mask = (masked_rsqs >= 0.02) & (masked_rsqs < 1.0) - nmasked_rsqs = np.argsort(masked_rsqs[mask])[::-1] + masked_rsqs.mask = mask + + # Switch to pandas + df = pd.DataFrame(masked_rsqs,columns=['rsq']) + nmasked_rsqs = df.sort_values('rsq',ascending=False).dropna().index._data nmasked_rsqs = nmasked_rsqs[:min(min_references_ideal, len(nmasked_rsqs))] if len(nmasked_rsqs) < 2 : raise MinStarError('Less than 2 clean stars.') - return references[nmasked_rsqs] + return references[nmasked_rsqs] # Return if len >= 2 + # Checks whether sensible input arrays and parameters were provided raise ValueError('input parameters were wrong, you should not reach here. Check that rescue_bad is True or False.') From af65ca60a388be37641ff2334d5ed1f1b083b978 Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 17 Feb 2021 12:28:15 +0100 Subject: [PATCH 13/43] Weave in wcs correction, bugfix, --- flows/photometry.py | 1430 ++++++++++++++++++++----------------------- 1 file changed, 650 insertions(+), 780 deletions(-) diff --git a/flows/photometry.py b/flows/photometry.py index 6dcb7d2..b537ac1 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -9,7 +9,7 @@ import os import numpy as np -from bottleneck import nansum, nanmedian, nanmax, allnan, replace +from bottleneck import nansum, allnan, replace import sep from timeit import default_timer import logging @@ -18,7 +18,7 @@ from astropy.utils.exceptions import AstropyDeprecationWarning import astropy.units as u import astropy.coordinates as coords -from astropy.stats import sigma_clip, SigmaClip, gaussian_fwhm_to_sigma +from astropy.stats import sigma_clip, SigmaClip from astropy.table import Table, vstack from astropy.nddata import NDData from astropy.modeling import models, fitting @@ -26,10 +26,9 @@ from astropy.time import Time warnings.simplefilter('ignore', category=AstropyDeprecationWarning) -from photutils import DAOStarFinder, CircularAperture, CircularAnnulus, aperture_photometry +from photutils import CircularAperture, CircularAnnulus, aperture_photometry from photutils.psf import EPSFBuilder, EPSFFitter, BasicPSFPhotometry, DAOGroup, extract_stars -from photutils.centroids import centroid_com -from photutils import Background2D, SExtractorBackground +from photutils import Background2D, SExtractorBackground, MedianBackground from photutils.utils import calc_total_error from photutils.centroids import centroid_com @@ -42,784 +41,655 @@ from .load_image import load_image from .run_imagematch import run_imagematch from .zeropoint import bootstrap_outlier, sigma_from_Chauvenet -from .wcs import force_reject_g2d, mkposxy, clean_with_rsq_and_get_fwhm, \ - try_astroalign, kdtree, get_new_wcs, get_clean_references +from .wcs import force_reject_g2d, clean_with_rsq_and_get_fwhm, get_clean_references from .coordinatematch import CoordinateMatch __version__ = get_version(pep440=False) warnings.simplefilter('ignore', category=AstropyDeprecationWarning) -#-------------------------------------------------------------------------------------------------- -def photometry(fileid, output_folder=None, attempt_imagematch=True): - """ - Run photometry. - - Parameters: - fileid (int): File ID to process. - output_folder (str, optional): Path to directory where output should be placed. - attempt_imagematch (bool, optional): If no subtracted image is available, but a - template image is, should we attempt to run ImageMatch using standard settings. - Default=True. - - .. codeauthor:: Rasmus Handberg - """ - - # Settings: - #ref_mag_limit = 22 # Lower limit on reference target brightness - ref_target_dist_limit = 10 * u.arcsec # Reference star must be further than this away to be included - - logger = logging.getLogger(__name__) - tic = default_timer() - - # Use local copy of archive if configured to do so: - config = load_config() - - # Get datafile dict from API: - datafile = api.get_datafile(fileid) - logger.debug("Datafile: %s", datafile) - targetid = datafile['targetid'] - target_name = datafile['target_name'] - photfilter = datafile['photfilter'] - - archive_local = config.get('photometry', 'archive_local', fallback=None) - if archive_local is not None: - datafile['archive_path'] = archive_local - if not os.path.isdir(datafile['archive_path']): - raise FileNotFoundError("ARCHIVE is not available: " + datafile['archive_path']) - - # Get the catalog containing the target and reference stars: - # TODO: Include proper-motion to the time of observation - catalog = api.get_catalog(targetid, output='table') - target = catalog['target'][0] - target_coord = coords.SkyCoord(ra=target['ra'], dec=target['decl'], unit='deg', frame='icrs') - - # Folder to save output: - if output_folder is None: - output_folder_root = config.get('photometry', 'output', fallback='.') - output_folder = os.path.join(output_folder_root, target_name, '%05d' % fileid) - logger.info("Placing output in '%s'", output_folder) - os.makedirs(output_folder, exist_ok=True) - - # The paths to the science image: - filepath = os.path.join(datafile['archive_path'], datafile['path']) - - # TODO: Download datafile using API to local drive: - # TODO: Is this a security concern? - #if archive_local: - # api.download_datafile(datafile, archive_local) - - # Translate photometric filter into table column: - ref_filter = { - 'up': 'u_mag', - 'gp': 'g_mag', - 'rp': 'r_mag', - 'ip': 'i_mag', - 'zp': 'z_mag', - 'B': 'B_mag', - 'V': 'V_mag', - 'J': 'J_mag', - 'H': 'H_mag', - 'K': 'K_mag', - }.get(photfilter, None) - - if ref_filter is None: - logger.warning("Could not find filter '%s' in catalogs. Using default gp filter.", photfilter) - ref_filter = 'g_mag' - - references = catalog['references'] - references.sort(ref_filter) - - # Check that there actually are reference stars in that filter: - if allnan(references[ref_filter]): - raise ValueError("No reference stars found in current photfilter.") - - # Load the image from the FITS file: - image = load_image(filepath) - - #============================================================================================== - # BARYCENTRIC CORRECTION OF TIME - #============================================================================================== - - ltt_bary = image.obstime.light_travel_time(target_coord, ephemeris='jpl') - image.obstime = image.obstime.tdb + ltt_bary - - #============================================================================================== - # BACKGROUND ESTIMATION - #============================================================================================== - - fig, ax = plt.subplots(1, 2, figsize=(20, 18)) - plot_image(image.clean, ax=ax[0], scale='log', cbar='right', title='Image') - plot_image(image.mask, ax=ax[1], scale='linear', cbar='right', title='Mask') - fig.savefig(os.path.join(output_folder, 'original.png'), bbox_inches='tight') - plt.close(fig) - - # Estimate image background: - # Not using image.clean here, since we are redefining the mask anyway - bkg = Background2D(image.clean, (128, 128), filter_size=(5, 5), - sigma_clip=SigmaClip(sigma=3.0), - bkg_estimator=SExtractorBackground(), - exclude_percentile=50.0) - image.background = bkg.background - image.std = bkg.background_rms_median - - # Create background-subtracted image: - image.subclean = image.clean - image.background - - # Plot background estimation: - fig, ax = plt.subplots(1, 3, figsize=(20, 6)) - plot_image(image.clean, ax=ax[0], scale='log', title='Original') - plot_image(image.background, ax=ax[1], scale='log', title='Background') - plot_image(image.subclean, ax=ax[2], scale='log', title='Background subtracted') - fig.savefig(os.path.join(output_folder, 'background.png'), bbox_inches='tight') - plt.close(fig) - - # TODO: Is this correct?! - image.error = calc_total_error(image.clean, bkg.background_rms, 1.0) - - # Use sep to for soure extraction - image.sepdata = image.image.byteswap().newbyteorder() - image.sepbkg = sep.Background(image.sepdata,mask=image.mask) - image.sepsub = image.sepdata - image.sepbkg - logger.debug('sub: {} bkg_rms: {} mask: {}'.format(np.shape(image.sepsub),np.shape(image.sepbkg.globalrms), - np.shape(image.mask))) - objects = sep.extract(image.sepsub, thresh=5., err=image.sepbkg.globalrms, mask=image.mask, - deblend_cont=0.1, minarea=9, clean_param=2.0) - - - #============================================================================================== - # DETECTION OF STARS AND MATCHING WITH CATALOG - #============================================================================================== - - # Account for proper motion: - # TODO: Are catalog RA-proper motions including cosdec? - replace(references['pm_ra'], np.NaN, 0) - replace(references['pm_dec'], np.NaN, 0) - refs_coord = coords.SkyCoord(ra=references['ra'], dec=references['decl'], - pm_ra_cosdec=references['pm_ra'], pm_dec=references['pm_dec'], - unit='deg', frame='icrs', obstime=Time(2015.5, format='decimalyear')) - - refs_coord = refs_coord.apply_space_motion(image.obstime) - - # Solve for new WCS - cm = CoordinateMatch( - xy = list(zip(objects['x'], objects['y'])), - rd = list(zip(refs_coord.ra.deg, refs_coord.dec.deg)), - xy_order = np.argsort(-2.5*np.log10(objects['flux'])), - rd_order = np.argsort(target_coord.separation(refs_coord)), - maximum_angle_distance = 0.002, - ) - - try: - i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=10))) - except (TimeoutError, StopIteration): - logging.warning('No new WCS solution found') - else: - image.wcs = fit_wcs_from_points( - np.array(list(zip(*cm.xy[i_xy]))), - coords.SkyCoord(*map(list, zip(*cm.rd[i_rd])), unit='deg') - ) - -# XXX # TESTING ################################################################ -# import matplotlib -# _backend = matplotlib.get_backend() -# matplotlib.pyplot.switch_backend('TkAgg') -# matplotlib.pyplot.subplot(projection=image.wcs) -# matplotlib.pyplot.imshow(image.subclean, origin='lower', cmap='gray_r', clim=np.nanquantile(image.subclean[image.subclean>0], (.01, .99))) -# matplotlib.pyplot.scatter(*zip(*image.wcs.wcs_world2pix(cm.rd, 0)), c=[], edgecolor='C0', s=200, label='References') -# for i, ((_x, _y), (_r, _d)) in enumerate(zip(cm.xy[i_xy], image.wcs.wcs_world2pix(cm.rd[i_rd], 0))): -# matplotlib.pyplot.plot([_x, _r], [_y, _d], color='C1', marker='o', ms=9, mfc='none', mec='C1', label='Solution' if not i else None) -# matplotlib.pyplot.scatter(*zip(*image.wcs.wcs_world2pix([(target['ra'], target['decl'])], 0)), c=[], edgecolor='C2', s=100, label='Target') -# matplotlib.pyplot.xlim(0, image.subclean.shape[1]) -# matplotlib.pyplot.ylim(0, image.subclean.shape[0]) -# matplotlib.pyplot.legend() -# matplotlib.pyplot.show() -# matplotlib.pyplot.switch_backend(_backend) -# XXX # TESTING ################################################################ - - # Calculate pixel-coordinates of references: - row_col_coords = image.wcs.all_world2pix(np.array([[ref.ra.deg, ref.dec.deg] for ref in refs_coord]), 0) - references['pixel_column'] = row_col_coords[:,0] - references['pixel_row'] = row_col_coords[:,1] - - # Calculate the targets position in the image: - target_pixel_pos = image.wcs.all_world2pix([[target['ra'], target['decl']]], 0)[0] - - # Clean out the references: - hsize = 10 - x = references['pixel_column'] - y = references['pixel_row'] - clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) - & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) - & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] - # & (references[ref_filter] < ref_mag_limit) - - logger.info("References:\n%s", references) - - # @TODO: These need to be based on the instrument! - radius = 10 - fwhm_guess = 6.0 - fwhm_min = 3.5 - fwhm_max = 18.0 - - # Clean extracted stars - masked_sep_xy,sep_mask,masked_sep_rsqs = force_reject_g2d(objects['x'], objects['y'], image, get_fwhm=False, - radius=radius, fwhm_guess=fwhm_guess, rsq_min=0.3, fwhm_max=fwhm_max, fwhm_min=fwhm_min) - - # Clean reference star locations - masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], - clean_references['pixel_row'], - image, - get_fwhm=True, - radius=radius, - fwhm_guess=fwhm_guess, - fwhm_max=fwhm_max, - fwhm_min=fwhm_min, - rsq_min=0.15) - - - # Use R^2 to more robustly determine initial FWHM guess. - # This cleaning is good when we have FEW references. - fwhm, fwhm_clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, - min_fwhm_references=2, min_references=6, rsq_min=0.15) - logging.info('Initial FWHM guess is {} pixels'.format(fwhm)) - image.fwhm = fwhm - - - # Create plot of target and reference star positions from 2D Gaussian fits. - fig, ax = plt.subplots(1, 1, figsize=(20, 18)) - plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) - ax.scatter(fwhm_clean_references['pixel_column'], fwhm_clean_references['pixel_row'], c='r', marker='o', alpha=0.3) - ax.scatter(masked_sep_xy[:,0],masked_sep_xy[:,1],marker='s',alpha=1.0, edgecolors='green' ,facecolors='none') - ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') - fig.savefig(os.path.join(output_folder, 'positions_g2d.png'), bbox_inches='tight') - plt.close(fig) - - # # Sort by brightness - # clean_references.sort('g_mag') # Sorted by g mag - # _at = Table({'xy': masked_sep_xy, 'flux': objects['flux'][sep_mask]}) - # _at.sort('flux', reverse=True) # Sorted by flux - # masked_sep_xy = _at['xy'].data.data - - # # Check WCS - # wcs_rotation = 0 - # wcs_rota_max = 3 - # nreferences = len(clean_references['pixel_column']) - # try_aa = True - # while wcs_rotation < wcs_rota_max: - # nreferences_old = nreferences - # ref_xys = mkposxy(clean_references['pixel_column'], clean_references['pixel_row']) - # - # # Find matches using astroalign - # # try_kd = True - # # if try_aa: - # # for maxstars in [80,4]: - # # ref_ind, sep_ind, success_aa = try_astroalign(ref_xys, masked_sep_xy, - # # pixeltol=4*fwhm, - # # nnearest=min(20,len(ref_xys)), - # # max_stars_n=max(maxstars,len(ref_xys))) - # # # Break if successful - # # if success_aa: - # # astroalign_nmatches = len(ref_ind) - # # try_kd = False - # # if wcs_rotation > 1 and astroalign_nmatches <= 4: - # # try_kd = True - # # success_aa = False - # # break - # - # try_kd = True - # success_aa, try_aa = False, False # Don't use astroalign for now; it's giving false matches! - # - # # Find matches using nearest neighbor - # if try_kd: - # ref_ind_kd, sep_ind_kd, success_kd = kdtree(ref_xys, masked_sep_xy, fwhm, fwhm_max=4) - # if success_kd: - # kdtree_nmatches = len(ref_ind_kd) - # if try_kd and kdtree_nmatches > 3: - # ref_ind = ref_ind_kd - # sep_ind = sep_ind_kd - # else: - # success_kd = False - # - # if success_aa or success_kd: - # # Fit for new WCS - # wcs_rotation += 1 - # image.new_wcs = get_new_wcs(sep_ind, masked_sep_xy, clean_references, ref_ind, image.obstime) - # - # # Calculate pixel-coordinates of references: - # row_col_coords = image.new_wcs.all_world2pix( - # np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) - # references['pixel_column'] = row_col_coords[:, 0] - # references['pixel_row'] = row_col_coords[:, 1] - # - # # Clean out the references: - # hsize = 10 - # x = references['pixel_column'] - # y = references['pixel_row'] - # refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', - # frame='icrs') - # clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) - # & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) - # & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] - # try: - # masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], - # clean_references['pixel_row'], - # image, - # get_fwhm=True, - # radius=radius, - # fwhm_guess=fwhm, - # fwhm_max=fwhm_max, - # fwhm_min=fwhm_min, - # rsq_min=0.15) - # # Clean with R^2 - # fwhm, clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, - # min_fwhm_references=2, min_references=6, rsq_min=0.15) - # image.fwhm = fwhm - # nreferences_new = len(clean_references) - # logging.info('{} References were found after new wcs compared to {} references before'.format(nreferences_old,nreferences_new)) - # nreferences = nreferences_new - # wcs_success = True - # - # # Break early if no improvement after 2nd pass! - # # Note: New references can actually be less in a better WCS - # # if the actual stars were within radius (10) pixels of the edge. - # # @TODO: Adjust nreferences new and old based on whether extracted stars are within radius pix of edge. - # if wcs_rotation > 1 and nreferences_new <= nreferences_old: - # break - # except: - # # Calculate pixel-coordinates of references using old wcs: - # row_col_coords = image.wcs.all_world2pix( - # np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) - # references['pixel_column'] = row_col_coords[:, 0] - # references['pixel_row'] = row_col_coords[:, 1] - # - # # Clean out the references: - # hsize = 10 - # x = references['pixel_column'] - # y = references['pixel_row'] - # refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', - # frame='icrs') - # clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) - # & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) - # & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] - # wcs_success = False - # if try_aa: try_aa = False - # elif try_kd: break - # - # else: - # logging.info('New WCS could not be computed due to lack of matches.') - # wcs_success = False - # break - # - # if wcs_success: - # image.wcs = image.new_wcs - # - # # @Todo: Is the below block needed? - # # Final cleanout of references - # # Calculate pixel-coordinates of references using old wcs: - # row_col_coords = image.wcs.all_world2pix( - # np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) - # references['pixel_column'] = row_col_coords[:, 0] - # references['pixel_row'] = row_col_coords[:, 1] - # - # # Clean out the references: - # hsize = 10 - # x = references['pixel_column'] - # y = references['pixel_row'] - # refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', - # frame='icrs') - # references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) - # & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) - # & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] - - # masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], - # clean_references['pixel_row'], - # image, - # get_fwhm=True, - # radius=radius, - # fwhm_guess=fwhm, - # fwhm_max=fwhm_max, - # fwhm_min=fwhm_min, - # rsq_min=0.15) - - logger.debug("Number of references before final cleaning: %d", len(clean_references)) - references = get_clean_references(clean_references, masked_rsqs, rsq_ideal=0.8) - logger.debug("Number of references after final cleaning: %d", len(references)) - - # Create plot of target and reference star positions: - fig, ax = plt.subplots(1, 1, figsize=(20, 18)) - plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) - ax.scatter(references['pixel_column'], references['pixel_row'], c='r', marker='o', alpha=0.6) - ax.scatter(masked_sep_xy[:,0],masked_sep_xy[:,1], marker='s' , alpha=0.6, edgecolors='green' ,facecolors='none') - ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') - fig.savefig(os.path.join(output_folder, 'positions.png'), bbox_inches='tight') - plt.close(fig) - - #============================================================================================== - # CREATE EFFECTIVE PSF MODEL - #============================================================================================== - - # Make cutouts of stars using extract_stars: - # Scales with FWHM - size = int(np.round(29*fwhm/6)) - if size % 2 == 0: - size += 1 # Make sure it's a uneven number - size = max(size, 15) # Never go below 15 pixels - hsize = (size - 1) / 2 - - x = references['pixel_column'] - y = references['pixel_row'] - mask_near_edge = ((x > hsize) & (x < (image.shape[1] - 1 - hsize)) - & (y > hsize) & (y < (image.shape[0] - 1 - hsize))) - - stars_for_epsf = Table() - stars_for_epsf['x'] = x[mask_near_edge] - stars_for_epsf['y'] = y[mask_near_edge] - - # Store which stars were used in ePSF in the table: - logger.info("Number of stars used for ePSF: %d", len(stars_for_epsf)) - references['used_for_epsf'] = mask_near_edge - - # Extract stars sub-images: - stars = extract_stars( - NDData(data=image.subclean, mask=image.mask), - stars_for_epsf, - size=size - ) - - # Plot the stars being used for ePSF: - nrows = 5 - ncols = 5 - imgnr = 0 - for k in range(int(np.ceil(len(stars_for_epsf)/(nrows*ncols)))): - fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True) - ax = ax.ravel() - for i in range(nrows*ncols): - if imgnr > len(stars_for_epsf)-1: - ax[i].axis('off') - else: - plot_image(stars[imgnr], ax=ax[i], scale='log', cmap='viridis') - imgnr += 1 - - fig.savefig(os.path.join(output_folder, 'epsf_stars%02d.png' % (k+1)), bbox_inches='tight') - plt.close(fig) - - # Build the ePSF: - epsf = EPSFBuilder( - oversampling=1.0, - maxiters=500, - fitter=EPSFFitter(fit_boxsize=np.round(2*fwhm,0).astype(int)), - progress_bar=True, - recentering_func=centroid_com - )(stars)[0] - - logger.info('Successfully built PSF model') - - fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 15)) - plot_image(epsf.data, ax=ax1, cmap='viridis') - - fwhms = [] - bad_epsf_detected = False - for a, ax in ((0, ax3), (1, ax2)): - # Collapse the PDF along this axis: - profile = epsf.data.sum(axis=a) - itop = profile.argmax() - poffset = profile[itop]/2 - - # Run a spline through the points, but subtract half of the peak value, and find the roots: - # We have to use a cubic spline, since roots() is not supported for other splines - # for some reason - profile_intp = UnivariateSpline(np.arange(0, len(profile)), profile - poffset, k=3, s=0, ext=3) - lr = profile_intp.roots() - - # Plot the profile and spline: - x_fine = np.linspace(-0.5, len(profile)-0.5, 500) - ax.plot(profile, 'k.-') - ax.plot(x_fine, profile_intp(x_fine) + poffset, 'g-') - ax.axvline(itop) - ax.set_xlim(-0.5, len(profile)-0.5) - - # Do some sanity checks on the ePSF: - # It should pass 50% exactly twice and have the maximum inside that region. - # I.e. it should be a single gaussian-like peak - if len(lr) != 2 or itop < lr[0] or itop > lr[1]: - logger.error("Bad PSF along axis %d", a) - bad_epsf_detected = True - else: - axis_fwhm = lr[1] - lr[0] - fwhms.append(axis_fwhm) - ax.axvspan(lr[0], lr[1], facecolor='g', alpha=0.2) - - # Save the ePSF figure: - ax4.axis('off') - fig.savefig(os.path.join(output_folder, 'epsf.png'), bbox_inches='tight') - plt.close(fig) - - # There was a problem with the ePSF: - if bad_epsf_detected: - raise Exception("Bad ePSF detected.") - - # Let's make the final FWHM the largest one we found: - fwhm = np.max(fwhms) - image.fwhm = fwhm - logger.info("Final FWHM based on ePSF: %f", fwhm) - - #============================================================================================== - # COORDINATES TO DO PHOTOMETRY AT - #============================================================================================== - - coordinates = np.array([[ref['pixel_column'], ref['pixel_row']] for ref in references]) - - # Add the main target position as the first entry for doing photometry directly in the - # science image: - coordinates = np.concatenate(([target_pixel_pos], coordinates), axis=0) - - #============================================================================================== - # APERTURE PHOTOMETRY - #============================================================================================== - - # Define apertures for aperture photometry: - apertures = CircularAperture(coordinates, r=fwhm) - annuli = CircularAnnulus(coordinates, r_in=1.5*fwhm, r_out=2.5*fwhm) - - apphot_tbl = aperture_photometry(image.subclean, [apertures, annuli], mask=image.mask, error=image.error) - - logger.debug("Aperture Photometry Table:\n%s", apphot_tbl) - logger.info('Apperature Photometry Success') - - #============================================================================================== - # PSF PHOTOMETRY - #============================================================================================== - - # Are we fixing the postions? - epsf.fixed.update({'x_0': False, 'y_0': False}) - - # Create photometry object: - photometry = BasicPSFPhotometry( - group_maker=DAOGroup(fwhm), - bkg_estimator=SExtractorBackground(), - psf_model=epsf, - fitter=fitting.LevMarLSQFitter(), - fitshape=size, - aperture_radius=fwhm - ) - - psfphot_tbl = photometry( - image=image.subclean, - init_guesses=Table(coordinates, names=['x_0', 'y_0']) - ) - - logger.debug("PSF Photometry Table:\n%s", psfphot_tbl) - logger.info('PSF Photometry Success') - - #============================================================================================== - # TEMPLATE SUBTRACTION AND TARGET PHOTOMETRY - #============================================================================================== - - # Find the pixel-scale of the science image: - pixel_area = proj_plane_pixel_area(image.wcs.celestial) - pixel_scale = np.sqrt(pixel_area)*3600 # arcsec/pixel - #print(image.wcs.celestial.cunit) % Doesn't work? - logger.info("Science image pixel scale: %f", pixel_scale) - - diffimage = None - if datafile.get('diffimg') is not None: - - diffimg_path = os.path.join(datafile['archive_path'], datafile['diffimg']['path']) - diffimage = load_image(diffimg_path) - diffimage = diffimage.image - - elif attempt_imagematch and datafile.get('template') is not None: - # Run the template subtraction, and get back - # the science image where the template has been subtracted: - diffimage = run_imagematch(datafile, target, star_coord=coordinates, fwhm=fwhm, pixel_scale=pixel_scale) - - # We have a diff image, so let's do photometry of the target using this: - if diffimage is not None: - # Include mask from original image: - diffimage = np.ma.masked_array(diffimage, image.mask) - - # Create apertures around the target: - apertures = CircularAperture(target_pixel_pos, r=fwhm) - annuli = CircularAnnulus(target_pixel_pos, r_in=1.5*fwhm, r_out=2.5*fwhm) - - # Create two plots of the difference image: - fig, ax = plt.subplots(1, 1, squeeze=True, figsize=(20, 20)) - plot_image(diffimage, ax=ax, cbar='right', title=target_name) - ax.plot(target_pixel_pos[0], target_pixel_pos[1], marker='+', color='r') - fig.savefig(os.path.join(output_folder, 'diffimg.png'), bbox_inches='tight') - apertures.plot(color='r') - annuli.plot(color='k') - ax.set_xlim(target_pixel_pos[0]-50, target_pixel_pos[0]+50) - ax.set_ylim(target_pixel_pos[1]-50, target_pixel_pos[1]+50) - fig.savefig(os.path.join(output_folder, 'diffimg_zoom.png'), bbox_inches='tight') - plt.close(fig) - - # Run aperture photometry on subtracted image: - target_apphot_tbl = aperture_photometry(diffimage, [apertures, annuli], mask=image.mask, error=image.error) - - # Run PSF photometry on template subtracted image: - target_psfphot_tbl = photometry( - diffimage, - init_guesses=Table(target_pixel_pos, names=['x_0', 'y_0']) - ) - - # Combine the output tables from the target and the reference stars into one: - apphot_tbl = vstack([target_apphot_tbl, apphot_tbl], join_type='exact') - psfphot_tbl = vstack([target_psfphot_tbl, psfphot_tbl], join_type='exact') - - # Build results table: - tab = references.copy() - tab.insert_row(0, {'starid': 0, 'ra': target['ra'], 'decl': target['decl'], 'pixel_column': target_pixel_pos[0], 'pixel_row': target_pixel_pos[1]}) - if diffimage is not None: - tab.insert_row(0, {'starid': -1, 'ra': target['ra'], 'decl': target['decl'], 'pixel_column': target_pixel_pos[0], 'pixel_row': target_pixel_pos[1]}) - indx_main_target = (tab['starid'] <= 0) - for key in ('pm_ra', 'pm_dec', 'gaia_mag', 'gaia_bp_mag', 'gaia_rp_mag', 'B_mag', 'V_mag', 'H_mag','J_mag','K_mag', 'u_mag', 'g_mag', 'r_mag', 'i_mag', 'z_mag'): - for i in np.where(indx_main_target)[0]: # No idea why this is needed, but giving a boolean array as slice doesn't work - tab[i][key] = np.NaN - - # Subtract background estimated from annuli: - flux_aperture = apphot_tbl['aperture_sum_0'] - (apphot_tbl['aperture_sum_1'] / annuli.area) * apertures.area - flux_aperture_error = np.sqrt(apphot_tbl['aperture_sum_err_0']**2 + (apphot_tbl['aperture_sum_err_1']/annuli.area * apertures.area)**2) - - # Add table columns with results: - tab['flux_aperture'] = flux_aperture/image.exptime - tab['flux_aperture_error'] = flux_aperture_error/image.exptime - tab['flux_psf'] = psfphot_tbl['flux_fit']/image.exptime - tab['flux_psf_error'] = psfphot_tbl['flux_unc']/image.exptime - tab['pixel_column_psf_fit'] = psfphot_tbl['x_fit'] - tab['pixel_row_psf_fit'] = psfphot_tbl['y_fit'] - tab['pixel_column_psf_fit_error'] = psfphot_tbl['x_0_unc'] - tab['pixel_row_psf_fit_error'] = psfphot_tbl['y_0_unc'] - - # Check that we got valid photometry: - if np.any(~np.isfinite(tab[indx_main_target]['flux_psf'])) or np.any(~np.isfinite(tab[indx_main_target]['flux_psf_error'])): - raise Exception("Target magnitude is undefined.") - - #============================================================================================== - # CALIBRATE - #============================================================================================== - - # Convert PSF fluxes to magnitudes: - mag_inst = -2.5 * np.log10(tab['flux_psf']) - mag_inst_err = (2.5/np.log(10)) * (tab['flux_psf_error'] / tab['flux_psf']) - - # Corresponding magnitudes in catalog: - #TODO: add color terms here - mag_catalog = tab[ref_filter] - - # Mask out things that should not be used in calibration: - use_for_calibration = np.ones_like(mag_catalog, dtype='bool') - use_for_calibration[indx_main_target] = False # Do not use target for calibration - use_for_calibration[~np.isfinite(mag_inst) | ~np.isfinite(mag_catalog)] = False - - # Just creating some short-hands: - x = mag_catalog[use_for_calibration] - y = mag_inst[use_for_calibration] - yerr = mag_inst_err[use_for_calibration] - weights = 1.0/yerr**2 - - # Fit linear function with fixed slope, using sigma-clipping: - model = models.Linear1D(slope=1, fixed={'slope': True}) - fitter = fitting.FittingWithOutlierRemoval(fitting.LinearLSQFitter(), sigma_clip, sigma=3.0) - best_fit, sigma_clipped = fitter(model, x, y, weights=weights) - - # Extract zero-point and estimate its error using a single weighted fit: - # I don't know why there is not an error-estimate attached directly to the Parameter? - zp = -1*best_fit.intercept.value # Negative, because that is the way zeropoints are usually defined - - weights[sigma_clipped] = 0 # Trick to make following expression simpler - N = len(weights.nonzero()[0]) - if N > 1: - zp_error = np.sqrt( N * nansum(weights*(y - best_fit(x))**2) / nansum(weights) / (N-1) ) - else: - zp_error = np.NaN - logger.info('Leastsquare ZP = %.3f, ZP_error = %.3f', zp, zp_error) - - # Determine sigma clipping sigma according to Chauvenet method - # But don't allow less than sigma = sigmamin, setting to 1.5 for now. - # Should maybe be 2? - sigmamin = 1.5 - sigChauv = sigma_from_Chauvenet(len(x)) - sigChauv = sigChauv if sigChauv >= sigmamin else sigmamin - - # Extract zero point and error using bootstrap method - Nboot = 1000 - logger.info('Running bootstrap with sigma = %.2f and n = %d', sigChauv, Nboot) - pars = bootstrap_outlier(x, y, yerr, n=Nboot, model=model, fitter=fitting.LinearLSQFitter, - outlier=sigma_clip, outlier_kwargs={'sigma':sigChauv}, summary='median', - error='bootstrap', return_vals=False) - - zp_bs = pars['intercept'] * -1.0 - zp_error_bs = pars['intercept_error'] - - logger.info('Bootstrapped ZP = %.3f, ZP_error = %.3f', zp_bs, zp_error_bs) - - # Check that difference is not large - zp_diff = 0.4 - if np.abs(zp_bs - zp) >= zp_diff: - logger.warning("Bootstrap and weighted LSQ ZPs differ by %.2f, \ - which is more than the allowed %.2f mag.", np.abs(zp_bs - zp), zp_diff) - - # Add calibrated magnitudes to the photometry table: - tab['mag'] = mag_inst + zp_bs - tab['mag_error'] = np.sqrt(mag_inst_err**2 + zp_error_bs**2) - - fig, ax = plt.subplots(1, 1) - ax.errorbar(x, y, yerr=yerr, fmt='k.') - ax.scatter(x[sigma_clipped], y[sigma_clipped], marker='x', c='r') - ax.plot(x, best_fit(x), color='g', linewidth=3) - ax.set_xlabel('Catalog magnitude') - ax.set_ylabel('Instrumental magnitude') - fig.savefig(os.path.join(output_folder, 'calibration.png'), bbox_inches='tight') - plt.close(fig) - - # Check that we got valid photometry: - if not np.isfinite(tab[0]['mag']) or not np.isfinite(tab[0]['mag_error']): - raise Exception("Target magnitude is undefined.") - - #============================================================================================== - # SAVE PHOTOMETRY - #============================================================================================== - - # Descriptions of columns: - tab['flux_aperture'].unit = u.count/u.second - tab['flux_aperture_error'].unit = u.count/u.second - tab['flux_psf'].unit = u.count/u.second - tab['flux_psf_error'].unit = u.count/u.second - tab['pixel_column'].unit = u.pixel - tab['pixel_row'].unit = u.pixel - tab['pixel_column_psf_fit'].unit = u.pixel - tab['pixel_row_psf_fit'].unit = u.pixel - tab['pixel_column_psf_fit_error'].unit = u.pixel - tab['pixel_row_psf_fit_error'].unit = u.pixel - - # Meta-data: - tab.meta['version'] = __version__ - tab.meta['fileid'] = fileid - tab.meta['template'] = None if datafile.get('template') is None else datafile['template']['fileid'] - tab.meta['diffimg'] = None if datafile.get('diffimg') is None else datafile['diffimg']['fileid'] - tab.meta['photfilter'] = photfilter - tab.meta['fwhm'] = fwhm * u.pixel - tab.meta['pixel_scale'] = pixel_scale * u.arcsec/u.pixel - tab.meta['seeing'] = (fwhm*pixel_scale) * u.arcsec - tab.meta['obstime-bmjd'] = float(image.obstime.mjd) - tab.meta['zp'] = zp_bs - tab.meta['zp_error'] = zp_error_bs - tab.meta['zp_diff'] = np.abs(zp_bs - zp) - tab.meta['zp_error_weights'] = zp_error - - # Filepath where to save photometry: - photometry_output = os.path.join(output_folder, 'photometry.ecsv') - - # Write the final table to file: - tab.write(photometry_output, format='ascii.ecsv', delimiter=',', overwrite=True) - - toc = default_timer() - - logger.info("------------------------------------------------------") - logger.info("Success!") - logger.info("Main target: %f +/- %f", tab[0]['mag'], tab[0]['mag_error']) - logger.info("Photometry took: %f seconds", toc-tic) - - return photometry_output + +# -------------------------------------------------------------------------------------------------- +def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fixed=False): + """ + Run photometry. + + Parameters: + fileid (int): File ID to process. + output_folder (str, optional): Path to directory where output should be placed. + attempt_imagematch (bool, optional): If no subtracted image is available, but a + template image is, should we attempt to run ImageMatch using standard settings. + Default=True. + keep_diff_fixed (bool, optional): Whether to allow psf photometry to recenter when + calculating the flux for the difference image. Setting to true can help if diff + image has non-source flux in the region around the SN. This will also force + using Median background instead of Sextractor for the diff image PSF photometry. + .. codeauthor:: Rasmus Handberg + .. codeauthor:: Emir Karamehmetoglu + .. codeauthor:: Simon Holmbo + """ + + # Settings: + # ref_mag_limit = 22 # Lower limit on reference target brightness + ref_target_dist_limit = 10 * u.arcsec # Reference star must be further than this away to be included + + logger = logging.getLogger(__name__) + tic = default_timer() + + # Use local copy of archive if configured to do so: + config = load_config() + + # Get datafile dict from API: + datafile = api.get_datafile(fileid) + logger.debug("Datafile: %s", datafile) + targetid = datafile['targetid'] + target_name = datafile['target_name'] + photfilter = datafile['photfilter'] + + archive_local = config.get('photometry', 'archive_local', fallback=None) + if archive_local is not None: + datafile['archive_path'] = archive_local + if not os.path.isdir(datafile['archive_path']): + raise FileNotFoundError("ARCHIVE is not available: " + datafile['archive_path']) + + # Get the catalog containing the target and reference stars: + # TODO: Include proper-motion to the time of observation + catalog = api.get_catalog(targetid, output='table') + target = catalog['target'][0] + target_coord = coords.SkyCoord(ra=target['ra'], dec=target['decl'], unit='deg', frame='icrs') + + # Folder to save output: + if output_folder is None: + output_folder_root = config.get('photometry', 'output', fallback='.') + output_folder = os.path.join(output_folder_root, target_name, '%05d' % fileid) + logger.info("Placing output in '%s'", output_folder) + os.makedirs(output_folder, exist_ok=True) + + # The paths to the science image: + filepath = os.path.join(datafile['archive_path'], datafile['path']) + + # TODO: Download datafile using API to local drive: + # TODO: Is this a security concern? + # if archive_local: + # api.download_datafile(datafile, archive_local) + + # Translate photometric filter into table column: + ref_filter = { + 'up': 'u_mag', + 'gp': 'g_mag', + 'rp': 'r_mag', + 'ip': 'i_mag', + 'zp': 'z_mag', + 'B': 'B_mag', + 'V': 'V_mag', + 'J': 'J_mag', + 'H': 'H_mag', + 'K': 'K_mag', + }.get(photfilter, None) + + if ref_filter is None: + logger.warning("Could not find filter '%s' in catalogs. Using default gp filter.", photfilter) + ref_filter = 'g_mag' + + references = catalog['references'] + references.sort(ref_filter) + + # Check that there actually are reference stars in that filter: + if allnan(references[ref_filter]): + raise ValueError("No reference stars found in current photfilter.") + + # Load the image from the FITS file: + image = load_image(filepath) + + # ============================================================================================== + # BARYCENTRIC CORRECTION OF TIME + # ============================================================================================== + + ltt_bary = image.obstime.light_travel_time(target_coord, ephemeris='jpl') + image.obstime = image.obstime.tdb + ltt_bary + + # ============================================================================================== + # BACKGROUND ESTIMATION + # ============================================================================================== + + fig, ax = plt.subplots(1, 2, figsize=(20, 18)) + plot_image(image.clean, ax=ax[0], scale='log', cbar='right', title='Image') + plot_image(image.mask, ax=ax[1], scale='linear', cbar='right', title='Mask') + fig.savefig(os.path.join(output_folder, 'original.png'), bbox_inches='tight') + plt.close(fig) + + # Estimate image background: + # Not using image.clean here, since we are redefining the mask anyway + bkg = Background2D(image.clean, (128, 128), filter_size=(5, 5), + sigma_clip=SigmaClip(sigma=3.0), + bkg_estimator=SExtractorBackground(), + exclude_percentile=50.0) + image.background = bkg.background + image.std = bkg.background_rms_median + + # Create background-subtracted image: + image.subclean = image.clean - image.background + + # Plot background estimation: + fig, ax = plt.subplots(1, 3, figsize=(20, 6)) + plot_image(image.clean, ax=ax[0], scale='log', title='Original') + plot_image(image.background, ax=ax[1], scale='log', title='Background') + plot_image(image.subclean, ax=ax[2], scale='log', title='Background subtracted') + fig.savefig(os.path.join(output_folder, 'background.png'), bbox_inches='tight') + plt.close(fig) + + # TODO: Is this correct?! + image.error = calc_total_error(image.clean, bkg.background_rms, 1.0) + + # Use sep to for soure extraction + image.sepdata = image.image.byteswap().newbyteorder() + image.sepbkg = sep.Background(image.sepdata, mask=image.mask) + image.sepsub = image.sepdata - image.sepbkg + logger.debug('sub: {} bkg_rms: {} mask: {}'.format(np.shape(image.sepsub), np.shape(image.sepbkg.globalrms), + np.shape(image.mask))) + objects = sep.extract(image.sepsub, thresh=5., err=image.sepbkg.globalrms, mask=image.mask, + deblend_cont=0.1, minarea=9, clean_param=2.0) + + # ============================================================================================== + # DETECTION OF STARS AND MATCHING WITH CATALOG + # ============================================================================================== + + # Account for proper motion: + # TODO: Are catalog RA-proper motions including cosdec? + replace(references['pm_ra'], np.NaN, 0) + replace(references['pm_dec'], np.NaN, 0) + refs_coord = coords.SkyCoord(ra=references['ra'], dec=references['decl'], + pm_ra_cosdec=references['pm_ra'], pm_dec=references['pm_dec'], + unit='deg', frame='icrs', obstime=Time(2015.5, format='decimalyear')) + + refs_coord = refs_coord.apply_space_motion(image.obstime) + + # Solve for new WCS + cm = CoordinateMatch( + xy=list(zip(objects['x'], objects['y'])), + rd=list(zip(refs_coord.ra.deg, refs_coord.dec.deg)), + xy_order=np.argsort(-2.5 * np.log10(objects['flux'])), + rd_order=np.argsort(target_coord.separation(refs_coord)), + maximum_angle_distance=0.002, + ) + + try: + i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=100))) + except TimeoutError: + logging.warning('TimeoutError: No new WCS solution found') + except StopIteration: + logging.warning('StopIterationError: No new WCS solution found') + else: + image.wcs = fit_wcs_from_points( + np.array(list(zip(*cm.xy[i_xy]))), + coords.SkyCoord(*map(list, zip(*cm.rd[i_rd])), unit='deg') + ) + + # Calculate pixel-coordinates of references: + row_col_coords = image.wcs.all_world2pix(np.array([[ref.ra.deg, ref.dec.deg] for ref in refs_coord]), 0) + references['pixel_column'] = row_col_coords[:, 0] + references['pixel_row'] = row_col_coords[:, 1] + + # Calculate the targets position in the image: + target_pixel_pos = image.wcs.all_world2pix([[target['ra'], target['decl']]], 0)[0] + + # Clean out the references: + hsize = 10 + x = references['pixel_column'] + y = references['pixel_row'] + clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) + & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) + & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] + # & (references[ref_filter] < ref_mag_limit) + + # @TODO: These need to be based on the instrument! + radius = 10 + fwhm_guess = 6.0 + fwhm_min = 3.5 + fwhm_max = 18.0 + + # Clean extracted stars + masked_sep_xy, sep_mask, masked_sep_rsqs = force_reject_g2d(objects['x'], objects['y'], image, get_fwhm=False, + radius=radius, fwhm_guess=fwhm_guess, rsq_min=0.3, + fwhm_max=fwhm_max, fwhm_min=fwhm_min) + + # Clean reference star locations + masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], + clean_references['pixel_row'], + image, + get_fwhm=True, + radius=radius, + fwhm_guess=fwhm_guess, + fwhm_max=fwhm_max, + fwhm_min=fwhm_min, + rsq_min=0.15) + + # Use R^2 to more robustly determine initial FWHM guess. + # This cleaning is good when we have FEW references. + fwhm, fwhm_clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, + min_fwhm_references=2, min_references=6, rsq_min=0.15) + logging.info('Initial FWHM guess is {} pixels'.format(fwhm)) + image.fwhm = fwhm + + # Create plot of target and reference star positions from 2D Gaussian fits. + fig, ax = plt.subplots(1, 1, figsize=(20, 18)) + plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) + ax.scatter(fwhm_clean_references['pixel_column'], fwhm_clean_references['pixel_row'], c='r', marker='o', alpha=0.3) + ax.scatter(masked_sep_xy[:, 0], masked_sep_xy[:, 1], marker='s', alpha=1.0, edgecolors='green', facecolors='none') + ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') + fig.savefig(os.path.join(output_folder, 'positions_g2d.png'), bbox_inches='tight') + plt.close(fig) + + # Uncomment For Debugging + # return references, clean_references, masked_rsqs, rsq_mask + + # Final clean of wcs corrected references + logger.debug("Number of references before final cleaning: %d", len(clean_references)) + references = get_clean_references(clean_references, masked_rsqs, rsq_ideal=0.8) + logger.debug("Number of references after final cleaning: %d", len(references)) + + # Create plot of target and reference star positions: + fig, ax = plt.subplots(1, 1, figsize=(20, 18)) + plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) + ax.scatter(references['pixel_column'], references['pixel_row'], c='r', marker='o', alpha=0.6) + ax.scatter(masked_sep_xy[:, 0], masked_sep_xy[:, 1], marker='s', alpha=0.6, edgecolors='green', facecolors='none') + ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') + fig.savefig(os.path.join(output_folder, 'positions.png'), bbox_inches='tight') + plt.close(fig) + + # ============================================================================================== + # CREATE EFFECTIVE PSF MODEL + # ============================================================================================== + + # Make cutouts of stars using extract_stars: + # Scales with FWHM + size = int(np.round(29 * fwhm / 6)) + if size % 2 == 0: + size += 1 # Make sure it's a uneven number + size = max(size, 15) # Never go below 15 pixels + hsize = (size + 10) / 2 # higher hsize than before to do more aggressive edge masking. + + x = references['pixel_column'] + y = references['pixel_row'] + mask_near_edge = ((x > hsize) & (x < (image.shape[1] - 1 - hsize)) + & (y > hsize) & (y < (image.shape[0] - 1 - hsize))) + + stars_for_epsf = Table() + stars_for_epsf['x'] = x[mask_near_edge] + stars_for_epsf['y'] = y[mask_near_edge] + + # Store which stars were used in ePSF in the table: + logger.info("Number of stars used for ePSF: %d", len(stars_for_epsf)) + references['used_for_epsf'] = mask_near_edge + + # Extract stars sub-images: + stars = extract_stars( + NDData(data=image.subclean, mask=image.mask), + stars_for_epsf, + size=size + ) + + # Plot the stars being used for ePSF: + nrows = 5 + ncols = 5 + imgnr = 0 + for k in range(int(np.ceil(len(stars_for_epsf) / (nrows * ncols)))): + fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True) + ax = ax.ravel() + for i in range(nrows * ncols): + if imgnr > len(stars_for_epsf) - 1: + ax[i].axis('off') + else: + plot_image(stars[imgnr], ax=ax[i], scale='log', cmap='viridis') + imgnr += 1 + + fig.savefig(os.path.join(output_folder, 'epsf_stars%02d.png' % (k + 1)), bbox_inches='tight') + plt.close(fig) + + # Build the ePSF: + epsf = EPSFBuilder( + oversampling=1.0, + maxiters=500, + fitter=EPSFFitter(fit_boxsize=np.round(2 * fwhm, 0).astype(int)), + progress_bar=True, + recentering_func=centroid_com + )(stars)[0] + + logger.info('Successfully built PSF model') + + fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 15)) + plot_image(epsf.data, ax=ax1, cmap='viridis') + + fwhms = [] + bad_epsf_detected = False + for a, ax in ((0, ax3), (1, ax2)): + # Collapse the PDF along this axis: + profile = epsf.data.sum(axis=a) + itop = profile.argmax() + poffset = profile[itop] / 2 + + # Run a spline through the points, but subtract half of the peak value, and find the roots: + # We have to use a cubic spline, since roots() is not supported for other splines + # for some reason + profile_intp = UnivariateSpline(np.arange(0, len(profile)), profile - poffset, k=3, s=0, ext=3) + lr = profile_intp.roots() + + # Plot the profile and spline: + x_fine = np.linspace(-0.5, len(profile) - 0.5, 500) + ax.plot(profile, 'k.-') + ax.plot(x_fine, profile_intp(x_fine) + poffset, 'g-') + ax.axvline(itop) + ax.set_xlim(-0.5, len(profile) - 0.5) + + # Do some sanity checks on the ePSF: + # It should pass 50% exactly twice and have the maximum inside that region. + # I.e. it should be a single gaussian-like peak + if len(lr) != 2 or itop < lr[0] or itop > lr[1]: + logger.error("Bad PSF along axis %d", a) + bad_epsf_detected = True + else: + axis_fwhm = lr[1] - lr[0] + fwhms.append(axis_fwhm) + ax.axvspan(lr[0], lr[1], facecolor='g', alpha=0.2) + + # Save the ePSF figure: + ax4.axis('off') + fig.savefig(os.path.join(output_folder, 'epsf.png'), bbox_inches='tight') + plt.close(fig) + + # There was a problem with the ePSF: + if bad_epsf_detected: + raise Exception("Bad ePSF detected.") + + # Let's make the final FWHM the largest one we found: + fwhm = np.max(fwhms) + image.fwhm = fwhm + logger.info("Final FWHM based on ePSF: %f", fwhm) + + # ============================================================================================== + # COORDINATES TO DO PHOTOMETRY AT + # ============================================================================================== + + coordinates = np.array([[ref['pixel_column'], ref['pixel_row']] for ref in references]) + + # Add the main target position as the first entry for doing photometry directly in the + # science image: + coordinates = np.concatenate(([target_pixel_pos], coordinates), axis=0) + + # ============================================================================================== + # APERTURE PHOTOMETRY + # ============================================================================================== + + # Define apertures for aperture photometry: + apertures = CircularAperture(coordinates, r=fwhm) + annuli = CircularAnnulus(coordinates, r_in=1.5 * fwhm, r_out=2.5 * fwhm) + + apphot_tbl = aperture_photometry(image.subclean, [apertures, annuli], mask=image.mask, error=image.error) + + logger.debug("Aperture Photometry Table:\n%s", apphot_tbl) + logger.info('Apperature Photometry Success') + + # ============================================================================================== + # PSF PHOTOMETRY + # ============================================================================================== + + # Are we fixing the postions? + epsf.fixed.update({'x_0': False, 'y_0': False}) + + # Create photometry object: + photometry_obj = BasicPSFPhotometry( + group_maker=DAOGroup(fwhm), + bkg_estimator=SExtractorBackground(), + psf_model=epsf, + fitter=fitting.LevMarLSQFitter(), + fitshape=size, + aperture_radius=fwhm + ) + + psfphot_tbl = photometry_obj( + image=image.subclean, + init_guesses=Table(coordinates, names=['x_0', 'y_0']) + ) + + logger.debug("PSF Photometry Table:\n%s", psfphot_tbl) + logger.info('PSF Photometry Success') + + # ============================================================================================== + # TEMPLATE SUBTRACTION AND TARGET PHOTOMETRY + # ============================================================================================== + + # Find the pixel-scale of the science image: + pixel_area = proj_plane_pixel_area(image.wcs.celestial) + pixel_scale = np.sqrt(pixel_area) * 3600 # arcsec/pixel + # print(image.wcs.celestial.cunit) % Doesn't work? + logger.info("Science image pixel scale: %f", pixel_scale) + + diffimage = None + if datafile.get('diffimg') is not None: + + diffimg_path = os.path.join(datafile['archive_path'], datafile['diffimg']['path']) + diffimage = load_image(diffimg_path) + diffimage = diffimage.image + + elif attempt_imagematch and datafile.get('template') is not None: + # Run the template subtraction, and get back + # the science image where the template has been subtracted: + diffimage = run_imagematch(datafile, target, star_coord=coordinates, fwhm=fwhm, pixel_scale=pixel_scale) + + # We have a diff image, so let's do photometry of the target using this: + if diffimage is not None: + # Include mask from original image: + diffimage = np.ma.masked_array(diffimage, image.mask) + + # Create apertures around the target: + apertures = CircularAperture(target_pixel_pos, r=fwhm) + annuli = CircularAnnulus(target_pixel_pos, r_in=1.5 * fwhm, r_out=2.5 * fwhm) + + # Create two plots of the difference image: + fig, ax = plt.subplots(1, 1, squeeze=True, figsize=(20, 20)) + plot_image(diffimage, ax=ax, cbar='right', title=target_name) + ax.plot(target_pixel_pos[0], target_pixel_pos[1], marker='+', color='r') + fig.savefig(os.path.join(output_folder, 'diffimg.png'), bbox_inches='tight') + apertures.plot(color='r') + annuli.plot(color='k') + ax.set_xlim(target_pixel_pos[0] - 50, target_pixel_pos[0] + 50) + ax.set_ylim(target_pixel_pos[1] - 50, target_pixel_pos[1] + 50) + fig.savefig(os.path.join(output_folder, 'diffimg_zoom.png'), bbox_inches='tight') + plt.close(fig) + + # Run aperture photometry on subtracted image: + target_apphot_tbl = aperture_photometry(diffimage, [apertures, annuli], mask=image.mask, error=image.error) + + # Make target only photometry object if keep_diff_fixed = True + if keep_diff_fixed: + epsf.fixed.update({'x_0': True, 'y_0': True}) + + # @TODO: Try iteraratively subtracted photometry + # Create photometry object: + photometry_obj = BasicPSFPhotometry( + group_maker=DAOGroup(0.0001), + bkg_estimator=MedianBackground(), + psf_model=epsf, + fitter=fitting.LevMarLSQFitter(), + fitshape=size, + aperture_radius=fwhm + ) + + # Run PSF photometry on template subtracted image: + target_psfphot_tbl = photometry_obj( + diffimage, + init_guesses=Table(target_pixel_pos, names=['x_0', 'y_0']) + ) + + if keep_diff_fixed: # Need to adjust table columns if x_0 and y_0 were fixed + target_psfphot_tbl['x_0_unc'] = 0.0 + target_psfphot_tbl['y_0_unc'] = 0.0 + + # Combine the output tables from the target and the reference stars into one: + apphot_tbl = vstack([target_apphot_tbl, apphot_tbl], join_type='exact') + psfphot_tbl = vstack([target_psfphot_tbl, psfphot_tbl], join_type='exact') + + # Build results table: + tab = references.copy() + tab.insert_row(0, {'starid': 0, 'ra': target['ra'], 'decl': target['decl'], 'pixel_column': target_pixel_pos[0], + 'pixel_row': target_pixel_pos[1]}) + if diffimage is not None: + tab.insert_row(0, + {'starid': -1, 'ra': target['ra'], 'decl': target['decl'], 'pixel_column': target_pixel_pos[0], + 'pixel_row': target_pixel_pos[1]}) + indx_main_target = (tab['starid'] <= 0) + for key in ( + 'pm_ra', 'pm_dec', 'gaia_mag', 'gaia_bp_mag', 'gaia_rp_mag', 'B_mag', 'V_mag', 'H_mag', 'J_mag', 'K_mag', + 'u_mag', + 'g_mag', 'r_mag', 'i_mag', 'z_mag'): + for i in np.where(indx_main_target)[0]: + # No idea why this is needed, but giving a boolean array as slice doesn't work + tab[i][key] = np.NaN + + # Subtract background estimated from annuli: + flux_aperture = apphot_tbl['aperture_sum_0'] - (apphot_tbl['aperture_sum_1'] / annuli.area) * apertures.area + flux_aperture_error = np.sqrt( + apphot_tbl['aperture_sum_err_0'] ** 2 + (apphot_tbl['aperture_sum_err_1'] / annuli.area * apertures.area) ** 2) + + # Add table columns with results: + tab['flux_aperture'] = flux_aperture / image.exptime + tab['flux_aperture_error'] = flux_aperture_error / image.exptime + tab['flux_psf'] = psfphot_tbl['flux_fit'] / image.exptime + tab['flux_psf_error'] = psfphot_tbl['flux_unc'] / image.exptime + tab['pixel_column_psf_fit'] = psfphot_tbl['x_fit'] + tab['pixel_row_psf_fit'] = psfphot_tbl['y_fit'] + tab['pixel_column_psf_fit_error'] = psfphot_tbl['x_0_unc'] + tab['pixel_row_psf_fit_error'] = psfphot_tbl['y_0_unc'] + + # Check that we got valid photometry: + if np.any(~np.isfinite(tab[indx_main_target]['flux_psf'])) or np.any( + ~np.isfinite(tab[indx_main_target]['flux_psf_error'])): + raise Exception("Target magnitude is undefined.") + + # ============================================================================================== + # CALIBRATE + # ============================================================================================== + + # Convert PSF fluxes to magnitudes: + mag_inst = -2.5 * np.log10(tab['flux_psf']) + mag_inst_err = (2.5 / np.log(10)) * (tab['flux_psf_error'] / tab['flux_psf']) + + # Corresponding magnitudes in catalog: + # TODO: add color terms here + mag_catalog = tab[ref_filter] + + # Mask out things that should not be used in calibration: + use_for_calibration = np.ones_like(mag_catalog, dtype='bool') + use_for_calibration[indx_main_target] = False # Do not use target for calibration + use_for_calibration[~np.isfinite(mag_inst) | ~np.isfinite(mag_catalog)] = False + + # Just creating some short-hands: + x = mag_catalog[use_for_calibration] + y = mag_inst[use_for_calibration] + yerr = mag_inst_err[use_for_calibration] + weights = 1.0 / yerr ** 2 + + # Fit linear function with fixed slope, using sigma-clipping: + model = models.Linear1D(slope=1, fixed={'slope': True}) + fitter = fitting.FittingWithOutlierRemoval(fitting.LinearLSQFitter(), sigma_clip, sigma=3.0) + best_fit, sigma_clipped = fitter(model, x, y, weights=weights) + + # Extract zero-point and estimate its error using a single weighted fit: + # I don't know why there is not an error-estimate attached directly to the Parameter? + zp = -1 * best_fit.intercept.value # Negative, because that is the way zeropoints are usually defined + + weights[sigma_clipped] = 0 # Trick to make following expression simpler + n_weights = len(weights.nonzero()[0]) + if n_weights > 1: + zp_error = np.sqrt(n_weights * nansum(weights * (y - best_fit(x)) ** 2) / nansum(weights) / (n_weights - 1)) + else: + zp_error = np.NaN + logger.info('Leastsquare ZP = %.3f, ZP_error = %.3f', zp, zp_error) + + # Determine sigma clipping sigma according to Chauvenet method + # But don't allow less than sigma = sigmamin, setting to 1.5 for now. + # Should maybe be 2? + sigmamin = 1.5 + sig_chauv = sigma_from_Chauvenet(len(x)) + sig_chauv = sig_chauv if sig_chauv >= sigmamin else sigmamin + + # Extract zero point and error using bootstrap method + nboot = 1000 + logger.info('Running bootstrap with sigma = %.2f and n = %d', sig_chauv, nboot) + pars = bootstrap_outlier(x, y, yerr, n=nboot, model=model, fitter=fitting.LinearLSQFitter, + outlier=sigma_clip, outlier_kwargs={'sigma': sig_chauv}, summary='median', + error='bootstrap', return_vals=False) + + zp_bs = pars['intercept'] * -1.0 + zp_error_bs = pars['intercept_error'] + + logger.info('Bootstrapped ZP = %.3f, ZP_error = %.3f', zp_bs, zp_error_bs) + + # Check that difference is not large + zp_diff = 0.4 + if np.abs(zp_bs - zp) >= zp_diff: + logger.warning("Bootstrap and weighted LSQ ZPs differ by %.2f, \ + which is more than the allowed %.2f mag.", np.abs(zp_bs - zp), zp_diff) + + # Add calibrated magnitudes to the photometry table: + tab['mag'] = mag_inst + zp_bs + tab['mag_error'] = np.sqrt(mag_inst_err ** 2 + zp_error_bs ** 2) + + fig, ax = plt.subplots(1, 1) + ax.errorbar(x, y, yerr=yerr, fmt='k.') + ax.scatter(x[sigma_clipped], y[sigma_clipped], marker='x', c='r') + ax.plot(x, best_fit(x), color='g', linewidth=3) + ax.set_xlabel('Catalog magnitude') + ax.set_ylabel('Instrumental magnitude') + fig.savefig(os.path.join(output_folder, 'calibration.png'), bbox_inches='tight') + plt.close(fig) + + # Check that we got valid photometry: + if not np.isfinite(tab[0]['mag']) or not np.isfinite(tab[0]['mag_error']): + raise Exception("Target magnitude is undefined.") + + # ============================================================================================== + # SAVE PHOTOMETRY + # ============================================================================================== + + # Descriptions of columns: + tab['flux_aperture'].unit = u.count / u.second + tab['flux_aperture_error'].unit = u.count / u.second + tab['flux_psf'].unit = u.count / u.second + tab['flux_psf_error'].unit = u.count / u.second + tab['pixel_column'].unit = u.pixel + tab['pixel_row'].unit = u.pixel + tab['pixel_column_psf_fit'].unit = u.pixel + tab['pixel_row_psf_fit'].unit = u.pixel + tab['pixel_column_psf_fit_error'].unit = u.pixel + tab['pixel_row_psf_fit_error'].unit = u.pixel + + # Meta-data: + tab.meta['version'] = __version__ + tab.meta['fileid'] = fileid + tab.meta['template'] = None if datafile.get('template') is None else datafile['template']['fileid'] + tab.meta['diffimg'] = None if datafile.get('diffimg') is None else datafile['diffimg']['fileid'] + tab.meta['photfilter'] = photfilter + tab.meta['fwhm'] = fwhm * u.pixel + tab.meta['pixel_scale'] = pixel_scale * u.arcsec / u.pixel + tab.meta['seeing'] = (fwhm * pixel_scale) * u.arcsec + tab.meta['obstime-bmjd'] = float(image.obstime.mjd) + tab.meta['zp'] = zp_bs + tab.meta['zp_error'] = zp_error_bs + tab.meta['zp_diff'] = np.abs(zp_bs - zp) + tab.meta['zp_error_weights'] = zp_error + + # Filepath where to save photometry: + photometry_output = os.path.join(output_folder, 'photometry.ecsv') + + # Write the final table to file: + tab.write(photometry_output, format='ascii.ecsv', delimiter=',', overwrite=True) + + toc = default_timer() + + logger.info("------------------------------------------------------") + logger.info("Success!") + logger.info("Main target: %f +/- %f", tab[0]['mag'], tab[0]['mag_error']) + logger.info("Photometry took: %f seconds", toc - tic) + + return photometry_output From ffc790c874b57a3a97f9408cbc95e934f368eced Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 17 Feb 2021 12:31:03 +0100 Subject: [PATCH 14/43] add pandas to requirements @TODO: this dependency can be removed with smarter numpy or astropy table usage. --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 5eaf4aa..2b1350b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ Bottleneck == 1.3.2 matplotlib == 3.3.1 mplcursors == 0.3 seaborn +pandas requests astropy >=4.2 photutils > 1.0 From 67fa7c57ed68c966c7062609a80ecde9773348e1 Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 17 Feb 2021 12:40:43 +0100 Subject: [PATCH 15/43] Add keep_fixed_diff arg --- run_photometry.py | 276 ++++++++++++++++++++++++---------------------- 1 file changed, 143 insertions(+), 133 deletions(-) diff --git a/run_photometry.py b/run_photometry.py index 1667317..3c183f4 100644 --- a/run_photometry.py +++ b/run_photometry.py @@ -13,137 +13,147 @@ import multiprocessing from flows import api, photometry, load_config -#-------------------------------------------------------------------------------------------------- -def process_fileid(fid, output_folder_root=None, attempt_imagematch=True, autoupload=False): - - logger = logging.getLogger('flows') - logging.captureWarnings(True) - logger_warn = logging.getLogger('py.warnings') - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - - datafile = api.get_datafile(fid) - target_name = datafile['target_name'] - - # Folder to save output: - output_folder = os.path.join(output_folder_root, target_name, '%05d' % fid) - - photfile = None - _filehandler = None - try: - # Set the status to indicate that we have started processing: - api.set_photometry_status(fid, 'running') - - # Create the output directory if it doesn't exist: - os.makedirs(output_folder, exist_ok=True) - - # Also write any logging output to the - _filehandler = logging.FileHandler(os.path.join(output_folder, 'photometry.log'), mode='w') - _filehandler.setFormatter(formatter) - _filehandler.setLevel(logging.INFO) - logger.addHandler(_filehandler) - logger_warn.addHandler(_filehandler) - - photfile = photometry(fileid=fid, - output_folder=output_folder, - attempt_imagematch=attempt_imagematch) - - except (SystemExit, KeyboardInterrupt): - logger.error("Aborted by user or system.") - if os.path.exists(output_folder): - shutil.rmtree(output_folder, ignore_errors=True) - photfile = None - api.set_photometry_status(fid, 'abort') - - except: # noqa: E722, pragma: no cover - logger.exception("Photometry failed") - photfile = None - api.set_photometry_status(fid, 'error') - - if _filehandler is not None: - logger.removeHandler(_filehandler) - logger_warn.removeHandler(_filehandler) - - if photfile is not None: - if autoupload: - api.upload_photometry(fid, delete_completed=True) - api.set_photometry_status(fid, 'done') - - return photfile - -#-------------------------------------------------------------------------------------------------- + +# -------------------------------------------------------------------------------------------------- +def process_fileid(fid, output_folder_root=None, attempt_imagematch=True, autoupload=False, keep_diff_fixed=False): + logger = logging.getLogger('flows') + logging.captureWarnings(True) + logger_warn = logging.getLogger('py.warnings') + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + + datafile = api.get_datafile(fid) + target_name = datafile['target_name'] + + # Folder to save output: + output_folder = os.path.join(output_folder_root, target_name, '%05d' % fid) + + photfile = None + _filehandler = None + try: + # Set the status to indicate that we have started processing: + api.set_photometry_status(fid, 'running') + + # Create the output directory if it doesn't exist: + os.makedirs(output_folder, exist_ok=True) + + # Also write any logging output to the + _filehandler = logging.FileHandler(os.path.join(output_folder, 'photometry.log'), mode='w') + _filehandler.setFormatter(formatter) + _filehandler.setLevel(logging.INFO) + logger.addHandler(_filehandler) + logger_warn.addHandler(_filehandler) + + photfile = photometry(fileid=fid, + output_folder=output_folder, + attempt_imagematch=attempt_imagematch, + keep_diff_fixed=keep_diff_fixed) + + except (SystemExit, KeyboardInterrupt): + logger.error("Aborted by user or system.") + if os.path.exists(output_folder): + shutil.rmtree(output_folder, ignore_errors=True) + photfile = None + api.set_photometry_status(fid, 'abort') + + except: # noqa: E722, pragma: no cover + logger.exception("Photometry failed") + photfile = None + api.set_photometry_status(fid, 'error') + + if _filehandler is not None: + logger.removeHandler(_filehandler) + logger_warn.removeHandler(_filehandler) + + if photfile is not None: + if autoupload: + api.upload_photometry(fid, delete_completed=True) + api.set_photometry_status(fid, 'done') + + return photfile + + +# -------------------------------------------------------------------------------------------------- if __name__ == '__main__': - # Parse command line arguments: - parser = argparse.ArgumentParser(description='Run photometry pipeline.') - parser.add_argument('-d', '--debug', help='Print debug messages.', action='store_true') - parser.add_argument('-q', '--quiet', help='Only report warnings and errors.', action='store_true') - parser.add_argument('-o', '--overwrite', help='Overwrite existing results.', action='store_true') - - group = parser.add_argument_group('Selecting which files to process') - group.add_argument('--fileid', help="Process this file ID. Overrides all other filters.", type=int, default=None) - group.add_argument('--targetid', help="Only process files from this target.", type=int, default=None) - group.add_argument('--filter', type=str, default=None, choices=['missing','all','error']) - - group = parser.add_argument_group('Processing details') - group.add_argument('--threads', type=int, default=1, help="Number of parallel threads to use.") - group.add_argument('--no-imagematch', help="Disable ImageMatch.", action='store_true') - group.add_argument('--autoupload', help="Automatically upload completed photometry to Flows website. Only do this, if you know what you are doing!", action='store_true') - args = parser.parse_args() - - # Ensure that all input has been given: - if not args.fileid and not args.targetid and args.filter is None: - parser.error("Please select either a specific FILEID .") - - # Set logging level: - logging_level = logging.INFO - if args.quiet: - logging_level = logging.WARNING - elif args.debug: - logging_level = logging.DEBUG - - # Number of threads to use: - threads = args.threads - if threads <= 0: - threads = multiprocessing.cpu_count() - - # Setup logging: - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - console = logging.StreamHandler() - console.setFormatter(formatter) - logger = logging.getLogger('flows') - if not logger.hasHandlers(): - logger.addHandler(console) - logger.setLevel(logging_level) - - if args.fileid is not None: - # Run the specified fileid: - fileids = [args.fileid] - else: - # Ask the API for a list of fileids which are yet to be processed: - fileids = api.get_datafiles(targetid=args.targetid, filt=args.filter) - - config = load_config() - output_folder_root = config.get('photometry', 'output', fallback='.') - - # Create function wrapper: - process_fileid_wrapper = functools.partial(process_fileid, - output_folder_root=output_folder_root, - attempt_imagematch=not args.no_imagematch, - autoupload=args.autoupload) - - if threads > 1: - # Disable printing info messages from the parent function. - # It is going to be all jumbled up anyway. - #logger.setLevel(logging.WARNING) - - # There is more than one area to process, so let's start - # a process pool and process them in parallel: - with multiprocessing.Pool(threads) as pool: - pool.map(process_fileid_wrapper, fileids) - - else: - # Only single thread so simply run it directly: - for fid in fileids: - print("="*72) - print(fid) - print("="*72) - process_fileid_wrapper(fid) \ No newline at end of file + # Parse command line arguments: + parser = argparse.ArgumentParser(description='Run photometry pipeline.') + parser.add_argument('-d', '--debug', help='Print debug messages.', action='store_true') + parser.add_argument('-q', '--quiet', help='Only report warnings and errors.', action='store_true') + parser.add_argument('-o', '--overwrite', help='Overwrite existing results.', action='store_true') + + group = parser.add_argument_group('Selecting which files to process') + group.add_argument('--fileid', help="Process this file ID. Overrides all other filters.", type=int, default=None) + group.add_argument('--targetid', help="Only process files from this target.", type=int, default=None) + group.add_argument('--filter', type=str, default=None, choices=['missing', 'all', 'error']) + + group = parser.add_argument_group('Processing details') + group.add_argument('--threads', type=int, default=1, help="Number of parallel threads to use.") + group.add_argument('--no-imagematch', help="Disable ImageMatch.", action='store_true') + group.add_argument('--autoupload', + help="Automatically upload completed photometry to Flows website. \ + Only do this, if you know what you are doing!", + action='store_true') + group.add_argument('--fixposdiff', + help="Fix SN position during PSF photometry of difference image. \ + Useful when difference image is noisy.", + action='store_true') + args = parser.parse_args() + + # Ensure that all input has been given: + if not args.fileid and not args.targetid and args.filter is None: + parser.error("Please select either a specific FILEID .") + + # Set logging level: + logging_level = logging.INFO + if args.quiet: + logging_level = logging.WARNING + elif args.debug: + logging_level = logging.DEBUG + + # Number of threads to use: + threads = args.threads + if threads <= 0: + threads = multiprocessing.cpu_count() + + # Setup logging: + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + console = logging.StreamHandler() + console.setFormatter(formatter) + logger = logging.getLogger('flows') + if not logger.hasHandlers(): + logger.addHandler(console) + logger.setLevel(logging_level) + + if args.fileid is not None: + # Run the specified fileid: + fileids = [args.fileid] + else: + # Ask the API for a list of fileids which are yet to be processed: + fileids = api.get_datafiles(targetid=args.targetid, filt=args.filter) + + config = load_config() + output_folder_root = config.get('photometry', 'output', fallback='.') + + # Create function wrapper: + process_fileid_wrapper = functools.partial(process_fileid, + output_folder_root=output_folder_root, + attempt_imagematch=not args.no_imagematch, + autoupload=args.autoupload, + keep_diff_fixed=args.fixposdiff) + + if threads > 1: + # Disable printing info messages from the parent function. + # It is going to be all jumbled up anyway. + # logger.setLevel(logging.WARNING) + + # There is more than one area to process, so let's start + # a process pool and process them in parallel: + with multiprocessing.Pool(threads) as pool: + pool.map(process_fileid_wrapper, fileids) + + else: + # Only single thread so simply run it directly: + for fid in fileids: + print("=" * 72) + print(fid) + print("=" * 72) + process_fileid_wrapper(fid) From 487a1b403b3ec7b58bdfb84dc93024a5f1ce6847 Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 17 Feb 2021 13:27:17 +0100 Subject: [PATCH 16/43] missing return statement --- flows/wcs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flows/wcs.py b/flows/wcs.py index 2fab12c..9fbf5c5 100644 --- a/flows/wcs.py +++ b/flows/wcs.py @@ -242,7 +242,7 @@ def get_clean_references(references, masked_rsqs, min_references_ideal=6, import pandas as pd # @TODO: Convert to pure numpy implementation df = pd.DataFrame(masked_rsqs,columns=['rsq']) nmasked_rsqs = df.sort_values('rsq',ascending=False).dropna().index._data - references[nmasked_rsqs[:keep_max]] + return references[nmasked_rsqs[:keep_max]] # Desperate second try mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) From 8e353b6b2c680e7ead59d84720e498f391a2ce6f Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 17 Feb 2021 13:31:44 +0100 Subject: [PATCH 17/43] log rsq values for debug --- flows/photometry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flows/photometry.py b/flows/photometry.py index b537ac1..489437f 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -268,7 +268,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # This cleaning is good when we have FEW references. fwhm, fwhm_clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, min_fwhm_references=2, min_references=6, rsq_min=0.15) - logging.info('Initial FWHM guess is {} pixels'.format(fwhm)) + logger.info('Initial FWHM guess is {} pixels'.format(fwhm)) image.fwhm = fwhm # Create plot of target and reference star positions from 2D Gaussian fits. @@ -285,6 +285,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Final clean of wcs corrected references logger.debug("Number of references before final cleaning: %d", len(clean_references)) + logger.info('masked R^2 values: {}'.format(masked_rsqs[mask])) references = get_clean_references(clean_references, masked_rsqs, rsq_ideal=0.8) logger.debug("Number of references after final cleaning: %d", len(references)) From b6a51cfc33352ff3ba64d3a16f3b7abe49adcc9e Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 17 Feb 2021 13:34:41 +0100 Subject: [PATCH 18/43] typo --- flows/photometry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flows/photometry.py b/flows/photometry.py index 489437f..d688beb 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -285,7 +285,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Final clean of wcs corrected references logger.debug("Number of references before final cleaning: %d", len(clean_references)) - logger.info('masked R^2 values: {}'.format(masked_rsqs[mask])) + logger.info('masked R^2 values: {}'.format(masked_rsqs[rsq_mask])) references = get_clean_references(clean_references, masked_rsqs, rsq_ideal=0.8) logger.debug("Number of references after final cleaning: %d", len(references)) From 99c42e9056bb396e1f30bcacc686406955886e81 Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 17 Feb 2021 13:39:15 +0100 Subject: [PATCH 19/43] testing wcs.py --- flows/wcs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flows/wcs.py b/flows/wcs.py index 9fbf5c5..4fe6a3b 100644 --- a/flows/wcs.py +++ b/flows/wcs.py @@ -242,6 +242,7 @@ def get_clean_references(references, masked_rsqs, min_references_ideal=6, import pandas as pd # @TODO: Convert to pure numpy implementation df = pd.DataFrame(masked_rsqs,columns=['rsq']) nmasked_rsqs = df.sort_values('rsq',ascending=False).dropna().index._data + print(nmasked_rsqs,df,nmasked_rsqs[:keep_max],references[nmasked_rsqs[:keep_max]]) return references[nmasked_rsqs[:keep_max]] # Desperate second try From 47f39d50669d37592f2deb33dd3fb4e184b3d606 Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 17 Feb 2021 14:38:50 +0100 Subject: [PATCH 20/43] lower timeout error --- flows/photometry.py | 2 +- flows/wcs.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/flows/photometry.py b/flows/photometry.py index d688beb..6b8fc27 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -214,7 +214,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi ) try: - i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=100))) + i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=10))) except TimeoutError: logging.warning('TimeoutError: No new WCS solution found') except StopIteration: diff --git a/flows/wcs.py b/flows/wcs.py index 4fe6a3b..9fbf5c5 100644 --- a/flows/wcs.py +++ b/flows/wcs.py @@ -242,7 +242,6 @@ def get_clean_references(references, masked_rsqs, min_references_ideal=6, import pandas as pd # @TODO: Convert to pure numpy implementation df = pd.DataFrame(masked_rsqs,columns=['rsq']) nmasked_rsqs = df.sort_values('rsq',ascending=False).dropna().index._data - print(nmasked_rsqs,df,nmasked_rsqs[:keep_max],references[nmasked_rsqs[:keep_max]]) return references[nmasked_rsqs[:keep_max]] # Desperate second try From 5a0720fac4fd3f42e6069fd733f765635cb243bb Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 17 Feb 2021 15:30:41 +0100 Subject: [PATCH 21/43] fix masked array --- flows/wcs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/flows/wcs.py b/flows/wcs.py index 9fbf5c5..7ee7d51 100644 --- a/flows/wcs.py +++ b/flows/wcs.py @@ -238,20 +238,21 @@ def get_clean_references(references, masked_rsqs, min_references_ideal=6, if np.sum(np.isfinite(masked_rsqs[mask])) >= min_references_ideal: if len(references[mask]) <= keep_max: return references[mask] - else: + elif len(references[mask]) >= keep_max: import pandas as pd # @TODO: Convert to pure numpy implementation df = pd.DataFrame(masked_rsqs,columns=['rsq']) + masked_rsqs.mask = ~mask nmasked_rsqs = df.sort_values('rsq',ascending=False).dropna().index._data return references[nmasked_rsqs[:keep_max]] # Desperate second try mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) - masked_rsqs.mask = mask + masked_rsqs.mask = ~mask # Switching to pandas for easier selection import pandas as pd # @TODO: Convert to pure numpy implementation df = pd.DataFrame(masked_rsqs,columns=['rsq']) - nmasked_rsqs = df.sort_values('rsq',ascending=False).dropna().index._data + nmasked_rsqs = deepcopy(df.sort_values('rsq',ascending=False).dropna().index._data) nmasked_rsqs = nmasked_rsqs[:min(min_references_ideal, len(nmasked_rsqs))] if len(nmasked_rsqs) >= min_references_abs: return references[nmasked_rsqs] @@ -261,7 +262,7 @@ def get_clean_references(references, masked_rsqs, min_references_ideal=6, # Extremely desperate last ditch attempt i.e. "rescue bad" elif rescue_bad: mask = (masked_rsqs >= 0.02) & (masked_rsqs < 1.0) - masked_rsqs.mask = mask + masked_rsqs.mask = ~mask # Switch to pandas df = pd.DataFrame(masked_rsqs,columns=['rsq']) From b448b8fbd65370e916b17ae8e70c106772b8f639 Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 17 Feb 2021 17:20:20 +0100 Subject: [PATCH 22/43] add force hidpi option --- run_plotlc.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/run_plotlc.py b/run_plotlc.py index d64072b..2490332 100644 --- a/run_plotlc.py +++ b/run_plotlc.py @@ -28,10 +28,11 @@ def main(): Can be either the SN name (e.g. 2019yvr) or the Flows target ID.""") parser.add_argument('--fileid', '-i', type=int, nargs='*', default=None, help='Specific file ids within target separated by spaces: -i ') - parser.add_argument('--filters', '-f', type=str, nargs='*', default=None, choices=all_filters, - help='List of space delimited filters. If not provided will use all') + parser.add_argument('--filters', '-f', type=str, nargs='*', default=None, + help='List of space delimited filters. If not provided will use all. Choose between {}'.format(all_filters)) parser.add_argument('--offset', '-jd', type=float, default=2458800.0) parser.add_argument('--subonly', help='Only show template subtracted data points.', action='store_true') + parser.add_argument('--hidpi', help='double DPI fo 4k resolution', action='store_true') args = parser.parse_args() # To use when only plotting some filters @@ -110,7 +111,8 @@ def main(): # Create the plot: plots_interactive() sns.set(style='ticks') - fig, ax = plt.subplots(figsize=(6.4,4), dpi=130) + dpi_mult = 1 if not args.subonly else 2 + fig, ax = plt.subplots(figsize=(6.4,4), dpi=130*dpi_mult) fig.subplots_adjust(top=0.95, left=0.1, bottom=0.1, right=0.97) cps = sns.color_palette() From 511b38064c21c49e7e1cb702551072a367c92481 Mon Sep 17 00:00:00 2001 From: Emir Date: Thu, 18 Feb 2021 13:12:03 +0100 Subject: [PATCH 23/43] Timeout Time as parameter --- flows/photometry.py | 4 ++-- run_photometry.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/flows/photometry.py b/flows/photometry.py index 6b8fc27..016ab0a 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -50,7 +50,7 @@ # -------------------------------------------------------------------------------------------------- -def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fixed=False): +def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fixed=False, timeoutpar=10): """ Run photometry. @@ -214,7 +214,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi ) try: - i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=10))) + i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=timeoutpar))) except TimeoutError: logging.warning('TimeoutError: No new WCS solution found') except StopIteration: diff --git a/run_photometry.py b/run_photometry.py index 3c183f4..298efe5 100644 --- a/run_photometry.py +++ b/run_photometry.py @@ -15,7 +15,8 @@ # -------------------------------------------------------------------------------------------------- -def process_fileid(fid, output_folder_root=None, attempt_imagematch=True, autoupload=False, keep_diff_fixed=False): +def process_fileid(fid, output_folder_root=None, attempt_imagematch=True, autoupload=False, keep_diff_fixed=False, + timeoutpar=10): logger = logging.getLogger('flows') logging.captureWarnings(True) logger_warn = logging.getLogger('py.warnings') @@ -46,7 +47,8 @@ def process_fileid(fid, output_folder_root=None, attempt_imagematch=True, autoup photfile = photometry(fileid=fid, output_folder=output_folder, attempt_imagematch=attempt_imagematch, - keep_diff_fixed=keep_diff_fixed) + keep_diff_fixed=keep_diff_fixed, + timeoutpar=timeoutpar) except (SystemExit, KeyboardInterrupt): logger.error("Aborted by user or system.") From 52b9479e23844ca37968433b246a88e39f6d76e2 Mon Sep 17 00:00:00 2001 From: Emir Date: Thu, 18 Feb 2021 13:14:50 +0100 Subject: [PATCH 24/43] add timeout par as command line arg --- run_photometry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/run_photometry.py b/run_photometry.py index 298efe5..25b35b8 100644 --- a/run_photometry.py +++ b/run_photometry.py @@ -98,6 +98,7 @@ def process_fileid(fid, output_folder_root=None, attempt_imagematch=True, autoup help="Fix SN position during PSF photometry of difference image. \ Useful when difference image is noisy.", action='store_true') + group.add_argument('--timeoutpar', type=int, default=10, help='Timeout in Seconds for WCS') args = parser.parse_args() # Ensure that all input has been given: From 5c79ebec49c0fed9ab3acdce0c6cd627ae5103be Mon Sep 17 00:00:00 2001 From: Emir Date: Thu, 18 Feb 2021 13:17:23 +0100 Subject: [PATCH 25/43] timeout par bug --- run_photometry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run_photometry.py b/run_photometry.py index 25b35b8..b3c66f0 100644 --- a/run_photometry.py +++ b/run_photometry.py @@ -141,7 +141,8 @@ def process_fileid(fid, output_folder_root=None, attempt_imagematch=True, autoup output_folder_root=output_folder_root, attempt_imagematch=not args.no_imagematch, autoupload=args.autoupload, - keep_diff_fixed=args.fixposdiff) + keep_diff_fixed=args.fixposdiff, + timeoutpar=args.timeoutpar) if threads > 1: # Disable printing info messages from the parent function. From 332af3c559f53f21e92264ee2f9bf2346db70596 Mon Sep 17 00:00:00 2001 From: Emir Date: Thu, 18 Feb 2021 13:29:22 +0100 Subject: [PATCH 26/43] Testing impact of changes --- flows/photometry.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flows/photometry.py b/flows/photometry.py index 016ab0a..1b567e7 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -284,10 +284,10 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # return references, clean_references, masked_rsqs, rsq_mask # Final clean of wcs corrected references - logger.debug("Number of references before final cleaning: %d", len(clean_references)) + logger.info("Number of references before final cleaning: %d", len(clean_references)) logger.info('masked R^2 values: {}'.format(masked_rsqs[rsq_mask])) references = get_clean_references(clean_references, masked_rsqs, rsq_ideal=0.8) - logger.debug("Number of references after final cleaning: %d", len(references)) + logger.info("Number of references after final cleaning: %d", len(references)) # Create plot of target and reference star positions: fig, ax = plt.subplots(1, 1, figsize=(20, 18)) @@ -308,7 +308,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi if size % 2 == 0: size += 1 # Make sure it's a uneven number size = max(size, 15) # Never go below 15 pixels - hsize = (size + 10) / 2 # higher hsize than before to do more aggressive edge masking. + hsize = (size - 1) / 2 # higher hsize than before to do more aggressive edge masking. x = references['pixel_column'] y = references['pixel_row'] From a7ec43fd87b69f914aa88e0f55663572a9760f5b Mon Sep 17 00:00:00 2001 From: Emir Date: Thu, 18 Feb 2021 13:35:16 +0100 Subject: [PATCH 27/43] ? --- flows/photometry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flows/photometry.py b/flows/photometry.py index 1b567e7..1ff6918 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -351,7 +351,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi epsf = EPSFBuilder( oversampling=1.0, maxiters=500, - fitter=EPSFFitter(fit_boxsize=np.round(2 * fwhm, 0).astype(int)), + fitter=EPSFFitter(fit_boxsize=np.round(2 * fwhm, 0)), progress_bar=True, recentering_func=centroid_com )(stars)[0] From 1aff965bc6da326fd34838a088f8e9e1eb06a67e Mon Sep 17 00:00:00 2001 From: Simon Holmbo Date: Thu, 18 Feb 2021 14:43:57 +0100 Subject: [PATCH 28/43] FITS commands and local webserver --- flows/fitscmd.py | 94 +++++++++++++++++++++++++++++++++++ flows/fitting.py | 12 +++++ flows/load_image.py | 4 ++ flows/photometry.py | 56 +++++++++++++-------- requirements.txt | 1 + run_fitscmd.py | 91 ++++++++++++++++++++++++++++++++++ run_localweb.py | 118 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 356 insertions(+), 20 deletions(-) create mode 100644 flows/fitscmd.py create mode 100644 flows/fitting.py create mode 100644 run_fitscmd.py create mode 100644 run_localweb.py diff --git a/flows/fitscmd.py b/flows/fitscmd.py new file mode 100644 index 0000000..c0d04a5 --- /dev/null +++ b/flows/fitscmd.py @@ -0,0 +1,94 @@ +import re, logging + +import numpy as np +import astropy.units as u + +from astropy.table import Table + +COMMANDS = { + 'maskstar': ( # mask star for galaxy subtracted psf photometry + '(\d+(?:\.\d*)?),\s?([+-]?\d+(?:\.\d*)?)', # parse + lambda ra, dec: (float(ra), float(dec)), # convert + lambda ra, dec, hdul: 0 <= ra < 360 and -90 <= dec <= 90 # validate + ), + 'localseq': ( # specifiy hdu name with custom local sequence + '(.+)', # parse + lambda lsqhdu: (lsqhdu.upper(),), # convert + lambda lsqhdu, hdul: lsqhdu in [hdu.name for hdu in hdul], # validate + ), + 'colorterm': ( # color term for moving references to another system + '([+-]?\d+(?:\.\d*)?)\s?\((.+)-(.+)\)', # parse + lambda cterm, A_mag, B_mag: (float(cterm), A_mag, B_mag), # convert + lambda cterm, A_mag, B_mag, hdul: True, # validate + ) +} + +def maskstar(data, wcs, stars, fwhm): + + if not stars: + return + + data = data.copy() + if not hasattr(data, 'mask'): + data = np.ma.array(data, mask=np.zeros_like(data)) + + X, Y = np.meshgrid(*map(np.arange, data.shape[::-1])) + + for x, y in wcs.all_world2pix(stars, 0): + i = np.where(((X-x)**2 + (Y-y)**2 < fwhm**2)) + data.mask[i] = True + + return data + +def localseq(lsqhdus, hdul): + + if not lsqhdus: + return + + hdu = hdul[[hdu.name for hdu in hdul].index(lsqhdus[-1][0])] + + references = Table(hdu.data) + n = len(references) + + references.add_column(np.arange(n) + 1, name='starid', index=0) + references['pm_ra'] = np.zeros(n) * u.deg / u.yr + references['pm_dec'] = np.zeros(n) * u.deg / u.yr + + references.meta['localseq'] = hdu.name + + return references + +def colorterm(ref_filter, colorterms, references): + + if not colorterms: + return + + colorterm, A_mag, B_mag = colorterms[-1] + + if not {A_mag, B_mag} <= set(references.colnames): + missing = ', '.join({A_mag, B_mag} - set(references.colnames)) + logging.warning('%s not in references', missing) + return + + references[ref_filter] += colorterm * (references[A_mag] - references[B_mag]) + + return references + +def get_fitscmd(image, command): + + if not command in COMMANDS: + logging.warning('fitscmd %s unknown', command) + return + + if not '' in image.head: + return + + parameters = [ + c[13+len(command):] for c in image.head[''] + if c.startswith('FLOWSCMD: %s' % command) + ] + + return [ + COMMANDS[command][1](*re.match(COMMANDS[command][0], p).groups()) + for p in parameters + ] diff --git a/flows/fitting.py b/flows/fitting.py new file mode 100644 index 0000000..de30eeb --- /dev/null +++ b/flows/fitting.py @@ -0,0 +1,12 @@ +import numpy as np + +from astropy.modeling.fitting import LevMarLSQFitter + +class MaskableLevMarLSQFitter(LevMarLSQFitter): + + def __call__(self, model, x, y, z=None, *args, **kwargs): + + if hasattr(z, 'mask'): + x, y, z = x[~z.mask], y[~z.mask], z[~z.mask] + + return super().__call__(model, x, y, z, *args, **kwargs) diff --git a/flows/load_image.py b/flows/load_image.py index 9ca1169..8e6e0d4 100644 --- a/flows/load_image.py +++ b/flows/load_image.py @@ -76,6 +76,7 @@ def load_image(FILENAME): # get image and wcs solution with fits.open(FILENAME, mode='readonly') as hdul: + hdr = hdul[0].header origin = hdr.get('ORIGIN') telescope = hdr.get('TELESCOP') @@ -84,6 +85,9 @@ def load_image(FILENAME): image.image = np.asarray(hdul[0].data) image.shape = image.image.shape + image.head = hdr + image.exthdu = [hdu.copy() for hdu in hdul[1:]] + if origin == 'LCOGT': image.mask = np.asarray(hdul['BPM'].data, dtype='bool') else: diff --git a/flows/photometry.py b/flows/photometry.py index 6dcb7d2..adbbef0 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -45,6 +45,8 @@ from .wcs import force_reject_g2d, mkposxy, clean_with_rsq_and_get_fwhm, \ try_astroalign, kdtree, get_new_wcs, get_clean_references from .coordinatematch import CoordinateMatch +from .fitting import MaskableLevMarLSQFitter +from .fitscmd import get_fitscmd, maskstar, localseq, colorterm __version__ = get_version(pep440=False) @@ -127,16 +129,21 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): logger.warning("Could not find filter '%s' in catalogs. Using default gp filter.", photfilter) ref_filter = 'g_mag' - references = catalog['references'] + # Load the image from the FITS file: + image = load_image(filepath) + + lsqhdus = get_fitscmd(image, 'localseq') # look for local sequence in fits table + references = catalog['references'] if not lsqhdus else localseq(lsqhdus, image.exthdu) + + colorterms = get_fitscmd(image, 'colorterm') + references = colorterm(ref_filter, colorterms, references) if colorterms else references + references.sort(ref_filter) # Check that there actually are reference stars in that filter: if allnan(references[ref_filter]): raise ValueError("No reference stars found in current photfilter.") - # Load the image from the FITS file: - image = load_image(filepath) - #============================================================================================== # BARYCENTRIC CORRECTION OF TIME #============================================================================================== @@ -249,10 +256,12 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): hsize = 10 x = references['pixel_column'] y = references['pixel_row'] + clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] # & (references[ref_filter] < ref_mag_limit) + assert len(clean_references), 'No references in field' logger.info("References:\n%s", references) @@ -262,7 +271,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): fwhm_min = 3.5 fwhm_max = 18.0 - # Clean extracted stars + # Clean extracted stars (FIXME is this one necessary? it's only being used for plots) masked_sep_xy,sep_mask,masked_sep_rsqs = force_reject_g2d(objects['x'], objects['y'], image, get_fwhm=False, radius=radius, fwhm_guess=fwhm_guess, rsq_min=0.3, fwhm_max=fwhm_max, fwhm_min=fwhm_min) @@ -277,7 +286,6 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): fwhm_min=fwhm_min, rsq_min=0.15) - # Use R^2 to more robustly determine initial FWHM guess. # This cleaning is good when we have FEW references. fwhm, fwhm_clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, @@ -599,7 +607,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): group_maker=DAOGroup(fwhm), bkg_estimator=SExtractorBackground(), psf_model=epsf, - fitter=fitting.LevMarLSQFitter(), + fitter=MaskableLevMarLSQFitter(), fitshape=size, aperture_radius=fwhm ) @@ -626,8 +634,8 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): if datafile.get('diffimg') is not None: diffimg_path = os.path.join(datafile['archive_path'], datafile['diffimg']['path']) - diffimage = load_image(diffimg_path) - diffimage = diffimage.image + diffimg = load_image(diffimg_path) + diffimage = diffimg.image elif attempt_imagematch and datafile.get('template') is not None: # Run the template subtraction, and get back @@ -658,11 +666,15 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # Run aperture photometry on subtracted image: target_apphot_tbl = aperture_photometry(diffimage, [apertures, annuli], mask=image.mask, error=image.error) - # Run PSF photometry on template subtracted image: + # Mask stars from FITS header + stars = get_fitscmd(diffimg, 'maskstar') + masked_diffimage = maskstar(diffimage, image.wcs, stars, image.fwhm) + + # Run PSF photometry on template subtracted image: target_psfphot_tbl = photometry( - diffimage, + diffimage if masked_diffimage is None else masked_diffimage, init_guesses=Table(target_pixel_pos, names=['x_0', 'y_0']) - ) + ) # Combine the output tables from the target and the reference stars into one: apphot_tbl = vstack([target_apphot_tbl, apphot_tbl], join_type='exact') @@ -670,13 +682,17 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # Build results table: tab = references.copy() - tab.insert_row(0, {'starid': 0, 'ra': target['ra'], 'decl': target['decl'], 'pixel_column': target_pixel_pos[0], 'pixel_row': target_pixel_pos[1]}) + extkeys = {'pm_ra', 'pm_dec', 'gaia_mag', 'gaia_bp_mag', 'gaia_rp_mag', 'B_mag', 'V_mag', 'H_mag','J_mag','K_mag', 'u_mag', 'g_mag', 'r_mag', 'i_mag', 'z_mag'} + + row = {'starid': 0, 'ra': target['ra'], 'decl': target['decl'], 'pixel_column': target_pixel_pos[0], 'pixel_row': target_pixel_pos[1]} + row.update([(k, np.NaN) for k in extkeys & set(tab.keys())]) + tab.insert_row(0, row) + if diffimage is not None: - tab.insert_row(0, {'starid': -1, 'ra': target['ra'], 'decl': target['decl'], 'pixel_column': target_pixel_pos[0], 'pixel_row': target_pixel_pos[1]}) - indx_main_target = (tab['starid'] <= 0) - for key in ('pm_ra', 'pm_dec', 'gaia_mag', 'gaia_bp_mag', 'gaia_rp_mag', 'B_mag', 'V_mag', 'H_mag','J_mag','K_mag', 'u_mag', 'g_mag', 'r_mag', 'i_mag', 'z_mag'): - for i in np.where(indx_main_target)[0]: # No idea why this is needed, but giving a boolean array as slice doesn't work - tab[i][key] = np.NaN + row['starid'] = -1 + tab.insert_row(0, row) + + indx_target = tab['starid'] <= 0 # Subtract background estimated from annuli: flux_aperture = apphot_tbl['aperture_sum_0'] - (apphot_tbl['aperture_sum_1'] / annuli.area) * apertures.area @@ -693,7 +709,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): tab['pixel_row_psf_fit_error'] = psfphot_tbl['y_0_unc'] # Check that we got valid photometry: - if np.any(~np.isfinite(tab[indx_main_target]['flux_psf'])) or np.any(~np.isfinite(tab[indx_main_target]['flux_psf_error'])): + if np.any(~np.isfinite(tab[indx_target]['flux_psf'])) or np.any(~np.isfinite(tab[indx_target]['flux_psf_error'])): raise Exception("Target magnitude is undefined.") #============================================================================================== @@ -710,7 +726,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True): # Mask out things that should not be used in calibration: use_for_calibration = np.ones_like(mag_catalog, dtype='bool') - use_for_calibration[indx_main_target] = False # Do not use target for calibration + use_for_calibration[indx_target] = False # Do not use target for calibration use_for_calibration[~np.isfinite(mag_inst) | ~np.isfinite(mag_catalog)] = False # Just creating some short-hands: diff --git a/requirements.txt b/requirements.txt index 5eaf4aa..b8025b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,4 @@ pandas sep astroalign > 2.3 networkx +flask diff --git a/run_fitscmd.py b/run_fitscmd.py new file mode 100644 index 0000000..65fd0fc --- /dev/null +++ b/run_fitscmd.py @@ -0,0 +1,91 @@ +import os, sys, re +import argparse, logging + +from astropy.io import fits + +from flows.fitscmd import COMMANDS + +def _add(): + + parser = argparse.ArgumentParser(description='Add FITS specific command.') + parser.add_argument('command', choices=list(COMMANDS.keys()), help='command') + parser.add_argument('parameters', help='command parameters') + parser.add_argument('fitsfiles', nargs='+', help='FITS files') + args = parser.parse_args() + + try: + parameters = COMMANDS[args.command][1](*re.match(COMMANDS[args.command][0], args.parameters).groups()) + except: + logging.critical('can not parse %s', args.parameters) + quit(1) + + for f in args.fitsfiles: + try: + with fits.open(f, mode='update') as hdul: + if not COMMANDS[args.command][2](*parameters, hdul): + logging.error('%s %s is not valid for %s', args.command, args.parameters, f) + continue + hdul[0].header[''] = 'FLOWSCMD: %s = %s' % (args.command, args.parameters) + except Exception as e: + logging.error('could not open %s', f) + continue + +def _get(): + + parser = argparse.ArgumentParser(description='Get FITS specific commands.') + parser.add_argument('fitsfile', help='FITS file') + args = parser.parse_args() + + with fits.open(args.fitsfile) as hdul: + + if not '' in hdul[0].header: + return + + commands = [(i, c[10:]) for i, c in enumerate(hdul[0].header['']) if c.startswith('FLOWSCMD: ')] + for i, c in commands: + print('%%%dd) %%s' % (len(str(len(commands))),) % (i, c)) + +def _del(): + + parser = argparse.ArgumentParser(description='Delete FITS specific command.') + parser.add_argument('command', type=int, help='Command ID') + parser.add_argument('fitsfile', help='FITS file') + args = parser.parse_args() + + with fits.open(args.fitsfile, mode='update') as hdul: + + if not '' in hdul[0].header: + logging.critical('no commands in fits') + quit(1) + + commands = [i for i, c in enumerate(hdul[0].header['']) if c.startswith('FLOWSCMD: ')] + if not args.command in commands: + logging.critical('command id does not exist') + quit(1) + + head = list(hdul[0].header['']) + head.pop(args.command) + del hdul[0].header[''] + + for l in head: + hdul[0].header[''] = l + +if __name__ == '__main__': + + if len(sys.argv) > 1: + + method = sys.argv.pop(1) + + if not { + 'add' : _add, + 'get' : _get, + 'del' : _del, + }.get(method, lambda *a: 1)(): + + quit(0) + + sys.argv.insert(1, method) + + parser = argparse.ArgumentParser(description='Control FITS specific commands.') + parser.add_argument('method', choices=('add', 'get', 'del'), help='method') + args = parser.parse_args() diff --git a/run_localweb.py b/run_localweb.py new file mode 100644 index 0000000..fe02c47 --- /dev/null +++ b/run_localweb.py @@ -0,0 +1,118 @@ +import os, io + +import numpy as np +import matplotlib.pyplot as plt + +from flask import Flask, session; session = dict() # XXX +from base64 import b64encode as b64 +from astropy.io import ascii + +from flows import api, load_config + +app = Flask(__name__) +#app.secret_key = os.urandom(24) + +@app.route('/') +def index(): + s = str() + target_by_name = {target['target_name']: target for target in api.get_targets()} + local_targets = os.listdir(load_config()['photometry']['archive_local']) + for target_name in sorted(target_by_name): + targetid = target_by_name[target_name]['targetid'] + s += f'{target_name} \n' if target_name in local_targets else f'{target_name} \n' + return s + +def lightcurve(targetid): + data = dict() + target = {target['targetid']: target for target in api.get_targets()}[targetid] + output = load_config()['photometry']['output'] + if not os.path.exists(output + '/%s' % (target['target_name'])): return '' + for fileid in os.listdir(output + '/%s' % (target['target_name'])): + f = output + '/%s/%s/photometry.ecsv' % (target['target_name'], fileid) + if not os.path.exists(f): continue + table = ascii.read(f) + filt, mjd = table.meta['photfilter'], table.meta['obstime-bmjd'] + if not filt in data: data[filt] = [] + mag, err, starid = table[0]['mag'], table[0]['mag_error'], table[0]['starid'] + data[filt].append((fileid, mjd, mag, err, starid)) + plt.figure(figsize=(20, 10)) + for filt in data: + fileid, mjd, mag, err, starid = map(np.array, zip(*data[filt])) + _ = plt.errorbar([], [], yerr=[], ls='', marker='o', label=filt) + plt.errorbar(mjd[starid==-1], mag[starid==-1], yerr=err[starid==-1], ls='', marker='o', color=_.lines[0].get_color()) + plt.errorbar(mjd[starid==0], mag[starid==0], yerr=err[starid==0], ls='', marker='s', color=_.lines[0].get_color()) + for i in range(len(fileid)): plt.text(mjd[i], mag[i], str(int(fileid[i])), fontsize=8) + plt.title(target['target_name']) + plt.xlabel('MJD'); plt.ylabel('Magnitude') + plt.legend() + plt.tight_layout() + ymin, ymax = min([min(list(zip(*d))[2]) for d in data.values()]), max([max(list(zip(*d))[2]) for d in data.values()]) + plt.ylim(ymin - (ymax-ymin)*0.05, ymax + (ymax-ymin)*0.05) + plt.gca().invert_yaxis() + png = io.BytesIO() + plt.savefig(png, format='png', dpi=100) + plt.close() + return b64(png.getvalue()).decode('utf-8') + +@app.route('/') +def target(targetid): + s = ''.format(lightcurve(targetid)) + target = {target['targetid']: target for target in api.get_targets()}[targetid] + archive = os.listdir(load_config()['photometry']['archive_local'] + '/%s' % target['target_name']) + output = load_config()['photometry']['output'] + fileids = list(map(str, api.get_datafiles(targetid, 'all'))) # XXX + for fileid in set(fileids) - set(session.keys()): session[fileid] = api.get_datafile(int(fileid)) + s += '' + for fileid in fileids: + s += ''.format(**session[fileid]) + if session[fileid]['path'].rsplit('/',1)[1] in archive: + if not session[fileid]['path'].rsplit('/',1)[1][:-8] + '.png' in archive: s += '' + else: s += f'' + else: s += '' + if not os.path.exists(output + '/%s/%0.5d/photometry.log' % (target['target_name'], int(fileid))): s += '' + else: s += f'' + if not os.path.exists(output + '/%s/%0.5d/photometry.ecsv' % (target['target_name'], int(fileid))): s += '' + else: s += f'' + s += '' + s += '
{fileid}{path}{obstime}{photfilter}IMGIMGLOGLOGPHOTPHOT
' + return s + +@app.route('///img') +def image(targetid, fileid): + s = str() + target = {target['targetid']: target for target in api.get_targets()}[targetid] + archive = load_config()['photometry']['archive_local'] + output = load_config()['photometry']['output'] + img = '%s/%s.png' % (archive, session[str(fileid)]['path'][:-8]) + with open(img, 'rb') as fd: img = b64(fd.read()).decode('utf-8') + s += f'' + if not os.path.exists(output + '/%s/%0.5d' % (target['target_name'], fileid)): return s + for f in sorted(os.listdir(output + '/%s/%0.5d' % (target['target_name'], fileid)))[::-1]: + if not '.' in f or not f.rsplit('.', 1)[1] == 'png': continue + img = output + '/%s/%0.5d/%s' % (target['target_name'], fileid, f) + with open(img, 'rb') as fd: img = b64(fd.read()).decode('utf-8') + s += f'' + return s + +@app.route('///phot') +def photometry(targetid, fileid): + target = {target['targetid']: target for target in api.get_targets()}[targetid] + output = load_config()['photometry']['output'] + phot = output + '/%s/%0.5d/photometry.ecsv' % (target['target_name'], fileid) + table = ascii.read(phot) + s = str(table.meta) + '

' + s += '' + for row in table: s += '' + s += '
' + ''.join(table.colnames) + '
' + ''.join(map(str, row)) + '
' + return s + + +@app.route('///log') +def log(targetid, fileid): + target = {target['targetid']: target for target in api.get_targets()}[targetid] + output = load_config()['photometry']['output'] + log = output + '/%s/%0.5d/photometry.log' % (target['target_name'], fileid) + with open(log, 'r') as fd: return fd.read().replace('\n', '
') + +if __name__ == '__main__': + app.run(debug=True) From 5d9ef2a23f4aa5b14333fd45e7c08593253854eb Mon Sep 17 00:00:00 2001 From: Simon Holmbo Date: Thu, 18 Feb 2021 15:12:18 +0100 Subject: [PATCH 29/43] Tabs to spaces in photometry.py --- flows/photometry.py | 1554 +++++++++++++++++++++---------------------- 1 file changed, 777 insertions(+), 777 deletions(-) diff --git a/flows/photometry.py b/flows/photometry.py index adbbef0..a797f9b 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -43,7 +43,7 @@ from .run_imagematch import run_imagematch from .zeropoint import bootstrap_outlier, sigma_from_Chauvenet from .wcs import force_reject_g2d, mkposxy, clean_with_rsq_and_get_fwhm, \ - try_astroalign, kdtree, get_new_wcs, get_clean_references + try_astroalign, kdtree, get_new_wcs, get_clean_references from .coordinatematch import CoordinateMatch from .fitting import MaskableLevMarLSQFitter from .fitscmd import get_fitscmd, maskstar, localseq, colorterm @@ -54,788 +54,788 @@ #-------------------------------------------------------------------------------------------------- def photometry(fileid, output_folder=None, attempt_imagematch=True): - """ - Run photometry. - - Parameters: - fileid (int): File ID to process. - output_folder (str, optional): Path to directory where output should be placed. - attempt_imagematch (bool, optional): If no subtracted image is available, but a - template image is, should we attempt to run ImageMatch using standard settings. - Default=True. - - .. codeauthor:: Rasmus Handberg - """ - - # Settings: - #ref_mag_limit = 22 # Lower limit on reference target brightness - ref_target_dist_limit = 10 * u.arcsec # Reference star must be further than this away to be included - - logger = logging.getLogger(__name__) - tic = default_timer() - - # Use local copy of archive if configured to do so: - config = load_config() - - # Get datafile dict from API: - datafile = api.get_datafile(fileid) - logger.debug("Datafile: %s", datafile) - targetid = datafile['targetid'] - target_name = datafile['target_name'] - photfilter = datafile['photfilter'] - - archive_local = config.get('photometry', 'archive_local', fallback=None) - if archive_local is not None: - datafile['archive_path'] = archive_local - if not os.path.isdir(datafile['archive_path']): - raise FileNotFoundError("ARCHIVE is not available: " + datafile['archive_path']) - - # Get the catalog containing the target and reference stars: - # TODO: Include proper-motion to the time of observation - catalog = api.get_catalog(targetid, output='table') - target = catalog['target'][0] - target_coord = coords.SkyCoord(ra=target['ra'], dec=target['decl'], unit='deg', frame='icrs') - - # Folder to save output: - if output_folder is None: - output_folder_root = config.get('photometry', 'output', fallback='.') - output_folder = os.path.join(output_folder_root, target_name, '%05d' % fileid) - logger.info("Placing output in '%s'", output_folder) - os.makedirs(output_folder, exist_ok=True) - - # The paths to the science image: - filepath = os.path.join(datafile['archive_path'], datafile['path']) - - # TODO: Download datafile using API to local drive: - # TODO: Is this a security concern? - #if archive_local: - # api.download_datafile(datafile, archive_local) - - # Translate photometric filter into table column: - ref_filter = { - 'up': 'u_mag', - 'gp': 'g_mag', - 'rp': 'r_mag', - 'ip': 'i_mag', - 'zp': 'z_mag', - 'B': 'B_mag', - 'V': 'V_mag', - 'J': 'J_mag', - 'H': 'H_mag', - 'K': 'K_mag', - }.get(photfilter, None) - - if ref_filter is None: - logger.warning("Could not find filter '%s' in catalogs. Using default gp filter.", photfilter) - ref_filter = 'g_mag' - - # Load the image from the FITS file: - image = load_image(filepath) - - lsqhdus = get_fitscmd(image, 'localseq') # look for local sequence in fits table - references = catalog['references'] if not lsqhdus else localseq(lsqhdus, image.exthdu) - - colorterms = get_fitscmd(image, 'colorterm') - references = colorterm(ref_filter, colorterms, references) if colorterms else references - - references.sort(ref_filter) - - # Check that there actually are reference stars in that filter: - if allnan(references[ref_filter]): - raise ValueError("No reference stars found in current photfilter.") - - #============================================================================================== - # BARYCENTRIC CORRECTION OF TIME - #============================================================================================== - - ltt_bary = image.obstime.light_travel_time(target_coord, ephemeris='jpl') - image.obstime = image.obstime.tdb + ltt_bary - - #============================================================================================== - # BACKGROUND ESTIMATION - #============================================================================================== - - fig, ax = plt.subplots(1, 2, figsize=(20, 18)) - plot_image(image.clean, ax=ax[0], scale='log', cbar='right', title='Image') - plot_image(image.mask, ax=ax[1], scale='linear', cbar='right', title='Mask') - fig.savefig(os.path.join(output_folder, 'original.png'), bbox_inches='tight') - plt.close(fig) - - # Estimate image background: - # Not using image.clean here, since we are redefining the mask anyway - bkg = Background2D(image.clean, (128, 128), filter_size=(5, 5), - sigma_clip=SigmaClip(sigma=3.0), - bkg_estimator=SExtractorBackground(), - exclude_percentile=50.0) - image.background = bkg.background - image.std = bkg.background_rms_median - - # Create background-subtracted image: - image.subclean = image.clean - image.background - - # Plot background estimation: - fig, ax = plt.subplots(1, 3, figsize=(20, 6)) - plot_image(image.clean, ax=ax[0], scale='log', title='Original') - plot_image(image.background, ax=ax[1], scale='log', title='Background') - plot_image(image.subclean, ax=ax[2], scale='log', title='Background subtracted') - fig.savefig(os.path.join(output_folder, 'background.png'), bbox_inches='tight') - plt.close(fig) - - # TODO: Is this correct?! - image.error = calc_total_error(image.clean, bkg.background_rms, 1.0) - - # Use sep to for soure extraction - image.sepdata = image.image.byteswap().newbyteorder() - image.sepbkg = sep.Background(image.sepdata,mask=image.mask) - image.sepsub = image.sepdata - image.sepbkg - logger.debug('sub: {} bkg_rms: {} mask: {}'.format(np.shape(image.sepsub),np.shape(image.sepbkg.globalrms), - np.shape(image.mask))) - objects = sep.extract(image.sepsub, thresh=5., err=image.sepbkg.globalrms, mask=image.mask, - deblend_cont=0.1, minarea=9, clean_param=2.0) - - - #============================================================================================== - # DETECTION OF STARS AND MATCHING WITH CATALOG - #============================================================================================== - - # Account for proper motion: - # TODO: Are catalog RA-proper motions including cosdec? - replace(references['pm_ra'], np.NaN, 0) - replace(references['pm_dec'], np.NaN, 0) - refs_coord = coords.SkyCoord(ra=references['ra'], dec=references['decl'], - pm_ra_cosdec=references['pm_ra'], pm_dec=references['pm_dec'], - unit='deg', frame='icrs', obstime=Time(2015.5, format='decimalyear')) - - refs_coord = refs_coord.apply_space_motion(image.obstime) - - # Solve for new WCS - cm = CoordinateMatch( - xy = list(zip(objects['x'], objects['y'])), - rd = list(zip(refs_coord.ra.deg, refs_coord.dec.deg)), - xy_order = np.argsort(-2.5*np.log10(objects['flux'])), - rd_order = np.argsort(target_coord.separation(refs_coord)), + """ + Run photometry. + + Parameters: + fileid (int): File ID to process. + output_folder (str, optional): Path to directory where output should be placed. + attempt_imagematch (bool, optional): If no subtracted image is available, but a + template image is, should we attempt to run ImageMatch using standard settings. + Default=True. + + .. codeauthor:: Rasmus Handberg + """ + + # Settings: + #ref_mag_limit = 22 # Lower limit on reference target brightness + ref_target_dist_limit = 10 * u.arcsec # Reference star must be further than this away to be included + + logger = logging.getLogger(__name__) + tic = default_timer() + + # Use local copy of archive if configured to do so: + config = load_config() + + # Get datafile dict from API: + datafile = api.get_datafile(fileid) + logger.debug("Datafile: %s", datafile) + targetid = datafile['targetid'] + target_name = datafile['target_name'] + photfilter = datafile['photfilter'] + + archive_local = config.get('photometry', 'archive_local', fallback=None) + if archive_local is not None: + datafile['archive_path'] = archive_local + if not os.path.isdir(datafile['archive_path']): + raise FileNotFoundError("ARCHIVE is not available: " + datafile['archive_path']) + + # Get the catalog containing the target and reference stars: + # TODO: Include proper-motion to the time of observation + catalog = api.get_catalog(targetid, output='table') + target = catalog['target'][0] + target_coord = coords.SkyCoord(ra=target['ra'], dec=target['decl'], unit='deg', frame='icrs') + + # Folder to save output: + if output_folder is None: + output_folder_root = config.get('photometry', 'output', fallback='.') + output_folder = os.path.join(output_folder_root, target_name, '%05d' % fileid) + logger.info("Placing output in '%s'", output_folder) + os.makedirs(output_folder, exist_ok=True) + + # The paths to the science image: + filepath = os.path.join(datafile['archive_path'], datafile['path']) + + # TODO: Download datafile using API to local drive: + # TODO: Is this a security concern? + #if archive_local: + # api.download_datafile(datafile, archive_local) + + # Translate photometric filter into table column: + ref_filter = { + 'up': 'u_mag', + 'gp': 'g_mag', + 'rp': 'r_mag', + 'ip': 'i_mag', + 'zp': 'z_mag', + 'B': 'B_mag', + 'V': 'V_mag', + 'J': 'J_mag', + 'H': 'H_mag', + 'K': 'K_mag', + }.get(photfilter, None) + + if ref_filter is None: + logger.warning("Could not find filter '%s' in catalogs. Using default gp filter.", photfilter) + ref_filter = 'g_mag' + + # Load the image from the FITS file: + image = load_image(filepath) + + lsqhdus = get_fitscmd(image, 'localseq') # look for local sequence in fits table + references = catalog['references'] if not lsqhdus else localseq(lsqhdus, image.exthdu) + + colorterms = get_fitscmd(image, 'colorterm') + references = colorterm(ref_filter, colorterms, references) if colorterms else references + + references.sort(ref_filter) + + # Check that there actually are reference stars in that filter: + if allnan(references[ref_filter]): + raise ValueError("No reference stars found in current photfilter.") + + #============================================================================================== + # BARYCENTRIC CORRECTION OF TIME + #============================================================================================== + + ltt_bary = image.obstime.light_travel_time(target_coord, ephemeris='jpl') + image.obstime = image.obstime.tdb + ltt_bary + + #============================================================================================== + # BACKGROUND ESTIMATION + #============================================================================================== + + fig, ax = plt.subplots(1, 2, figsize=(20, 18)) + plot_image(image.clean, ax=ax[0], scale='log', cbar='right', title='Image') + plot_image(image.mask, ax=ax[1], scale='linear', cbar='right', title='Mask') + fig.savefig(os.path.join(output_folder, 'original.png'), bbox_inches='tight') + plt.close(fig) + + # Estimate image background: + # Not using image.clean here, since we are redefining the mask anyway + bkg = Background2D(image.clean, (128, 128), filter_size=(5, 5), + sigma_clip=SigmaClip(sigma=3.0), + bkg_estimator=SExtractorBackground(), + exclude_percentile=50.0) + image.background = bkg.background + image.std = bkg.background_rms_median + + # Create background-subtracted image: + image.subclean = image.clean - image.background + + # Plot background estimation: + fig, ax = plt.subplots(1, 3, figsize=(20, 6)) + plot_image(image.clean, ax=ax[0], scale='log', title='Original') + plot_image(image.background, ax=ax[1], scale='log', title='Background') + plot_image(image.subclean, ax=ax[2], scale='log', title='Background subtracted') + fig.savefig(os.path.join(output_folder, 'background.png'), bbox_inches='tight') + plt.close(fig) + + # TODO: Is this correct?! + image.error = calc_total_error(image.clean, bkg.background_rms, 1.0) + + # Use sep to for soure extraction + image.sepdata = image.image.byteswap().newbyteorder() + image.sepbkg = sep.Background(image.sepdata,mask=image.mask) + image.sepsub = image.sepdata - image.sepbkg + logger.debug('sub: {} bkg_rms: {} mask: {}'.format(np.shape(image.sepsub),np.shape(image.sepbkg.globalrms), + np.shape(image.mask))) + objects = sep.extract(image.sepsub, thresh=5., err=image.sepbkg.globalrms, mask=image.mask, + deblend_cont=0.1, minarea=9, clean_param=2.0) + + + #============================================================================================== + # DETECTION OF STARS AND MATCHING WITH CATALOG + #============================================================================================== + + # Account for proper motion: + # TODO: Are catalog RA-proper motions including cosdec? + replace(references['pm_ra'], np.NaN, 0) + replace(references['pm_dec'], np.NaN, 0) + refs_coord = coords.SkyCoord(ra=references['ra'], dec=references['decl'], + pm_ra_cosdec=references['pm_ra'], pm_dec=references['pm_dec'], + unit='deg', frame='icrs', obstime=Time(2015.5, format='decimalyear')) + + refs_coord = refs_coord.apply_space_motion(image.obstime) + + # Solve for new WCS + cm = CoordinateMatch( + xy = list(zip(objects['x'], objects['y'])), + rd = list(zip(refs_coord.ra.deg, refs_coord.dec.deg)), + xy_order = np.argsort(-2.5*np.log10(objects['flux'])), + rd_order = np.argsort(target_coord.separation(refs_coord)), maximum_angle_distance = 0.002, - ) - - try: - i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=10))) - except (TimeoutError, StopIteration): - logging.warning('No new WCS solution found') - else: - image.wcs = fit_wcs_from_points( - np.array(list(zip(*cm.xy[i_xy]))), - coords.SkyCoord(*map(list, zip(*cm.rd[i_rd])), unit='deg') - ) + ) + + try: + i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=10))) + except (TimeoutError, StopIteration): + logging.warning('No new WCS solution found') + else: + image.wcs = fit_wcs_from_points( + np.array(list(zip(*cm.xy[i_xy]))), + coords.SkyCoord(*map(list, zip(*cm.rd[i_rd])), unit='deg') + ) # XXX # TESTING ################################################################ -# import matplotlib -# _backend = matplotlib.get_backend() -# matplotlib.pyplot.switch_backend('TkAgg') -# matplotlib.pyplot.subplot(projection=image.wcs) -# matplotlib.pyplot.imshow(image.subclean, origin='lower', cmap='gray_r', clim=np.nanquantile(image.subclean[image.subclean>0], (.01, .99))) -# matplotlib.pyplot.scatter(*zip(*image.wcs.wcs_world2pix(cm.rd, 0)), c=[], edgecolor='C0', s=200, label='References') -# for i, ((_x, _y), (_r, _d)) in enumerate(zip(cm.xy[i_xy], image.wcs.wcs_world2pix(cm.rd[i_rd], 0))): -# matplotlib.pyplot.plot([_x, _r], [_y, _d], color='C1', marker='o', ms=9, mfc='none', mec='C1', label='Solution' if not i else None) -# matplotlib.pyplot.scatter(*zip(*image.wcs.wcs_world2pix([(target['ra'], target['decl'])], 0)), c=[], edgecolor='C2', s=100, label='Target') -# matplotlib.pyplot.xlim(0, image.subclean.shape[1]) -# matplotlib.pyplot.ylim(0, image.subclean.shape[0]) -# matplotlib.pyplot.legend() -# matplotlib.pyplot.show() -# matplotlib.pyplot.switch_backend(_backend) +# import matplotlib +# _backend = matplotlib.get_backend() +# matplotlib.pyplot.switch_backend('TkAgg') +# matplotlib.pyplot.subplot(projection=image.wcs) +# matplotlib.pyplot.imshow(image.subclean, origin='lower', cmap='gray_r', clim=np.nanquantile(image.subclean[image.subclean>0], (.01, .99))) +# matplotlib.pyplot.scatter(*zip(*image.wcs.wcs_world2pix(cm.rd, 0)), c=[], edgecolor='C0', s=200, label='References') +# for i, ((_x, _y), (_r, _d)) in enumerate(zip(cm.xy[i_xy], image.wcs.wcs_world2pix(cm.rd[i_rd], 0))): +# matplotlib.pyplot.plot([_x, _r], [_y, _d], color='C1', marker='o', ms=9, mfc='none', mec='C1', label='Solution' if not i else None) +# matplotlib.pyplot.scatter(*zip(*image.wcs.wcs_world2pix([(target['ra'], target['decl'])], 0)), c=[], edgecolor='C2', s=100, label='Target') +# matplotlib.pyplot.xlim(0, image.subclean.shape[1]) +# matplotlib.pyplot.ylim(0, image.subclean.shape[0]) +# matplotlib.pyplot.legend() +# matplotlib.pyplot.show() +# matplotlib.pyplot.switch_backend(_backend) # XXX # TESTING ################################################################ - # Calculate pixel-coordinates of references: - row_col_coords = image.wcs.all_world2pix(np.array([[ref.ra.deg, ref.dec.deg] for ref in refs_coord]), 0) - references['pixel_column'] = row_col_coords[:,0] - references['pixel_row'] = row_col_coords[:,1] - - # Calculate the targets position in the image: - target_pixel_pos = image.wcs.all_world2pix([[target['ra'], target['decl']]], 0)[0] - - # Clean out the references: - hsize = 10 - x = references['pixel_column'] - y = references['pixel_row'] - - clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) - & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) - & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] - # & (references[ref_filter] < ref_mag_limit) - assert len(clean_references), 'No references in field' - - logger.info("References:\n%s", references) - - # @TODO: These need to be based on the instrument! - radius = 10 - fwhm_guess = 6.0 - fwhm_min = 3.5 - fwhm_max = 18.0 - - # Clean extracted stars (FIXME is this one necessary? it's only being used for plots) - masked_sep_xy,sep_mask,masked_sep_rsqs = force_reject_g2d(objects['x'], objects['y'], image, get_fwhm=False, - radius=radius, fwhm_guess=fwhm_guess, rsq_min=0.3, fwhm_max=fwhm_max, fwhm_min=fwhm_min) - - # Clean reference star locations - masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], - clean_references['pixel_row'], - image, - get_fwhm=True, - radius=radius, - fwhm_guess=fwhm_guess, - fwhm_max=fwhm_max, - fwhm_min=fwhm_min, - rsq_min=0.15) - - # Use R^2 to more robustly determine initial FWHM guess. - # This cleaning is good when we have FEW references. - fwhm, fwhm_clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, - min_fwhm_references=2, min_references=6, rsq_min=0.15) - logging.info('Initial FWHM guess is {} pixels'.format(fwhm)) - image.fwhm = fwhm - - - # Create plot of target and reference star positions from 2D Gaussian fits. - fig, ax = plt.subplots(1, 1, figsize=(20, 18)) - plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) - ax.scatter(fwhm_clean_references['pixel_column'], fwhm_clean_references['pixel_row'], c='r', marker='o', alpha=0.3) - ax.scatter(masked_sep_xy[:,0],masked_sep_xy[:,1],marker='s',alpha=1.0, edgecolors='green' ,facecolors='none') - ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') - fig.savefig(os.path.join(output_folder, 'positions_g2d.png'), bbox_inches='tight') - plt.close(fig) - - # # Sort by brightness - # clean_references.sort('g_mag') # Sorted by g mag - # _at = Table({'xy': masked_sep_xy, 'flux': objects['flux'][sep_mask]}) - # _at.sort('flux', reverse=True) # Sorted by flux - # masked_sep_xy = _at['xy'].data.data - - # # Check WCS - # wcs_rotation = 0 - # wcs_rota_max = 3 - # nreferences = len(clean_references['pixel_column']) - # try_aa = True - # while wcs_rotation < wcs_rota_max: - # nreferences_old = nreferences - # ref_xys = mkposxy(clean_references['pixel_column'], clean_references['pixel_row']) - # - # # Find matches using astroalign - # # try_kd = True - # # if try_aa: - # # for maxstars in [80,4]: - # # ref_ind, sep_ind, success_aa = try_astroalign(ref_xys, masked_sep_xy, - # # pixeltol=4*fwhm, - # # nnearest=min(20,len(ref_xys)), - # # max_stars_n=max(maxstars,len(ref_xys))) - # # # Break if successful - # # if success_aa: - # # astroalign_nmatches = len(ref_ind) - # # try_kd = False - # # if wcs_rotation > 1 and astroalign_nmatches <= 4: - # # try_kd = True - # # success_aa = False - # # break - # - # try_kd = True - # success_aa, try_aa = False, False # Don't use astroalign for now; it's giving false matches! - # - # # Find matches using nearest neighbor - # if try_kd: - # ref_ind_kd, sep_ind_kd, success_kd = kdtree(ref_xys, masked_sep_xy, fwhm, fwhm_max=4) - # if success_kd: - # kdtree_nmatches = len(ref_ind_kd) - # if try_kd and kdtree_nmatches > 3: - # ref_ind = ref_ind_kd - # sep_ind = sep_ind_kd - # else: - # success_kd = False - # - # if success_aa or success_kd: - # # Fit for new WCS - # wcs_rotation += 1 - # image.new_wcs = get_new_wcs(sep_ind, masked_sep_xy, clean_references, ref_ind, image.obstime) - # - # # Calculate pixel-coordinates of references: - # row_col_coords = image.new_wcs.all_world2pix( - # np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) - # references['pixel_column'] = row_col_coords[:, 0] - # references['pixel_row'] = row_col_coords[:, 1] - # - # # Clean out the references: - # hsize = 10 - # x = references['pixel_column'] - # y = references['pixel_row'] - # refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', - # frame='icrs') - # clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) - # & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) - # & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] - # try: - # masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], - # clean_references['pixel_row'], - # image, - # get_fwhm=True, - # radius=radius, - # fwhm_guess=fwhm, - # fwhm_max=fwhm_max, - # fwhm_min=fwhm_min, - # rsq_min=0.15) - # # Clean with R^2 - # fwhm, clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, - # min_fwhm_references=2, min_references=6, rsq_min=0.15) - # image.fwhm = fwhm - # nreferences_new = len(clean_references) - # logging.info('{} References were found after new wcs compared to {} references before'.format(nreferences_old,nreferences_new)) - # nreferences = nreferences_new - # wcs_success = True - # - # # Break early if no improvement after 2nd pass! - # # Note: New references can actually be less in a better WCS - # # if the actual stars were within radius (10) pixels of the edge. - # # @TODO: Adjust nreferences new and old based on whether extracted stars are within radius pix of edge. - # if wcs_rotation > 1 and nreferences_new <= nreferences_old: - # break - # except: - # # Calculate pixel-coordinates of references using old wcs: - # row_col_coords = image.wcs.all_world2pix( - # np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) - # references['pixel_column'] = row_col_coords[:, 0] - # references['pixel_row'] = row_col_coords[:, 1] - # - # # Clean out the references: - # hsize = 10 - # x = references['pixel_column'] - # y = references['pixel_row'] - # refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', - # frame='icrs') - # clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) - # & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) - # & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] - # wcs_success = False - # if try_aa: try_aa = False - # elif try_kd: break - # - # else: - # logging.info('New WCS could not be computed due to lack of matches.') - # wcs_success = False - # break - # - # if wcs_success: - # image.wcs = image.new_wcs - # - # # @Todo: Is the below block needed? - # # Final cleanout of references - # # Calculate pixel-coordinates of references using old wcs: - # row_col_coords = image.wcs.all_world2pix( - # np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) - # references['pixel_column'] = row_col_coords[:, 0] - # references['pixel_row'] = row_col_coords[:, 1] - # - # # Clean out the references: - # hsize = 10 - # x = references['pixel_column'] - # y = references['pixel_row'] - # refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', - # frame='icrs') - # references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) - # & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) - # & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] - - # masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], - # clean_references['pixel_row'], - # image, - # get_fwhm=True, - # radius=radius, - # fwhm_guess=fwhm, - # fwhm_max=fwhm_max, - # fwhm_min=fwhm_min, - # rsq_min=0.15) - - logger.debug("Number of references before final cleaning: %d", len(clean_references)) - references = get_clean_references(clean_references, masked_rsqs, rsq_ideal=0.8) - logger.debug("Number of references after final cleaning: %d", len(references)) - - # Create plot of target and reference star positions: - fig, ax = plt.subplots(1, 1, figsize=(20, 18)) - plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) - ax.scatter(references['pixel_column'], references['pixel_row'], c='r', marker='o', alpha=0.6) - ax.scatter(masked_sep_xy[:,0],masked_sep_xy[:,1], marker='s' , alpha=0.6, edgecolors='green' ,facecolors='none') - ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') - fig.savefig(os.path.join(output_folder, 'positions.png'), bbox_inches='tight') - plt.close(fig) - - #============================================================================================== - # CREATE EFFECTIVE PSF MODEL - #============================================================================================== - - # Make cutouts of stars using extract_stars: - # Scales with FWHM - size = int(np.round(29*fwhm/6)) - if size % 2 == 0: - size += 1 # Make sure it's a uneven number - size = max(size, 15) # Never go below 15 pixels - hsize = (size - 1) / 2 - - x = references['pixel_column'] - y = references['pixel_row'] - mask_near_edge = ((x > hsize) & (x < (image.shape[1] - 1 - hsize)) - & (y > hsize) & (y < (image.shape[0] - 1 - hsize))) - - stars_for_epsf = Table() - stars_for_epsf['x'] = x[mask_near_edge] - stars_for_epsf['y'] = y[mask_near_edge] - - # Store which stars were used in ePSF in the table: - logger.info("Number of stars used for ePSF: %d", len(stars_for_epsf)) - references['used_for_epsf'] = mask_near_edge - - # Extract stars sub-images: - stars = extract_stars( - NDData(data=image.subclean, mask=image.mask), - stars_for_epsf, - size=size - ) - - # Plot the stars being used for ePSF: - nrows = 5 - ncols = 5 - imgnr = 0 - for k in range(int(np.ceil(len(stars_for_epsf)/(nrows*ncols)))): - fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True) - ax = ax.ravel() - for i in range(nrows*ncols): - if imgnr > len(stars_for_epsf)-1: - ax[i].axis('off') - else: - plot_image(stars[imgnr], ax=ax[i], scale='log', cmap='viridis') - imgnr += 1 - - fig.savefig(os.path.join(output_folder, 'epsf_stars%02d.png' % (k+1)), bbox_inches='tight') - plt.close(fig) - - # Build the ePSF: - epsf = EPSFBuilder( - oversampling=1.0, - maxiters=500, - fitter=EPSFFitter(fit_boxsize=np.round(2*fwhm,0).astype(int)), - progress_bar=True, - recentering_func=centroid_com - )(stars)[0] - - logger.info('Successfully built PSF model') - - fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 15)) - plot_image(epsf.data, ax=ax1, cmap='viridis') - - fwhms = [] - bad_epsf_detected = False - for a, ax in ((0, ax3), (1, ax2)): - # Collapse the PDF along this axis: - profile = epsf.data.sum(axis=a) - itop = profile.argmax() - poffset = profile[itop]/2 - - # Run a spline through the points, but subtract half of the peak value, and find the roots: - # We have to use a cubic spline, since roots() is not supported for other splines - # for some reason - profile_intp = UnivariateSpline(np.arange(0, len(profile)), profile - poffset, k=3, s=0, ext=3) - lr = profile_intp.roots() - - # Plot the profile and spline: - x_fine = np.linspace(-0.5, len(profile)-0.5, 500) - ax.plot(profile, 'k.-') - ax.plot(x_fine, profile_intp(x_fine) + poffset, 'g-') - ax.axvline(itop) - ax.set_xlim(-0.5, len(profile)-0.5) - - # Do some sanity checks on the ePSF: - # It should pass 50% exactly twice and have the maximum inside that region. - # I.e. it should be a single gaussian-like peak - if len(lr) != 2 or itop < lr[0] or itop > lr[1]: - logger.error("Bad PSF along axis %d", a) - bad_epsf_detected = True - else: - axis_fwhm = lr[1] - lr[0] - fwhms.append(axis_fwhm) - ax.axvspan(lr[0], lr[1], facecolor='g', alpha=0.2) - - # Save the ePSF figure: - ax4.axis('off') - fig.savefig(os.path.join(output_folder, 'epsf.png'), bbox_inches='tight') - plt.close(fig) - - # There was a problem with the ePSF: - if bad_epsf_detected: - raise Exception("Bad ePSF detected.") - - # Let's make the final FWHM the largest one we found: - fwhm = np.max(fwhms) - image.fwhm = fwhm - logger.info("Final FWHM based on ePSF: %f", fwhm) - - #============================================================================================== - # COORDINATES TO DO PHOTOMETRY AT - #============================================================================================== - - coordinates = np.array([[ref['pixel_column'], ref['pixel_row']] for ref in references]) - - # Add the main target position as the first entry for doing photometry directly in the - # science image: - coordinates = np.concatenate(([target_pixel_pos], coordinates), axis=0) - - #============================================================================================== - # APERTURE PHOTOMETRY - #============================================================================================== - - # Define apertures for aperture photometry: - apertures = CircularAperture(coordinates, r=fwhm) - annuli = CircularAnnulus(coordinates, r_in=1.5*fwhm, r_out=2.5*fwhm) - - apphot_tbl = aperture_photometry(image.subclean, [apertures, annuli], mask=image.mask, error=image.error) - - logger.debug("Aperture Photometry Table:\n%s", apphot_tbl) - logger.info('Apperature Photometry Success') - - #============================================================================================== - # PSF PHOTOMETRY - #============================================================================================== - - # Are we fixing the postions? - epsf.fixed.update({'x_0': False, 'y_0': False}) - - # Create photometry object: - photometry = BasicPSFPhotometry( - group_maker=DAOGroup(fwhm), - bkg_estimator=SExtractorBackground(), - psf_model=epsf, - fitter=MaskableLevMarLSQFitter(), - fitshape=size, - aperture_radius=fwhm - ) - - psfphot_tbl = photometry( - image=image.subclean, - init_guesses=Table(coordinates, names=['x_0', 'y_0']) - ) - - logger.debug("PSF Photometry Table:\n%s", psfphot_tbl) - logger.info('PSF Photometry Success') - - #============================================================================================== - # TEMPLATE SUBTRACTION AND TARGET PHOTOMETRY - #============================================================================================== - - # Find the pixel-scale of the science image: - pixel_area = proj_plane_pixel_area(image.wcs.celestial) - pixel_scale = np.sqrt(pixel_area)*3600 # arcsec/pixel - #print(image.wcs.celestial.cunit) % Doesn't work? - logger.info("Science image pixel scale: %f", pixel_scale) - - diffimage = None - if datafile.get('diffimg') is not None: - - diffimg_path = os.path.join(datafile['archive_path'], datafile['diffimg']['path']) - diffimg = load_image(diffimg_path) - diffimage = diffimg.image - - elif attempt_imagematch and datafile.get('template') is not None: - # Run the template subtraction, and get back - # the science image where the template has been subtracted: - diffimage = run_imagematch(datafile, target, star_coord=coordinates, fwhm=fwhm, pixel_scale=pixel_scale) - - # We have a diff image, so let's do photometry of the target using this: - if diffimage is not None: - # Include mask from original image: - diffimage = np.ma.masked_array(diffimage, image.mask) - - # Create apertures around the target: - apertures = CircularAperture(target_pixel_pos, r=fwhm) - annuli = CircularAnnulus(target_pixel_pos, r_in=1.5*fwhm, r_out=2.5*fwhm) - - # Create two plots of the difference image: - fig, ax = plt.subplots(1, 1, squeeze=True, figsize=(20, 20)) - plot_image(diffimage, ax=ax, cbar='right', title=target_name) - ax.plot(target_pixel_pos[0], target_pixel_pos[1], marker='+', color='r') - fig.savefig(os.path.join(output_folder, 'diffimg.png'), bbox_inches='tight') - apertures.plot(color='r') - annuli.plot(color='k') - ax.set_xlim(target_pixel_pos[0]-50, target_pixel_pos[0]+50) - ax.set_ylim(target_pixel_pos[1]-50, target_pixel_pos[1]+50) - fig.savefig(os.path.join(output_folder, 'diffimg_zoom.png'), bbox_inches='tight') - plt.close(fig) - - # Run aperture photometry on subtracted image: - target_apphot_tbl = aperture_photometry(diffimage, [apertures, annuli], mask=image.mask, error=image.error) - - # Mask stars from FITS header - stars = get_fitscmd(diffimg, 'maskstar') - masked_diffimage = maskstar(diffimage, image.wcs, stars, image.fwhm) + # Calculate pixel-coordinates of references: + row_col_coords = image.wcs.all_world2pix(np.array([[ref.ra.deg, ref.dec.deg] for ref in refs_coord]), 0) + references['pixel_column'] = row_col_coords[:,0] + references['pixel_row'] = row_col_coords[:,1] + + # Calculate the targets position in the image: + target_pixel_pos = image.wcs.all_world2pix([[target['ra'], target['decl']]], 0)[0] + + # Clean out the references: + hsize = 10 + x = references['pixel_column'] + y = references['pixel_row'] + + clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) + & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) + & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] + # & (references[ref_filter] < ref_mag_limit) + assert len(clean_references), 'No references in field' + + logger.info("References:\n%s", references) + + # @TODO: These need to be based on the instrument! + radius = 10 + fwhm_guess = 6.0 + fwhm_min = 3.5 + fwhm_max = 18.0 + + # Clean extracted stars (FIXME is this one necessary? it's only being used for plots) + masked_sep_xy,sep_mask,masked_sep_rsqs = force_reject_g2d(objects['x'], objects['y'], image, get_fwhm=False, + radius=radius, fwhm_guess=fwhm_guess, rsq_min=0.3, fwhm_max=fwhm_max, fwhm_min=fwhm_min) + + # Clean reference star locations + masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], + clean_references['pixel_row'], + image, + get_fwhm=True, + radius=radius, + fwhm_guess=fwhm_guess, + fwhm_max=fwhm_max, + fwhm_min=fwhm_min, + rsq_min=0.15) + + # Use R^2 to more robustly determine initial FWHM guess. + # This cleaning is good when we have FEW references. + fwhm, fwhm_clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, + min_fwhm_references=2, min_references=6, rsq_min=0.15) + logging.info('Initial FWHM guess is {} pixels'.format(fwhm)) + image.fwhm = fwhm + + + # Create plot of target and reference star positions from 2D Gaussian fits. + fig, ax = plt.subplots(1, 1, figsize=(20, 18)) + plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) + ax.scatter(fwhm_clean_references['pixel_column'], fwhm_clean_references['pixel_row'], c='r', marker='o', alpha=0.3) + ax.scatter(masked_sep_xy[:,0],masked_sep_xy[:,1],marker='s',alpha=1.0, edgecolors='green' ,facecolors='none') + ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') + fig.savefig(os.path.join(output_folder, 'positions_g2d.png'), bbox_inches='tight') + plt.close(fig) + + # # Sort by brightness + # clean_references.sort('g_mag') # Sorted by g mag + # _at = Table({'xy': masked_sep_xy, 'flux': objects['flux'][sep_mask]}) + # _at.sort('flux', reverse=True) # Sorted by flux + # masked_sep_xy = _at['xy'].data.data + + # # Check WCS + # wcs_rotation = 0 + # wcs_rota_max = 3 + # nreferences = len(clean_references['pixel_column']) + # try_aa = True + # while wcs_rotation < wcs_rota_max: + # nreferences_old = nreferences + # ref_xys = mkposxy(clean_references['pixel_column'], clean_references['pixel_row']) + # + # # Find matches using astroalign + # # try_kd = True + # # if try_aa: + # # for maxstars in [80,4]: + # # ref_ind, sep_ind, success_aa = try_astroalign(ref_xys, masked_sep_xy, + # # pixeltol=4*fwhm, + # # nnearest=min(20,len(ref_xys)), + # # max_stars_n=max(maxstars,len(ref_xys))) + # # # Break if successful + # # if success_aa: + # # astroalign_nmatches = len(ref_ind) + # # try_kd = False + # # if wcs_rotation > 1 and astroalign_nmatches <= 4: + # # try_kd = True + # # success_aa = False + # # break + # + # try_kd = True + # success_aa, try_aa = False, False # Don't use astroalign for now; it's giving false matches! + # + # # Find matches using nearest neighbor + # if try_kd: + # ref_ind_kd, sep_ind_kd, success_kd = kdtree(ref_xys, masked_sep_xy, fwhm, fwhm_max=4) + # if success_kd: + # kdtree_nmatches = len(ref_ind_kd) + # if try_kd and kdtree_nmatches > 3: + # ref_ind = ref_ind_kd + # sep_ind = sep_ind_kd + # else: + # success_kd = False + # + # if success_aa or success_kd: + # # Fit for new WCS + # wcs_rotation += 1 + # image.new_wcs = get_new_wcs(sep_ind, masked_sep_xy, clean_references, ref_ind, image.obstime) + # + # # Calculate pixel-coordinates of references: + # row_col_coords = image.new_wcs.all_world2pix( + # np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) + # references['pixel_column'] = row_col_coords[:, 0] + # references['pixel_row'] = row_col_coords[:, 1] + # + # # Clean out the references: + # hsize = 10 + # x = references['pixel_column'] + # y = references['pixel_row'] + # refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', + # frame='icrs') + # clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) + # & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) + # & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] + # try: + # masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], + # clean_references['pixel_row'], + # image, + # get_fwhm=True, + # radius=radius, + # fwhm_guess=fwhm, + # fwhm_max=fwhm_max, + # fwhm_min=fwhm_min, + # rsq_min=0.15) + # # Clean with R^2 + # fwhm, clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, + # min_fwhm_references=2, min_references=6, rsq_min=0.15) + # image.fwhm = fwhm + # nreferences_new = len(clean_references) + # logging.info('{} References were found after new wcs compared to {} references before'.format(nreferences_old,nreferences_new)) + # nreferences = nreferences_new + # wcs_success = True + # + # # Break early if no improvement after 2nd pass! + # # Note: New references can actually be less in a better WCS + # # if the actual stars were within radius (10) pixels of the edge. + # # @TODO: Adjust nreferences new and old based on whether extracted stars are within radius pix of edge. + # if wcs_rotation > 1 and nreferences_new <= nreferences_old: + # break + # except: + # # Calculate pixel-coordinates of references using old wcs: + # row_col_coords = image.wcs.all_world2pix( + # np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) + # references['pixel_column'] = row_col_coords[:, 0] + # references['pixel_row'] = row_col_coords[:, 1] + # + # # Clean out the references: + # hsize = 10 + # x = references['pixel_column'] + # y = references['pixel_row'] + # refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', + # frame='icrs') + # clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) + # & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) + # & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] + # wcs_success = False + # if try_aa: try_aa = False + # elif try_kd: break + # + # else: + # logging.info('New WCS could not be computed due to lack of matches.') + # wcs_success = False + # break + # + # if wcs_success: + # image.wcs = image.new_wcs + # + # # @Todo: Is the below block needed? + # # Final cleanout of references + # # Calculate pixel-coordinates of references using old wcs: + # row_col_coords = image.wcs.all_world2pix( + # np.array([[ref['ra_obs'], ref['decl_obs']] for ref in references]), 0) + # references['pixel_column'] = row_col_coords[:, 0] + # references['pixel_row'] = row_col_coords[:, 1] + # + # # Clean out the references: + # hsize = 10 + # x = references['pixel_column'] + # y = references['pixel_row'] + # refs_coord = coords.SkyCoord(ra=references['ra_obs'], dec=references['decl_obs'], unit='deg', + # frame='icrs') + # references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) + # & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) + # & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] + + # masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], + # clean_references['pixel_row'], + # image, + # get_fwhm=True, + # radius=radius, + # fwhm_guess=fwhm, + # fwhm_max=fwhm_max, + # fwhm_min=fwhm_min, + # rsq_min=0.15) + + logger.debug("Number of references before final cleaning: %d", len(clean_references)) + references = get_clean_references(clean_references, masked_rsqs, rsq_ideal=0.8) + logger.debug("Number of references after final cleaning: %d", len(references)) + + # Create plot of target and reference star positions: + fig, ax = plt.subplots(1, 1, figsize=(20, 18)) + plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) + ax.scatter(references['pixel_column'], references['pixel_row'], c='r', marker='o', alpha=0.6) + ax.scatter(masked_sep_xy[:,0],masked_sep_xy[:,1], marker='s' , alpha=0.6, edgecolors='green' ,facecolors='none') + ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') + fig.savefig(os.path.join(output_folder, 'positions.png'), bbox_inches='tight') + plt.close(fig) + + #============================================================================================== + # CREATE EFFECTIVE PSF MODEL + #============================================================================================== + + # Make cutouts of stars using extract_stars: + # Scales with FWHM + size = int(np.round(29*fwhm/6)) + if size % 2 == 0: + size += 1 # Make sure it's a uneven number + size = max(size, 15) # Never go below 15 pixels + hsize = (size - 1) / 2 + + x = references['pixel_column'] + y = references['pixel_row'] + mask_near_edge = ((x > hsize) & (x < (image.shape[1] - 1 - hsize)) + & (y > hsize) & (y < (image.shape[0] - 1 - hsize))) + + stars_for_epsf = Table() + stars_for_epsf['x'] = x[mask_near_edge] + stars_for_epsf['y'] = y[mask_near_edge] + + # Store which stars were used in ePSF in the table: + logger.info("Number of stars used for ePSF: %d", len(stars_for_epsf)) + references['used_for_epsf'] = mask_near_edge + + # Extract stars sub-images: + stars = extract_stars( + NDData(data=image.subclean, mask=image.mask), + stars_for_epsf, + size=size + ) + + # Plot the stars being used for ePSF: + nrows = 5 + ncols = 5 + imgnr = 0 + for k in range(int(np.ceil(len(stars_for_epsf)/(nrows*ncols)))): + fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True) + ax = ax.ravel() + for i in range(nrows*ncols): + if imgnr > len(stars_for_epsf)-1: + ax[i].axis('off') + else: + plot_image(stars[imgnr], ax=ax[i], scale='log', cmap='viridis') + imgnr += 1 + + fig.savefig(os.path.join(output_folder, 'epsf_stars%02d.png' % (k+1)), bbox_inches='tight') + plt.close(fig) + + # Build the ePSF: + epsf = EPSFBuilder( + oversampling=1.0, + maxiters=500, + fitter=EPSFFitter(fit_boxsize=np.round(2*fwhm,0).astype(int)), + progress_bar=True, + recentering_func=centroid_com + )(stars)[0] + + logger.info('Successfully built PSF model') + + fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 15)) + plot_image(epsf.data, ax=ax1, cmap='viridis') + + fwhms = [] + bad_epsf_detected = False + for a, ax in ((0, ax3), (1, ax2)): + # Collapse the PDF along this axis: + profile = epsf.data.sum(axis=a) + itop = profile.argmax() + poffset = profile[itop]/2 + + # Run a spline through the points, but subtract half of the peak value, and find the roots: + # We have to use a cubic spline, since roots() is not supported for other splines + # for some reason + profile_intp = UnivariateSpline(np.arange(0, len(profile)), profile - poffset, k=3, s=0, ext=3) + lr = profile_intp.roots() + + # Plot the profile and spline: + x_fine = np.linspace(-0.5, len(profile)-0.5, 500) + ax.plot(profile, 'k.-') + ax.plot(x_fine, profile_intp(x_fine) + poffset, 'g-') + ax.axvline(itop) + ax.set_xlim(-0.5, len(profile)-0.5) + + # Do some sanity checks on the ePSF: + # It should pass 50% exactly twice and have the maximum inside that region. + # I.e. it should be a single gaussian-like peak + if len(lr) != 2 or itop < lr[0] or itop > lr[1]: + logger.error("Bad PSF along axis %d", a) + bad_epsf_detected = True + else: + axis_fwhm = lr[1] - lr[0] + fwhms.append(axis_fwhm) + ax.axvspan(lr[0], lr[1], facecolor='g', alpha=0.2) + + # Save the ePSF figure: + ax4.axis('off') + fig.savefig(os.path.join(output_folder, 'epsf.png'), bbox_inches='tight') + plt.close(fig) + + # There was a problem with the ePSF: + if bad_epsf_detected: + raise Exception("Bad ePSF detected.") + + # Let's make the final FWHM the largest one we found: + fwhm = np.max(fwhms) + image.fwhm = fwhm + logger.info("Final FWHM based on ePSF: %f", fwhm) + + #============================================================================================== + # COORDINATES TO DO PHOTOMETRY AT + #============================================================================================== + + coordinates = np.array([[ref['pixel_column'], ref['pixel_row']] for ref in references]) + + # Add the main target position as the first entry for doing photometry directly in the + # science image: + coordinates = np.concatenate(([target_pixel_pos], coordinates), axis=0) + + #============================================================================================== + # APERTURE PHOTOMETRY + #============================================================================================== + + # Define apertures for aperture photometry: + apertures = CircularAperture(coordinates, r=fwhm) + annuli = CircularAnnulus(coordinates, r_in=1.5*fwhm, r_out=2.5*fwhm) + + apphot_tbl = aperture_photometry(image.subclean, [apertures, annuli], mask=image.mask, error=image.error) + + logger.debug("Aperture Photometry Table:\n%s", apphot_tbl) + logger.info('Apperature Photometry Success') + + #============================================================================================== + # PSF PHOTOMETRY + #============================================================================================== + + # Are we fixing the postions? + epsf.fixed.update({'x_0': False, 'y_0': False}) + + # Create photometry object: + photometry = BasicPSFPhotometry( + group_maker=DAOGroup(fwhm), + bkg_estimator=SExtractorBackground(), + psf_model=epsf, + fitter=MaskableLevMarLSQFitter(), + fitshape=size, + aperture_radius=fwhm + ) + + psfphot_tbl = photometry( + image=image.subclean, + init_guesses=Table(coordinates, names=['x_0', 'y_0']) + ) + + logger.debug("PSF Photometry Table:\n%s", psfphot_tbl) + logger.info('PSF Photometry Success') + + #============================================================================================== + # TEMPLATE SUBTRACTION AND TARGET PHOTOMETRY + #============================================================================================== + + # Find the pixel-scale of the science image: + pixel_area = proj_plane_pixel_area(image.wcs.celestial) + pixel_scale = np.sqrt(pixel_area)*3600 # arcsec/pixel + #print(image.wcs.celestial.cunit) % Doesn't work? + logger.info("Science image pixel scale: %f", pixel_scale) + + diffimage = None + if datafile.get('diffimg') is not None: + + diffimg_path = os.path.join(datafile['archive_path'], datafile['diffimg']['path']) + diffimg = load_image(diffimg_path) + diffimage = diffimg.image + + elif attempt_imagematch and datafile.get('template') is not None: + # Run the template subtraction, and get back + # the science image where the template has been subtracted: + diffimage = run_imagematch(datafile, target, star_coord=coordinates, fwhm=fwhm, pixel_scale=pixel_scale) + + # We have a diff image, so let's do photometry of the target using this: + if diffimage is not None: + # Include mask from original image: + diffimage = np.ma.masked_array(diffimage, image.mask) + + # Create apertures around the target: + apertures = CircularAperture(target_pixel_pos, r=fwhm) + annuli = CircularAnnulus(target_pixel_pos, r_in=1.5*fwhm, r_out=2.5*fwhm) + + # Create two plots of the difference image: + fig, ax = plt.subplots(1, 1, squeeze=True, figsize=(20, 20)) + plot_image(diffimage, ax=ax, cbar='right', title=target_name) + ax.plot(target_pixel_pos[0], target_pixel_pos[1], marker='+', color='r') + fig.savefig(os.path.join(output_folder, 'diffimg.png'), bbox_inches='tight') + apertures.plot(color='r') + annuli.plot(color='k') + ax.set_xlim(target_pixel_pos[0]-50, target_pixel_pos[0]+50) + ax.set_ylim(target_pixel_pos[1]-50, target_pixel_pos[1]+50) + fig.savefig(os.path.join(output_folder, 'diffimg_zoom.png'), bbox_inches='tight') + plt.close(fig) + + # Run aperture photometry on subtracted image: + target_apphot_tbl = aperture_photometry(diffimage, [apertures, annuli], mask=image.mask, error=image.error) + + # Mask stars from FITS header + stars = get_fitscmd(diffimg, 'maskstar') + masked_diffimage = maskstar(diffimage, image.wcs, stars, image.fwhm) # Run PSF photometry on template subtracted image: - target_psfphot_tbl = photometry( - diffimage if masked_diffimage is None else masked_diffimage, - init_guesses=Table(target_pixel_pos, names=['x_0', 'y_0']) + target_psfphot_tbl = photometry( + diffimage if masked_diffimage is None else masked_diffimage, + init_guesses=Table(target_pixel_pos, names=['x_0', 'y_0']) ) - # Combine the output tables from the target and the reference stars into one: - apphot_tbl = vstack([target_apphot_tbl, apphot_tbl], join_type='exact') - psfphot_tbl = vstack([target_psfphot_tbl, psfphot_tbl], join_type='exact') - - # Build results table: - tab = references.copy() - extkeys = {'pm_ra', 'pm_dec', 'gaia_mag', 'gaia_bp_mag', 'gaia_rp_mag', 'B_mag', 'V_mag', 'H_mag','J_mag','K_mag', 'u_mag', 'g_mag', 'r_mag', 'i_mag', 'z_mag'} - - row = {'starid': 0, 'ra': target['ra'], 'decl': target['decl'], 'pixel_column': target_pixel_pos[0], 'pixel_row': target_pixel_pos[1]} - row.update([(k, np.NaN) for k in extkeys & set(tab.keys())]) - tab.insert_row(0, row) - - if diffimage is not None: - row['starid'] = -1 - tab.insert_row(0, row) - - indx_target = tab['starid'] <= 0 - - # Subtract background estimated from annuli: - flux_aperture = apphot_tbl['aperture_sum_0'] - (apphot_tbl['aperture_sum_1'] / annuli.area) * apertures.area - flux_aperture_error = np.sqrt(apphot_tbl['aperture_sum_err_0']**2 + (apphot_tbl['aperture_sum_err_1']/annuli.area * apertures.area)**2) - - # Add table columns with results: - tab['flux_aperture'] = flux_aperture/image.exptime - tab['flux_aperture_error'] = flux_aperture_error/image.exptime - tab['flux_psf'] = psfphot_tbl['flux_fit']/image.exptime - tab['flux_psf_error'] = psfphot_tbl['flux_unc']/image.exptime - tab['pixel_column_psf_fit'] = psfphot_tbl['x_fit'] - tab['pixel_row_psf_fit'] = psfphot_tbl['y_fit'] - tab['pixel_column_psf_fit_error'] = psfphot_tbl['x_0_unc'] - tab['pixel_row_psf_fit_error'] = psfphot_tbl['y_0_unc'] - - # Check that we got valid photometry: - if np.any(~np.isfinite(tab[indx_target]['flux_psf'])) or np.any(~np.isfinite(tab[indx_target]['flux_psf_error'])): - raise Exception("Target magnitude is undefined.") - - #============================================================================================== - # CALIBRATE - #============================================================================================== - - # Convert PSF fluxes to magnitudes: - mag_inst = -2.5 * np.log10(tab['flux_psf']) - mag_inst_err = (2.5/np.log(10)) * (tab['flux_psf_error'] / tab['flux_psf']) - - # Corresponding magnitudes in catalog: - #TODO: add color terms here - mag_catalog = tab[ref_filter] - - # Mask out things that should not be used in calibration: - use_for_calibration = np.ones_like(mag_catalog, dtype='bool') - use_for_calibration[indx_target] = False # Do not use target for calibration - use_for_calibration[~np.isfinite(mag_inst) | ~np.isfinite(mag_catalog)] = False - - # Just creating some short-hands: - x = mag_catalog[use_for_calibration] - y = mag_inst[use_for_calibration] - yerr = mag_inst_err[use_for_calibration] - weights = 1.0/yerr**2 - - # Fit linear function with fixed slope, using sigma-clipping: - model = models.Linear1D(slope=1, fixed={'slope': True}) - fitter = fitting.FittingWithOutlierRemoval(fitting.LinearLSQFitter(), sigma_clip, sigma=3.0) - best_fit, sigma_clipped = fitter(model, x, y, weights=weights) - - # Extract zero-point and estimate its error using a single weighted fit: - # I don't know why there is not an error-estimate attached directly to the Parameter? - zp = -1*best_fit.intercept.value # Negative, because that is the way zeropoints are usually defined - - weights[sigma_clipped] = 0 # Trick to make following expression simpler - N = len(weights.nonzero()[0]) - if N > 1: - zp_error = np.sqrt( N * nansum(weights*(y - best_fit(x))**2) / nansum(weights) / (N-1) ) - else: - zp_error = np.NaN - logger.info('Leastsquare ZP = %.3f, ZP_error = %.3f', zp, zp_error) - - # Determine sigma clipping sigma according to Chauvenet method - # But don't allow less than sigma = sigmamin, setting to 1.5 for now. - # Should maybe be 2? - sigmamin = 1.5 - sigChauv = sigma_from_Chauvenet(len(x)) - sigChauv = sigChauv if sigChauv >= sigmamin else sigmamin - - # Extract zero point and error using bootstrap method - Nboot = 1000 - logger.info('Running bootstrap with sigma = %.2f and n = %d', sigChauv, Nboot) - pars = bootstrap_outlier(x, y, yerr, n=Nboot, model=model, fitter=fitting.LinearLSQFitter, - outlier=sigma_clip, outlier_kwargs={'sigma':sigChauv}, summary='median', - error='bootstrap', return_vals=False) - - zp_bs = pars['intercept'] * -1.0 - zp_error_bs = pars['intercept_error'] - - logger.info('Bootstrapped ZP = %.3f, ZP_error = %.3f', zp_bs, zp_error_bs) - - # Check that difference is not large - zp_diff = 0.4 - if np.abs(zp_bs - zp) >= zp_diff: - logger.warning("Bootstrap and weighted LSQ ZPs differ by %.2f, \ - which is more than the allowed %.2f mag.", np.abs(zp_bs - zp), zp_diff) - - # Add calibrated magnitudes to the photometry table: - tab['mag'] = mag_inst + zp_bs - tab['mag_error'] = np.sqrt(mag_inst_err**2 + zp_error_bs**2) - - fig, ax = plt.subplots(1, 1) - ax.errorbar(x, y, yerr=yerr, fmt='k.') - ax.scatter(x[sigma_clipped], y[sigma_clipped], marker='x', c='r') - ax.plot(x, best_fit(x), color='g', linewidth=3) - ax.set_xlabel('Catalog magnitude') - ax.set_ylabel('Instrumental magnitude') - fig.savefig(os.path.join(output_folder, 'calibration.png'), bbox_inches='tight') - plt.close(fig) - - # Check that we got valid photometry: - if not np.isfinite(tab[0]['mag']) or not np.isfinite(tab[0]['mag_error']): - raise Exception("Target magnitude is undefined.") - - #============================================================================================== - # SAVE PHOTOMETRY - #============================================================================================== - - # Descriptions of columns: - tab['flux_aperture'].unit = u.count/u.second - tab['flux_aperture_error'].unit = u.count/u.second - tab['flux_psf'].unit = u.count/u.second - tab['flux_psf_error'].unit = u.count/u.second - tab['pixel_column'].unit = u.pixel - tab['pixel_row'].unit = u.pixel - tab['pixel_column_psf_fit'].unit = u.pixel - tab['pixel_row_psf_fit'].unit = u.pixel - tab['pixel_column_psf_fit_error'].unit = u.pixel - tab['pixel_row_psf_fit_error'].unit = u.pixel - - # Meta-data: - tab.meta['version'] = __version__ - tab.meta['fileid'] = fileid - tab.meta['template'] = None if datafile.get('template') is None else datafile['template']['fileid'] - tab.meta['diffimg'] = None if datafile.get('diffimg') is None else datafile['diffimg']['fileid'] - tab.meta['photfilter'] = photfilter - tab.meta['fwhm'] = fwhm * u.pixel - tab.meta['pixel_scale'] = pixel_scale * u.arcsec/u.pixel - tab.meta['seeing'] = (fwhm*pixel_scale) * u.arcsec - tab.meta['obstime-bmjd'] = float(image.obstime.mjd) - tab.meta['zp'] = zp_bs - tab.meta['zp_error'] = zp_error_bs - tab.meta['zp_diff'] = np.abs(zp_bs - zp) - tab.meta['zp_error_weights'] = zp_error - - # Filepath where to save photometry: - photometry_output = os.path.join(output_folder, 'photometry.ecsv') - - # Write the final table to file: - tab.write(photometry_output, format='ascii.ecsv', delimiter=',', overwrite=True) - - toc = default_timer() - - logger.info("------------------------------------------------------") - logger.info("Success!") - logger.info("Main target: %f +/- %f", tab[0]['mag'], tab[0]['mag_error']) - logger.info("Photometry took: %f seconds", toc-tic) - - return photometry_output + # Combine the output tables from the target and the reference stars into one: + apphot_tbl = vstack([target_apphot_tbl, apphot_tbl], join_type='exact') + psfphot_tbl = vstack([target_psfphot_tbl, psfphot_tbl], join_type='exact') + + # Build results table: + tab = references.copy() + extkeys = {'pm_ra', 'pm_dec', 'gaia_mag', 'gaia_bp_mag', 'gaia_rp_mag', 'B_mag', 'V_mag', 'H_mag','J_mag','K_mag', 'u_mag', 'g_mag', 'r_mag', 'i_mag', 'z_mag'} + + row = {'starid': 0, 'ra': target['ra'], 'decl': target['decl'], 'pixel_column': target_pixel_pos[0], 'pixel_row': target_pixel_pos[1]} + row.update([(k, np.NaN) for k in extkeys & set(tab.keys())]) + tab.insert_row(0, row) + + if diffimage is not None: + row['starid'] = -1 + tab.insert_row(0, row) + + indx_target = tab['starid'] <= 0 + + # Subtract background estimated from annuli: + flux_aperture = apphot_tbl['aperture_sum_0'] - (apphot_tbl['aperture_sum_1'] / annuli.area) * apertures.area + flux_aperture_error = np.sqrt(apphot_tbl['aperture_sum_err_0']**2 + (apphot_tbl['aperture_sum_err_1']/annuli.area * apertures.area)**2) + + # Add table columns with results: + tab['flux_aperture'] = flux_aperture/image.exptime + tab['flux_aperture_error'] = flux_aperture_error/image.exptime + tab['flux_psf'] = psfphot_tbl['flux_fit']/image.exptime + tab['flux_psf_error'] = psfphot_tbl['flux_unc']/image.exptime + tab['pixel_column_psf_fit'] = psfphot_tbl['x_fit'] + tab['pixel_row_psf_fit'] = psfphot_tbl['y_fit'] + tab['pixel_column_psf_fit_error'] = psfphot_tbl['x_0_unc'] + tab['pixel_row_psf_fit_error'] = psfphot_tbl['y_0_unc'] + + # Check that we got valid photometry: + if np.any(~np.isfinite(tab[indx_target]['flux_psf'])) or np.any(~np.isfinite(tab[indx_target]['flux_psf_error'])): + raise Exception("Target magnitude is undefined.") + + #============================================================================================== + # CALIBRATE + #============================================================================================== + + # Convert PSF fluxes to magnitudes: + mag_inst = -2.5 * np.log10(tab['flux_psf']) + mag_inst_err = (2.5/np.log(10)) * (tab['flux_psf_error'] / tab['flux_psf']) + + # Corresponding magnitudes in catalog: + #TODO: add color terms here + mag_catalog = tab[ref_filter] + + # Mask out things that should not be used in calibration: + use_for_calibration = np.ones_like(mag_catalog, dtype='bool') + use_for_calibration[indx_target] = False # Do not use target for calibration + use_for_calibration[~np.isfinite(mag_inst) | ~np.isfinite(mag_catalog)] = False + + # Just creating some short-hands: + x = mag_catalog[use_for_calibration] + y = mag_inst[use_for_calibration] + yerr = mag_inst_err[use_for_calibration] + weights = 1.0/yerr**2 + + # Fit linear function with fixed slope, using sigma-clipping: + model = models.Linear1D(slope=1, fixed={'slope': True}) + fitter = fitting.FittingWithOutlierRemoval(fitting.LinearLSQFitter(), sigma_clip, sigma=3.0) + best_fit, sigma_clipped = fitter(model, x, y, weights=weights) + + # Extract zero-point and estimate its error using a single weighted fit: + # I don't know why there is not an error-estimate attached directly to the Parameter? + zp = -1*best_fit.intercept.value # Negative, because that is the way zeropoints are usually defined + + weights[sigma_clipped] = 0 # Trick to make following expression simpler + N = len(weights.nonzero()[0]) + if N > 1: + zp_error = np.sqrt( N * nansum(weights*(y - best_fit(x))**2) / nansum(weights) / (N-1) ) + else: + zp_error = np.NaN + logger.info('Leastsquare ZP = %.3f, ZP_error = %.3f', zp, zp_error) + + # Determine sigma clipping sigma according to Chauvenet method + # But don't allow less than sigma = sigmamin, setting to 1.5 for now. + # Should maybe be 2? + sigmamin = 1.5 + sigChauv = sigma_from_Chauvenet(len(x)) + sigChauv = sigChauv if sigChauv >= sigmamin else sigmamin + + # Extract zero point and error using bootstrap method + Nboot = 1000 + logger.info('Running bootstrap with sigma = %.2f and n = %d', sigChauv, Nboot) + pars = bootstrap_outlier(x, y, yerr, n=Nboot, model=model, fitter=fitting.LinearLSQFitter, + outlier=sigma_clip, outlier_kwargs={'sigma':sigChauv}, summary='median', + error='bootstrap', return_vals=False) + + zp_bs = pars['intercept'] * -1.0 + zp_error_bs = pars['intercept_error'] + + logger.info('Bootstrapped ZP = %.3f, ZP_error = %.3f', zp_bs, zp_error_bs) + + # Check that difference is not large + zp_diff = 0.4 + if np.abs(zp_bs - zp) >= zp_diff: + logger.warning("Bootstrap and weighted LSQ ZPs differ by %.2f, \ + which is more than the allowed %.2f mag.", np.abs(zp_bs - zp), zp_diff) + + # Add calibrated magnitudes to the photometry table: + tab['mag'] = mag_inst + zp_bs + tab['mag_error'] = np.sqrt(mag_inst_err**2 + zp_error_bs**2) + + fig, ax = plt.subplots(1, 1) + ax.errorbar(x, y, yerr=yerr, fmt='k.') + ax.scatter(x[sigma_clipped], y[sigma_clipped], marker='x', c='r') + ax.plot(x, best_fit(x), color='g', linewidth=3) + ax.set_xlabel('Catalog magnitude') + ax.set_ylabel('Instrumental magnitude') + fig.savefig(os.path.join(output_folder, 'calibration.png'), bbox_inches='tight') + plt.close(fig) + + # Check that we got valid photometry: + if not np.isfinite(tab[0]['mag']) or not np.isfinite(tab[0]['mag_error']): + raise Exception("Target magnitude is undefined.") + + #============================================================================================== + # SAVE PHOTOMETRY + #============================================================================================== + + # Descriptions of columns: + tab['flux_aperture'].unit = u.count/u.second + tab['flux_aperture_error'].unit = u.count/u.second + tab['flux_psf'].unit = u.count/u.second + tab['flux_psf_error'].unit = u.count/u.second + tab['pixel_column'].unit = u.pixel + tab['pixel_row'].unit = u.pixel + tab['pixel_column_psf_fit'].unit = u.pixel + tab['pixel_row_psf_fit'].unit = u.pixel + tab['pixel_column_psf_fit_error'].unit = u.pixel + tab['pixel_row_psf_fit_error'].unit = u.pixel + + # Meta-data: + tab.meta['version'] = __version__ + tab.meta['fileid'] = fileid + tab.meta['template'] = None if datafile.get('template') is None else datafile['template']['fileid'] + tab.meta['diffimg'] = None if datafile.get('diffimg') is None else datafile['diffimg']['fileid'] + tab.meta['photfilter'] = photfilter + tab.meta['fwhm'] = fwhm * u.pixel + tab.meta['pixel_scale'] = pixel_scale * u.arcsec/u.pixel + tab.meta['seeing'] = (fwhm*pixel_scale) * u.arcsec + tab.meta['obstime-bmjd'] = float(image.obstime.mjd) + tab.meta['zp'] = zp_bs + tab.meta['zp_error'] = zp_error_bs + tab.meta['zp_diff'] = np.abs(zp_bs - zp) + tab.meta['zp_error_weights'] = zp_error + + # Filepath where to save photometry: + photometry_output = os.path.join(output_folder, 'photometry.ecsv') + + # Write the final table to file: + tab.write(photometry_output, format='ascii.ecsv', delimiter=',', overwrite=True) + + toc = default_timer() + + logger.info("------------------------------------------------------") + logger.info("Success!") + logger.info("Main target: %f +/- %f", tab[0]['mag'], tab[0]['mag_error']) + logger.info("Photometry took: %f seconds", toc-tic) + + return photometry_output From 27a18a7a9faa4702651762a4f55ed22cd1d1cc7c Mon Sep 17 00:00:00 2001 From: Simon Holmbo Date: Wed, 24 Feb 2021 14:00:19 +0100 Subject: [PATCH 30/43] week08 --- flows/__init__.py | 1 + flows/api/catalogs.py | 10 +- flows/api/datafiles.py | 10 +- flows/api/filters.py | 1 + flows/api/sites.py | 10 +- flows/coordinatematch/__init__.py | 1 + flows/coordinatematch/coordinatematch.py | 10 +- flows/coordinatematch/wcs.py | 6 + flows/filters.py | 5 + flows/fitting.py | 12 -- flows/image/__init__.py | 1 + flows/image/image.py | 130 ++++++++++++++++++++ flows/instruments/__init__.py | 61 ++++++++++ flows/instruments/instrument.py | 78 ++++++++++++ flows/instruments/lcogt.py | 46 +++++++ flows/instruments/liverpool.py | 29 +++++ flows/photometry.py | 31 +++-- flows/references.py | 75 ++++++++++++ run_localweb.py | 147 ++++++----------------- web/__init__.py | 83 +++++++++++++ web/datafiles.py | 12 ++ web/reference_stars.py | 26 ++++ web/sites.py | 10 ++ 23 files changed, 655 insertions(+), 140 deletions(-) create mode 100644 flows/filters.py delete mode 100644 flows/fitting.py create mode 100644 flows/image/__init__.py create mode 100644 flows/image/image.py create mode 100644 flows/instruments/__init__.py create mode 100644 flows/instruments/instrument.py create mode 100644 flows/instruments/lcogt.py create mode 100644 flows/instruments/liverpool.py create mode 100644 flows/references.py create mode 100644 web/__init__.py create mode 100644 web/datafiles.py create mode 100644 web/reference_stars.py create mode 100644 web/sites.py diff --git a/flows/__init__.py b/flows/__init__.py index f65aed0..55ba1a9 100644 --- a/flows/__init__.py +++ b/flows/__init__.py @@ -6,6 +6,7 @@ from .download_catalog import download_catalog from .visibility import visibility from .config import load_config +from .filters import FILTERS from .version import get_version __version__ = get_version(pep440=False) diff --git a/flows/api/catalogs.py b/flows/api/catalogs.py index 97ab13e..c8461b0 100644 --- a/flows/api/catalogs.py +++ b/flows/api/catalogs.py @@ -36,12 +36,15 @@ def get_catalog(target, radius=None, output='table'): # Get API token from config file: config = load_config() + address = config.get('api', 'catalog', fallback=None) token = config.get('api', 'token', fallback=None) + if address is None: + raise Exception("No API catalog address has been defined") if token is None: raise Exception("No API token has been defined") # - r = requests.get('https://flows.phys.au.dk/api/reference_stars.php', + r = requests.get('%s/reference_stars.php' % address, params={'target': target}, headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() @@ -113,12 +116,15 @@ def get_catalog_missing(): # Get API token from config file: config = load_config() + address = config.get('api', 'catalog', fallback=None) token = config.get('api', 'token', fallback=None) + if address is None: + raise Exception("No API catalog address has been defined") if token is None: raise Exception("No API token has been defined") # - r = requests.get('https://flows.phys.au.dk/api/catalog_missing.php', + r = requests.get('%s/catalog_missing.php' % address, headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() return r.json() diff --git a/flows/api/datafiles.py b/flows/api/datafiles.py index 6cd8843..5650ded 100644 --- a/flows/api/datafiles.py +++ b/flows/api/datafiles.py @@ -15,11 +15,14 @@ def get_datafile(fileid): # Get API token from config file: config = load_config() + address = config.get('api', 'address', fallback=None) token = config.get('api', 'token', fallback=None) + if address is None: + raise Exception("No API catalog address has been defined") if token is None: raise Exception("No API token has been defined") - r = requests.get('https://flows.phys.au.dk/api/datafiles.php', + r = requests.get('%s/datafiles.php' % address, params={'fileid': fileid}, headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() @@ -56,7 +59,10 @@ def get_datafiles(targetid=None, filt=None): # Get API token from config file: config = load_config() + address = config.get('api', 'address', fallback=None) token = config.get('api', 'token', fallback=None) + if address is None: + raise Exception("No API catalog address has been defined") if token is None: raise Exception("No API token has been defined") @@ -65,7 +71,7 @@ def get_datafiles(targetid=None, filt=None): params['targetid'] = targetid params['filter'] = filt - r = requests.get('https://flows.phys.au.dk/api/datafiles.php', + r = requests.get('%s/datafiles.php' % address, params=params, headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() diff --git a/flows/api/filters.py b/flows/api/filters.py index 9ff1ca0..ca24aa8 100644 --- a/flows/api/filters.py +++ b/flows/api/filters.py @@ -22,6 +22,7 @@ def get_filters(): r = requests.get('https://flows.phys.au.dk/api/filters.php', headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() + print(r.text) jsn = r.json() # Add units: diff --git a/flows/api/sites.py b/flows/api/sites.py index 96dd67f..f6854c7 100644 --- a/flows/api/sites.py +++ b/flows/api/sites.py @@ -16,11 +16,14 @@ def get_site(siteid): # Get API token from config file: config = load_config() + address = config.get('api', 'address', fallback=None) token = config.get('api', 'token', fallback=None) + if address is None: + raise Exception("No API catalog address has been defined") if token is None: raise Exception("No API token has been defined") - r = requests.get('https://flows.phys.au.dk/api/sites.php', + r = requests.get('%s/sites.php' % address, params={'siteid': siteid}, headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() @@ -37,11 +40,14 @@ def get_all_sites(): # Get API token from config file: config = load_config() + address = config.get('api', 'address', fallback=None) token = config.get('api', 'token', fallback=None) + if address is None: + raise Exception("No API catalog address has been defined") if token is None: raise Exception("No API token has been defined") - r = requests.get('https://flows.phys.au.dk/api/sites.php', + r = requests.get('%s/sites.php' % address, headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() jsn = r.json() diff --git a/flows/coordinatematch/__init__.py b/flows/coordinatematch/__init__.py index 8326efe..ce2eb3c 100644 --- a/flows/coordinatematch/__init__.py +++ b/flows/coordinatematch/__init__.py @@ -1 +1,2 @@ from .coordinatematch import CoordinateMatch +from .wcs import WCS diff --git a/flows/coordinatematch/coordinatematch.py b/flows/coordinatematch/coordinatematch.py index df5aef7..f6bb2cb 100644 --- a/flows/coordinatematch/coordinatematch.py +++ b/flows/coordinatematch/coordinatematch.py @@ -12,7 +12,9 @@ class CoordinateMatch () : - def __init__(self, xy, rd, xy_order=None, rd_order=None, + def __init__(self, xy, rd, + xy_order=None, rd_order=None, + xy_nmax=None, rd_nmax=None, n_triangle_packages = 10, triangle_package_size = 10000, maximum_angle_distance = 0.001, @@ -25,8 +27,10 @@ def __init__(self, xy, rd, xy_order=None, rd_order=None, self._rd = rd - np.mean(rd, axis=0) self._rd[:,0] *= np.cos(np.deg2rad(self.rd[:,1])) - self.i_xy = xy_order if not xy_order is None else np.arange(len(xy)) - self.i_rd = rd_order if not rd_order is None else np.arange(len(rd)) + xy_n, rd_n = min(xy_nmax, len(xy)), min(rd_nmax, len(rd)) + + self.i_xy = xy_order[:xy_n] if not xy_order is None else np.arange(xy_n) + self.i_rd = rd_order[:rd_n] if not rd_order is None else np.arange(rd_n) self.n_triangle_packages = n_triangle_packages self.triangle_package_size = triangle_package_size diff --git a/flows/coordinatematch/wcs.py b/flows/coordinatematch/wcs.py index ff23230..e3e5c07 100644 --- a/flows/coordinatematch/wcs.py +++ b/flows/coordinatematch/wcs.py @@ -206,3 +206,9 @@ def __call__(self, **kwargs): obj.__setattr__(k, v) return obj + + def __repr__(self): + + ra, dec = self.astropy_wcs.wcs_pix2world([(0, 0)], 0)[0] + + return f'WCS(0, 0, {ra:.4f}, {dec:.4f}, {self.scale:.2f}, {self.mirror}, {self.angle:.2f})' diff --git a/flows/filters.py b/flows/filters.py new file mode 100644 index 0000000..f42aa1a --- /dev/null +++ b/flows/filters.py @@ -0,0 +1,5 @@ +FILTERS = { + 'u', 'g', 'r', 'i', 'z', + 'B', 'V', 'R', 'I', + 'J', 'H', 'K', +} diff --git a/flows/fitting.py b/flows/fitting.py deleted file mode 100644 index de30eeb..0000000 --- a/flows/fitting.py +++ /dev/null @@ -1,12 +0,0 @@ -import numpy as np - -from astropy.modeling.fitting import LevMarLSQFitter - -class MaskableLevMarLSQFitter(LevMarLSQFitter): - - def __call__(self, model, x, y, z=None, *args, **kwargs): - - if hasattr(z, 'mask'): - x, y, z = x[~z.mask], y[~z.mask], z[~z.mask] - - return super().__call__(model, x, y, z, *args, **kwargs) diff --git a/flows/image/__init__.py b/flows/image/__init__.py new file mode 100644 index 0000000..a0c1ffb --- /dev/null +++ b/flows/image/__init__.py @@ -0,0 +1 @@ +from .image import Image diff --git a/flows/image/image.py b/flows/image/image.py new file mode 100644 index 0000000..e8f6cbb --- /dev/null +++ b/flows/image/image.py @@ -0,0 +1,130 @@ +import pickle, warnings + +import numpy as np + +from astropy.io import fits +from astropy.wcs import WCS, FITSFixedWarning + +class Image: + + MASK_PARAMETERS = 'wcs', + + def __init__(self, data, hdr, wcs, exthdus=dict(), subtracted=None): + + self._data = data = np.asarray(data) + self.x, self.y = np.meshgrid(*map(np.arange, data.shape[::-1])) + + self.hdr, self.wcs = hdr, wcs + self.exthdus = exthdus + + self.set_subtracted(subtracted) + + self._lmasks = dict() + self._mask, self._mask_hash = None, None + self._data_mask = None + + self.instrument = None + + @classmethod + def from_fits(cls, filename, subtracted=None): + + with fits.open(filename, mode='readonly') as hdul: + + data = hdul[0].data + hdr = hdul[0].header + exthdus = {hdu.name:hdu.copy() for hdu in hdul[1:]} + + with warnings.catch_warnings(): + + warnings.simplefilter('ignore', category=FITSFixedWarning) + wcs = WCS(hdr) + + if not subtracted is None: + + with fits.open(subtracted, mode='readonly') as hdul: + subtracted = hdul[0].data + + else: + + subtracted = None + + return cls(data, hdr, wcs, exthdus, subtracted) + + def set_subtracted(self, subtracted): + + assert subtracted is None or np.shape(subtracted) == self._data.shape + + self._subtracted = np.asarray(subtracted) if not subtracted is None else None + self._subtracted_mask = None + + def set_instrument(self, instrument): + + self.instrument = instrument + self.filter = instrument.get_filter(self) + self.obstime = instrument.get_obstime(self) + self.exptime = instrument.get_exptime(self) + + if not (mask := instrument.get_mask(self)) is None: + self.add_mask(mask) + + def add_mask(self, mask): + + i = max(self._lmasks.keys()) + 1 if len(self._lmasks) else 0 + self._lmasks[i] = mask + + return i + + def del_mask(self, i): + + assert i in self._lmasks, 'mask id does not exist' + + del self._lmasks[i] + + def _update_mask(self): + + mask_parameters = sorted([hash(mask) for mask in self._lmasks.values()]) + mask_parameters += [getattr(self, p) for p in self.MASK_PARAMETERS] + mask_hash = hash(pickle.dumps(mask_parameters)) + + print(self._mask_hash == mask_hash) + if self._mask_hash == mask_hash: + return False + + self._mask_hash = mask_hash + + masks = [~mask(self).astype(bool) for mask in self._lmasks.values()] + self._mask = ~np.prod(masks, axis=0, dtype=bool) \ + if len(masks) else np.zeros_like(self._data, dtype=bool) + + return True + + @property + def data(self): + + if self._update_mask() or self._data_mask is None: + + self._data_mask = self._mask.copy() + self._data_mask |= ~np.isfinite(self._data) + + return np.ma.array(self._data, mask=self._data_mask) + + @property + def subtracted(self): + + if self._subtracted is None: + raise AttributeError('no subtracted image') + + if self._update_mask() or self._subtracted_mask is None: + + self._subtracted_mask = self._mask.copy() + self._subtracted_mask |= ~np.isfinite(self._subtracted) + + return np.ma.array(self._subtracted, mask=self._subtracted_mask) + + def __getitem__(self, item): + + if item in ('filter', 'obstime', 'exptime') and \ + self.instrument is None: + raise AttributeError('can not get %s without instrument' % item) + + return super().__getitem__(item) diff --git a/flows/instruments/__init__.py b/flows/instruments/__init__.py new file mode 100644 index 0000000..d18fc46 --- /dev/null +++ b/flows/instruments/__init__.py @@ -0,0 +1,61 @@ +import os, logging + +from inspect import getmro +from traceback import format_exc + +from importlib import import_module + +from .instrument import Instrument + +INSTRUMENTS = list() + +for instrument_file in os.listdir(__file__.rsplit('/',1)[0]): + + if not '.' in instrument_file: + continue + + instrument_filename, file_ext = instrument_file.rsplit('.', 1) + + if instrument_filename[0] == '_' or file_ext != 'py': + continue + + instrument = import_module('.' + instrument_filename, 'flows.instruments') + + for attribute in dir(instrument): + + if attribute[0] == '_': + continue + + attribute = getattr(instrument, attribute) + + if not hasattr(attribute, '__bases__'): + continue + + all_bases = [base for _base in attribute.__bases__ for base in getmro(_base)] + if not Instrument in all_bases or not hasattr(attribute, 'siteid'): + continue + + INSTRUMENTS.append(attribute) + +def get_instrument(image): + + container = list() + + for instrument in INSTRUMENTS: + try: + instrument.verify(image) + except Exception as e: + if not str(e): + e = format_exc().strip().split('\n')[-2].strip() + logging.debug(f'{instrument} : {e}') + else: + container.append(instrument) + + assert container, 'Instrument was not identified' + + if len(container) > 1: + msg = 'Data matched multiple instruments; ' + msg += ', '.join(map(str, container)) + logging.error(msg) + + return container[0] diff --git a/flows/instruments/instrument.py b/flows/instruments/instrument.py new file mode 100644 index 0000000..4281ebd --- /dev/null +++ b/flows/instruments/instrument.py @@ -0,0 +1,78 @@ +from inspect import getmro + +from astropy.time import Time +from astropy import units as u + +from .. import FILTERS +from ..api import get_site + +class MetaInstrument(type): + + def __new__(mcls, name, bases, attrs): + + all_attrs = dict() + for base in bases: + all_attrs.update(base.__dict__) + all_attrs.update(attrs) + + if not 'siteid' in all_attrs: + return super().__new__(mcls, name, bases, attrs) + + assert 'filters' in all_attrs, 'no filters variable' + assert not set(all_attrs['filters'].values()) - FILTERS, 'unknown filter(s)' + + assert 'verify' in all_attrs, 'no verify classmethod' + + return super().__new__(mcls, name, bases, attrs) + +class Instrument(metaclass=MetaInstrument): + + filter_keyword = 'filter' + exptime_keyword = 'exptime' + + peakmax = None + + scale = None + mirror = None + angle = None + + def __init__(self): + + site = get_site(self.siteid) + + self.name = site['sitename'] + self.longitude = site['longitude'] + self.latitude = site['latitude'] + self.elevation = site['elevation'] + self.location = site['EarthLocation'] + + def get_exptime(self, image): + + return float(image.hdr[self.exptime_keyword]) + + def get_obstime(self, image): + + if 'mjd-obs' in image.hdr: + obstime = Time(image.hdr['mjd-obs'], format='mjd', scale='utc', location=self.location) + elif 'date-obs' in image.hdr: + obstime = Time(image.hdr['date-obs'], format='isot', scale='utc', location=self.location) + else: + raise KeyError('mjd-obs nor date-obs in header') + + return obstime + + def get_filter(self, image): + + f = image.hdr[self.filter_keyword] + + assert f in self.filters, f'filter {f} not recognized' + + return self.filters[f] + + def get_mask(self, image): + + return None + + def __repr__(self): + + return self.name diff --git a/flows/instruments/lcogt.py b/flows/instruments/lcogt.py new file mode 100644 index 0000000..75c04d6 --- /dev/null +++ b/flows/instruments/lcogt.py @@ -0,0 +1,46 @@ +from astropy.time import Time +from astropy.coordinates import EarthLocation + +from .instrument import Instrument + +class LCOGT(Instrument): + + filters = { + 'B' : 'B', + } + + def get_obstime(self, image): + + lat, lon, height = image.hdr['latitude'], image.hdr['longitud'], image.hdr['height'] + location = EarthLocation.from_geodetic(lat, lon, height) + + return Time(image.hdr['mjd-obs'], format='mjd', scale='utc', location=location) + + def get_mask(self, image): + + return np.asarray(image.exthdus['BPM'].data, dtype=bool) + + @classmethod + def verify(cls, image): + + assert image.hdr['origin'] == 'LCOGT' + +class LCOGT_SAAO(LCOGT): + + siteid = 3 + + @classmethod + def verify(cls, image): + + LCOGT.verify(image) + assert image.hdr['site'] == 'LCOGT node at SAAO' + +class LCOGT_SSO(LCOGT): + + siteid = 6 + + @classmethod + def verify(cls, image): + + LCOGT.verify(image) + assert image.hdr['site'] == 'LCOGT node at Siding Spring Observatory' diff --git a/flows/instruments/liverpool.py b/flows/instruments/liverpool.py new file mode 100644 index 0000000..663733e --- /dev/null +++ b/flows/instruments/liverpool.py @@ -0,0 +1,29 @@ +from astropy.time import Time +from astropy import units as u + +from .instrument import Instrument + +class Liverpool(Instrument): + + siteid = 8 + + filters = { + 'Bessel-B' : 'B', + 'Bessell-B' : 'B', + 'V' : 'V' # XXX + } + + filter_keyword = 'filter'#1' XXX + + def get_obstime(self, image): + + obstime = super().get_obstime(image) + obstime += self.get_exptime(image) / 2 * u.second # Make time centre of exposure + + return obstime + + @classmethod + def verify(cls, image): + + return + assert image.hdr['telescop'] == 'Liverpool Telescope' diff --git a/flows/photometry.py b/flows/photometry.py index 1ea01d8..36da5ed 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -42,8 +42,7 @@ from .run_imagematch import run_imagematch from .zeropoint import bootstrap_outlier, sigma_from_Chauvenet from .wcs import force_reject_g2d, clean_with_rsq_and_get_fwhm, get_clean_references -from .coordinatematch import CoordinateMatch -from .fitting import MaskableLevMarLSQFitter +from .coordinatematch import CoordinateMatch, WCS from .fitscmd import get_fitscmd, maskstar, localseq, colorterm __version__ = get_version(pep440=False) @@ -207,30 +206,39 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi refs_coord = coords.SkyCoord(ra=references['ra'], dec=references['decl'], pm_ra_cosdec=references['pm_ra'], pm_dec=references['pm_dec'], unit='deg', frame='icrs', obstime=Time(2015.5, format='decimalyear')) - refs_coord = refs_coord.apply_space_motion(image.obstime) + head_wcs = str(WCS.from_astropy_wcs(image.wcs)) + logging.debug('Head WCS: %s', head_wcs) + references.meta['head_wcs'] = head_wcs + # Solve for new WCS cm = CoordinateMatch( - xy=list(zip(objects['x'], objects['y'])), - rd=list(zip(refs_coord.ra.deg, refs_coord.dec.deg)), - xy_order=np.argsort(-2.5 * np.log10(objects['flux'])), - rd_order=np.argsort(target_coord.separation(refs_coord)), - maximum_angle_distance=0.002, + xy = list(zip(objects['x'], objects['y'])), + rd = list(zip(refs_coord.ra.deg, refs_coord.dec.deg)), + xy_order = np.argsort(-2.5 * np.log10(objects['flux'])), + rd_order = np.argsort(target_coord.separation(refs_coord)), + xy_nmax = 200, rd_nmax = 200, + maximum_angle_distance = 0.002, ) try: - i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=timeoutpar))) + i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=float('inf')))) except TimeoutError: logging.warning('TimeoutError: No new WCS solution found') except StopIteration: logging.warning('StopIterationError: No new WCS solution found') else: + logging.info('Found new WCS') image.wcs = fit_wcs_from_points( np.array(list(zip(*cm.xy[i_xy]))), coords.SkyCoord(*map(list, zip(*cm.rd[i_rd])), unit='deg') ) + used_wcs = str(WCS.from_astropy_wcs(image.wcs)) + logging.debug('Used WCS: %s', used_wcs) + references.meta['used_wcs'] = used_wcs + # Calculate pixel-coordinates of references: row_col_coords = image.wcs.all_world2pix(np.array([[ref.ra.deg, ref.dec.deg] for ref in refs_coord]), 0) references['pixel_column'] = row_col_coords[:, 0] @@ -449,7 +457,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi group_maker=DAOGroup(fwhm), bkg_estimator=SExtractorBackground(), psf_model=epsf, - fitter=MaskableLevMarLSQFitter(), + fitter=fitting.LevMarLSQFitter(), fitshape=size, aperture_radius=fwhm ) @@ -663,6 +671,9 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # SAVE PHOTOMETRY # ============================================================================================== + # rename x, y columns to pixel_colum, pixel_row + #tab.rename_columns(('x', 'y'), ('pixel_column', 'pixel_row')) + # Descriptions of columns: tab['flux_aperture'].unit = u.count / u.second tab['flux_aperture_error'].unit = u.count / u.second diff --git a/flows/references.py b/flows/references.py new file mode 100644 index 0000000..385989d --- /dev/null +++ b/flows/references.py @@ -0,0 +1,75 @@ +from collections import OrderedDict + +import numpy as np + +from astropy.table import Table, TableColumns, Column + +class ReferenceColumns(TableColumns): + + def _set_image(self, image): + + self.image = image + + def keys(self): + + keys = list(super().keys()) + + if not hasattr(self, 'image'): + return keys + + keys += ['x'] if not 'x' in keys else [] + keys += ['y'] if not 'y' in keys else [] + + return keys + + def values(self): + + return [self[key] for key in self.keys()] + + def __len__(self): + + return len(self.keys()) + + def __iter__(self): + + return iter(key for key in self.keys()) + + def __getitem__(self, item): + + if item in ('x', 'y') and \ + {'ra', 'decl'} <= set(self.keys()) and \ + hasattr(self, 'image'): + + rd = list(zip(self['ra'], self['decl'])) + x, y = zip(*self.image.wcs.all_world2pix(rd, 0)) + + return Column({'x': x, 'y': y}[item], item) + + return super().__getitem__(item) + +class References(Table): + + def __init__(self, *args, **kwargs): + + super().__init__(*args, **kwargs) + + self.columns = ReferenceColumns(self.columns) + + def set_image(self, image): + + self.columns._set_image(image) + + def __getitem__(self, item): + + if item == 'xy': + return list(zip(self['x'], self['y'])) + + return super().__getitem__(item) + + def copy_with_image(self): + + table = self.copy() + del table['x'], table['y'] + table.set_image(self.columns.image) + + return table diff --git a/run_localweb.py b/run_localweb.py index fe02c47..72c1052 100644 --- a/run_localweb.py +++ b/run_localweb.py @@ -1,118 +1,47 @@ -import os, io - import numpy as np -import matplotlib.pyplot as plt -from flask import Flask, session; session = dict() # XXX -from base64 import b64encode as b64 -from astropy.io import ascii +from flask import Flask, request +from web import sites, datafiles, reference_stars -from flows import api, load_config +import json app = Flask(__name__) -#app.secret_key = os.urandom(24) - -@app.route('/') -def index(): - s = str() - target_by_name = {target['target_name']: target for target in api.get_targets()} - local_targets = os.listdir(load_config()['photometry']['archive_local']) - for target_name in sorted(target_by_name): - targetid = target_by_name[target_name]['targetid'] - s += f'{target_name} \n' if target_name in local_targets else f'{target_name} \n' - return s - -def lightcurve(targetid): - data = dict() - target = {target['targetid']: target for target in api.get_targets()}[targetid] - output = load_config()['photometry']['output'] - if not os.path.exists(output + '/%s' % (target['target_name'])): return '' - for fileid in os.listdir(output + '/%s' % (target['target_name'])): - f = output + '/%s/%s/photometry.ecsv' % (target['target_name'], fileid) - if not os.path.exists(f): continue - table = ascii.read(f) - filt, mjd = table.meta['photfilter'], table.meta['obstime-bmjd'] - if not filt in data: data[filt] = [] - mag, err, starid = table[0]['mag'], table[0]['mag_error'], table[0]['starid'] - data[filt].append((fileid, mjd, mag, err, starid)) - plt.figure(figsize=(20, 10)) - for filt in data: - fileid, mjd, mag, err, starid = map(np.array, zip(*data[filt])) - _ = plt.errorbar([], [], yerr=[], ls='', marker='o', label=filt) - plt.errorbar(mjd[starid==-1], mag[starid==-1], yerr=err[starid==-1], ls='', marker='o', color=_.lines[0].get_color()) - plt.errorbar(mjd[starid==0], mag[starid==0], yerr=err[starid==0], ls='', marker='s', color=_.lines[0].get_color()) - for i in range(len(fileid)): plt.text(mjd[i], mag[i], str(int(fileid[i])), fontsize=8) - plt.title(target['target_name']) - plt.xlabel('MJD'); plt.ylabel('Magnitude') - plt.legend() - plt.tight_layout() - ymin, ymax = min([min(list(zip(*d))[2]) for d in data.values()]), max([max(list(zip(*d))[2]) for d in data.values()]) - plt.ylim(ymin - (ymax-ymin)*0.05, ymax + (ymax-ymin)*0.05) - plt.gca().invert_yaxis() - png = io.BytesIO() - plt.savefig(png, format='png', dpi=100) - plt.close() - return b64(png.getvalue()).decode('utf-8') - -@app.route('/') -def target(targetid): - s = ''.format(lightcurve(targetid)) - target = {target['targetid']: target for target in api.get_targets()}[targetid] - archive = os.listdir(load_config()['photometry']['archive_local'] + '/%s' % target['target_name']) - output = load_config()['photometry']['output'] - fileids = list(map(str, api.get_datafiles(targetid, 'all'))) # XXX - for fileid in set(fileids) - set(session.keys()): session[fileid] = api.get_datafile(int(fileid)) - s += '' - for fileid in fileids: - s += ''.format(**session[fileid]) - if session[fileid]['path'].rsplit('/',1)[1] in archive: - if not session[fileid]['path'].rsplit('/',1)[1][:-8] + '.png' in archive: s += '' - else: s += f'' - else: s += '' - if not os.path.exists(output + '/%s/%0.5d/photometry.log' % (target['target_name'], int(fileid))): s += '' - else: s += f'' - if not os.path.exists(output + '/%s/%0.5d/photometry.ecsv' % (target['target_name'], int(fileid))): s += '' - else: s += f'' - s += '' - s += '
{fileid}{path}{obstime}{photfilter}IMGIMGLOGLOGPHOTPHOT
' - return s - -@app.route('///img') -def image(targetid, fileid): - s = str() - target = {target['targetid']: target for target in api.get_targets()}[targetid] - archive = load_config()['photometry']['archive_local'] - output = load_config()['photometry']['output'] - img = '%s/%s.png' % (archive, session[str(fileid)]['path'][:-8]) - with open(img, 'rb') as fd: img = b64(fd.read()).decode('utf-8') - s += f'' - if not os.path.exists(output + '/%s/%0.5d' % (target['target_name'], fileid)): return s - for f in sorted(os.listdir(output + '/%s/%0.5d' % (target['target_name'], fileid)))[::-1]: - if not '.' in f or not f.rsplit('.', 1)[1] == 'png': continue - img = output + '/%s/%0.5d/%s' % (target['target_name'], fileid, f) - with open(img, 'rb') as fd: img = b64(fd.read()).decode('utf-8') - s += f'' - return s - -@app.route('///phot') -def photometry(targetid, fileid): - target = {target['targetid']: target for target in api.get_targets()}[targetid] - output = load_config()['photometry']['output'] - phot = output + '/%s/%0.5d/photometry.ecsv' % (target['target_name'], fileid) - table = ascii.read(phot) - s = str(table.meta) + '

' - s += '' - for row in table: s += '' - s += '
' + ''.join(table.colnames) + '
' + ''.join(map(str, row)) + '
' - return s - -@app.route('///log') -def log(targetid, fileid): - target = {target['targetid']: target for target in api.get_targets()}[targetid] - output = load_config()['photometry']['output'] - log = output + '/%s/%0.5d/photometry.log' % (target['target_name'], fileid) - with open(log, 'r') as fd: return fd.read().replace('\n', '
') +@app.route('/api/sites.php') +def api_sites(): + if 'siteid' in request.args: + siteid = int(request.args['siteid']) + return sites[siteid] + for site in sites: + sites[site] = {**sites.site, **sites[site]} + return json.dumps(list(sites.values())) + +@app.route('/api/datafiles.php') +def api_datafiles(): + if 'fileid' in request.args: + fileid = int(request.args['fileid']) + for f in (f for target in datafiles for f in datafiles[target]): + if f['fileid'] != fileid: + continue + f.update({**datafiles.image, **f}) + if not f['diffimg'] is None: + f['diffimg'] = {**datafiles.diffimg, **f['diffimg']} + return f + elif 'targetid' in request.args: + targetid = int(request.args['targetid']) + return str([f['fileid'] for f in datafiles[targetid]]) + +@app.route('/api/reference_stars.php') +def api_reference_stars(): + targetid = int(request.args['target']) + reference_stars[targetid]['target'] = { + **reference_stars.target, **reference_stars[targetid]['target'] + } + for reference in reference_stars[targetid]['references']: + reference.update({**reference_stars.references, **reference}) + for avoid in reference_stars[targetid]['avoid']: + avoid.update({**reference_stars.avoid, **avoid}) + return reference_stars[targetid] if __name__ == '__main__': app.run(debug=True) diff --git a/web/__init__.py b/web/__init__.py new file mode 100644 index 0000000..ea7298a --- /dev/null +++ b/web/__init__.py @@ -0,0 +1,83 @@ +from .reference_stars import reference_stars +reference_stars = type('reference_stars', (dict,), { + 'target' : { + 'targetid' : -1, + 'target_name' : None, + 'target_status' : None, + 'ra' : None, + 'decl' : None, + 'redshift' : None, + 'redshift_error' : None, + 'discovery_mag' : None, + 'catalog_downloaded' : None, + 'pointing_model_created' : '1970-01-01 00:00:00.0', + 'inserted' : '1970-01-01 00:00:00.0', + 'discovery_date' : '1970-01-01 00:00:00.0', + 'project' : None, + 'host_galaxy' : None, + 'ztf_id' : None + }, + 'references' : { + 'starid' : -1, + 'ra' : None, + 'decl' : None, + 'pm_ra' : 0, + 'pm_dec' : 0, + 'gaia_mag' : None, + 'gaia_bp_mag' : None, + 'gaia_rp_mag' : None, + 'J_mag' : None, + 'H_mag' : None, + 'K_mag' : None, + 'g_mag' : None, + 'r_mag' : None, + 'i_mag' : None, + 'z_mag' : None, + 'gaia_variability' : 0, + 'V_mag' : None, + 'B_mag' : None, + 'u_mag' : None, + 'distance' : None + } +})(reference_stars) +reference_stars.avoid = reference_stars.references + +from .datafiles import datafiles +datafiles = type('datafiles', (dict,), { + 'image' : { + 'fileid' : None, + 'path' : None, + 'targetid' : None, + 'site' : None, + 'filesize' : None, + 'filehash' : None, + 'inserted' : '1970-01-01 00:00:00.0', + 'lastmodified' : '1970-01-01 00:00:00.0', + 'photfilter' : None, + 'obstime' : None, + 'exptime' : None, + 'version' : None, + 'archive_path' : None, + 'target_name' : None, + 'template' : None, + 'diffimg' : None + }, + 'diffimg' : { + 'fileid' : None, + 'path' : None, + 'filehash' : None, + 'filesize' : None + } +})(datafiles) + +from .sites import sites +sites = type('sites', (dict,), { + 'site' : { + 'siteid' : -1, + 'sitename' : None, + 'longitude': None, + 'latitude' : None, + 'elevation' : None, + 'site_keyword' : None, + } +})(sites) diff --git a/web/datafiles.py b/web/datafiles.py new file mode 100644 index 0000000..b65211a --- /dev/null +++ b/web/datafiles.py @@ -0,0 +1,12 @@ +datafiles = { + 248 : [ + { + 'fileid' : 0, + 'path' : '2016adj/AT2016adj_r_SWO_NC_2016_04_30.fits.gz', + 'diffimg' : { + 'fileid' : 1, + 'path' : '2016adj/subtracted/AT2016adj_r_SWO_NC_2016_04_30diff.fits.gz', + } + } + ] +} diff --git a/web/reference_stars.py b/web/reference_stars.py new file mode 100644 index 0000000..7683baf --- /dev/null +++ b/web/reference_stars.py @@ -0,0 +1,26 @@ +reference_stars = { + 248 : { + 'target' : { + 'ra' : 201.350458, + 'decl' : -43.015972, + 'inserted': '2020-12-15 12:30:27.694425', + 'discovery_date': '2016-01-29 08:00:00', + }, + 'references' : [ + { + 'ra' : 201.35558532, + 'decl' : -43.02340329, + 'B_mag' : 0, + 'V_mag' : 0, + }, + ], + 'avoid' : [ + { + 'ra' : 201.35558532, + 'decl' : -43.02340329, + 'B_mag' : 0, + 'V_mag' : 0, + }, + ] + } +} diff --git a/web/sites.py b/web/sites.py new file mode 100644 index 0000000..5b6e0d8 --- /dev/null +++ b/web/sites.py @@ -0,0 +1,10 @@ +sites = { + 1 : { + 'siteid' : 1, + 'sitename' : 'LCOGT at McDonald', + 'longitude': -104.015173, + 'latitude' : 30.679833, + 'elevation' : 2030, + 'site_keyword' : 'LCOGT node at McDonald Observatory', + }, +} From 290c3f5aa29691521f8284dcc704f5b26d3c6d74 Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 3 Mar 2021 10:01:17 +0100 Subject: [PATCH 31/43] remove explicit centroid_com --- flows/photometry.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/flows/photometry.py b/flows/photometry.py index 36da5ed..2fd9fd8 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -30,7 +30,6 @@ from photutils.psf import EPSFBuilder, EPSFFitter, BasicPSFPhotometry, DAOGroup, extract_stars from photutils import Background2D, SExtractorBackground, MedianBackground from photutils.utils import calc_total_error -from photutils.centroids import centroid_com from scipy.interpolate import UnivariateSpline @@ -368,7 +367,6 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi maxiters=500, fitter=EPSFFitter(fit_boxsize=np.round(2 * fwhm, 0)), progress_bar=True, - recentering_func=centroid_com )(stars)[0] logger.info('Successfully built PSF model') From d2cc0d547bd9eb23592c71c9478c2f7092cbaf8d Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 3 Mar 2021 13:20:02 +0100 Subject: [PATCH 32/43] nan to 0 --- flows/load_image.py | 2 +- flows/photometry.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/flows/load_image.py b/flows/load_image.py index 09bdbb0..63b8440 100644 --- a/flows/load_image.py +++ b/flows/load_image.py @@ -304,7 +304,7 @@ def load_image(FILENAME): raise Exception("Could not determine origin of image") # Create masked version of image: - image.image[image.mask] = np.NaN + image.image[image.mask] = 0.0 image.clean = np.ma.masked_array(image.image, image.mask) return image diff --git a/flows/photometry.py b/flows/photometry.py index 2fd9fd8..e110cb3 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -187,6 +187,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Use sep to for soure extraction image.sepdata = image.image.byteswap().newbyteorder() + #image.setpdate = np.asrray(image.image image.sepbkg = sep.Background(image.sepdata, mask=image.mask) image.sepsub = image.sepdata - image.sepbkg logger.debug('sub: {} bkg_rms: {} mask: {}'.format(np.shape(image.sepsub), np.shape(image.sepbkg.globalrms), @@ -208,7 +209,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi refs_coord = refs_coord.apply_space_motion(image.obstime) head_wcs = str(WCS.from_astropy_wcs(image.wcs)) - logging.debug('Head WCS: %s', head_wcs) + logger.debug('Head WCS: %s', head_wcs) references.meta['head_wcs'] = head_wcs # Solve for new WCS @@ -224,18 +225,18 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi try: i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=float('inf')))) except TimeoutError: - logging.warning('TimeoutError: No new WCS solution found') + logger.warning('TimeoutError: No new WCS solution found') except StopIteration: - logging.warning('StopIterationError: No new WCS solution found') + logger.warning('StopIterationError: No new WCS solution found') else: - logging.info('Found new WCS') + logger.info('Found new WCS') image.wcs = fit_wcs_from_points( np.array(list(zip(*cm.xy[i_xy]))), coords.SkyCoord(*map(list, zip(*cm.rd[i_rd])), unit='deg') ) used_wcs = str(WCS.from_astropy_wcs(image.wcs)) - logging.debug('Used WCS: %s', used_wcs) + logger.debug('Used WCS: %s', used_wcs) references.meta['used_wcs'] = used_wcs # Calculate pixel-coordinates of references: @@ -363,9 +364,9 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Build the ePSF: epsf = EPSFBuilder( - oversampling=1.0, + oversampling=1, maxiters=500, - fitter=EPSFFitter(fit_boxsize=np.round(2 * fwhm, 0)), + fitter=EPSFFitter(fit_boxsize=np.round(2 * fwhm, 0).astype(int)), progress_bar=True, )(stars)[0] @@ -453,7 +454,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Create photometry object: photometry_obj = BasicPSFPhotometry( group_maker=DAOGroup(fwhm), - bkg_estimator=SExtractorBackground(), + bkg_estimator=MedianBackground(), psf_model=epsf, fitter=fitting.LevMarLSQFitter(), fitshape=size, From 2ccb698ec96f464d137778374ed33face06f33cd Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 3 Mar 2021 13:29:56 +0100 Subject: [PATCH 33/43] byte order --- flows/photometry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flows/photometry.py b/flows/photometry.py index e110cb3..8320fbf 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -186,7 +186,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi image.error = calc_total_error(image.clean, bkg.background_rms, 1.0) # Use sep to for soure extraction - image.sepdata = image.image.byteswap().newbyteorder() + image.sepdata = np.asarray(image.image) #image.setpdate = np.asrray(image.image image.sepbkg = sep.Background(image.sepdata, mask=image.mask) image.sepsub = image.sepdata - image.sepbkg From 512b47816ec8b282661222916e51e6b06ed1727c Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 3 Mar 2021 13:39:27 +0100 Subject: [PATCH 34/43] byte swap --- flows/load_image.py | 2 +- flows/photometry.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/flows/load_image.py b/flows/load_image.py index 63b8440..ba9a8f8 100644 --- a/flows/load_image.py +++ b/flows/load_image.py @@ -304,7 +304,7 @@ def load_image(FILENAME): raise Exception("Could not determine origin of image") # Create masked version of image: - image.image[image.mask] = 0.0 + image.image[image.mask] = np.nan image.clean = np.ma.masked_array(image.image, image.mask) return image diff --git a/flows/photometry.py b/flows/photometry.py index 8320fbf..58ec5b6 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -186,8 +186,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi image.error = calc_total_error(image.clean, bkg.background_rms, 1.0) # Use sep to for soure extraction - image.sepdata = np.asarray(image.image) - #image.setpdate = np.asrray(image.image + image.sepdata = np.asarray(image.image,dtype=np.float64) image.sepbkg = sep.Background(image.sepdata, mask=image.mask) image.sepsub = image.sepdata - image.sepbkg logger.debug('sub: {} bkg_rms: {} mask: {}'.format(np.shape(image.sepsub), np.shape(image.sepbkg.globalrms), From f111c4d7184f79ed199f9e0c6d9ef573e1e7da84 Mon Sep 17 00:00:00 2001 From: Emir Date: Thu, 4 Mar 2021 18:54:05 +0100 Subject: [PATCH 35/43] Endian Fix for sep --- flows/photometry.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flows/photometry.py b/flows/photometry.py index 58ec5b6..bce310d 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -186,7 +186,11 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi image.error = calc_total_error(image.clean, bkg.background_rms, 1.0) # Use sep to for soure extraction - image.sepdata = np.asarray(image.image,dtype=np.float64) + image.sepdata = image.image.byteswap().newbyteorder() + if image.sepdata.dtype.byteorder != '<': + image.sepdata = np.asarray(image.image, dtype=np.float64) + + #image.setpdate = np.asrray(image.image image.sepbkg = sep.Background(image.sepdata, mask=image.mask) image.sepsub = image.sepdata - image.sepbkg logger.debug('sub: {} bkg_rms: {} mask: {}'.format(np.shape(image.sepsub), np.shape(image.sepbkg.globalrms), From 99fe2871aacff776beb76e3ec0788154982a6ace Mon Sep 17 00:00:00 2001 From: Simon Holmbo Date: Tue, 23 Mar 2021 18:01:26 +0100 Subject: [PATCH 36/43] week12 --- .gitignore | 3 +- flows/api/targets.py | 6 +- flows/epsfbuilder/__init__.py | 2 + flows/epsfbuilder/epsfbuilder.py | 131 ++++++++++++++++++ flows/epsfbuilder/gaussian_kernel.py | 12 ++ flows/fitscmd.py | 11 +- flows/image/image.py | 5 + flows/load_image.py | 15 ++- flows/photometry.py | 176 ++++++++++++------------ flows/wcs.py | 7 +- run_fitscmd.py | 2 + run_localweb.py | 194 ++++++++++++++++++++++----- run_photometry.py | 5 +- web/__init__.py | 83 ------------ web/api/__init__.py | 86 ++++++++++++ web/api/catalogs.json | 45 +++++++ web/api/datafiles.json | 15 +++ web/api/sites.json | 7 + web/api/targets.json | 8 ++ web/datafile.html | 152 +++++++++++++++++++++ web/datafiles.py | 12 -- web/index.html | 100 ++++++++++++++ web/photometry.js | 31 +++++ web/reference_stars.py | 26 ---- web/sites.py | 10 -- web/static/README.md | 1 + web/target.html | 92 +++++++++++++ 27 files changed, 977 insertions(+), 260 deletions(-) create mode 100644 flows/epsfbuilder/__init__.py create mode 100644 flows/epsfbuilder/epsfbuilder.py create mode 100644 flows/epsfbuilder/gaussian_kernel.py delete mode 100644 web/__init__.py create mode 100644 web/api/__init__.py create mode 100644 web/api/catalogs.json create mode 100644 web/api/datafiles.json create mode 100644 web/api/sites.json create mode 100644 web/api/targets.json create mode 100644 web/datafile.html delete mode 100644 web/datafiles.py create mode 100644 web/index.html create mode 100644 web/photometry.js delete mode 100644 web/reference_stars.py delete mode 100644 web/sites.py create mode 100644 web/static/README.md create mode 100644 web/target.html diff --git a/.gitignore b/.gitignore index efd2361..0803d48 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ flows/config.ini flows/casjobs/CasJobs.config +web/static/js9 # Byte-compiled / optimized / DLL files __pycache__/ @@ -138,4 +139,4 @@ dmypy.json .vscode/ # OSX stuff: -.DS_Store \ No newline at end of file +.DS_Store diff --git a/flows/api/targets.py b/flows/api/targets.py index be8f5e4..8afffa2 100644 --- a/flows/api/targets.py +++ b/flows/api/targets.py @@ -20,11 +20,12 @@ def get_target(target): # Get API token from config file: config = load_config() + address = config.get('api', 'address', fallback=None) token = config.get('api', 'token', fallback=None) if token is None: raise Exception("No API token has been defined") - r = requests.get('https://flows.phys.au.dk/api/targets.php', + r = requests.get('%s/targets.php' % address, params={'target': target}, headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() @@ -43,11 +44,12 @@ def get_targets(): # Get API token from config file: config = load_config() + address = config.get('api', 'address', fallback=None) token = config.get('api', 'token', fallback=None) if token is None: raise Exception("No API token has been defined") - r = requests.get('https://flows.phys.au.dk/api/targets.php', + r = requests.get('%s/targets.php' % address, headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() jsn = r.json() diff --git a/flows/epsfbuilder/__init__.py b/flows/epsfbuilder/__init__.py new file mode 100644 index 0000000..6a86ded --- /dev/null +++ b/flows/epsfbuilder/__init__.py @@ -0,0 +1,2 @@ +from .epsfbuilder import EPSFBuilder +from .gaussian_kernel import gaussian_kernel diff --git a/flows/epsfbuilder/epsfbuilder.py b/flows/epsfbuilder/epsfbuilder.py new file mode 100644 index 0000000..21e758a --- /dev/null +++ b/flows/epsfbuilder/epsfbuilder.py @@ -0,0 +1,131 @@ +#import sys +import time + +import numpy as np + +#from scipy.ndimage import gaussian_filter + +#from scipy.spatial import cKDTree +from scipy.interpolate import griddata + +import photutils.psf + +class EPSFBuilder(photutils.psf.EPSFBuilder): + + def _create_initial_epsf(self, stars): + + epsf = super()._create_initial_epsf(stars) + epsf.origin = None + + X, Y = np.meshgrid(*map(np.arange, epsf.shape[::-1])) + + X = X / epsf.oversampling[0] - epsf.x_origin + Y = Y / epsf.oversampling[1] - epsf.y_origin + + self._epsf_xy_grid = X, Y + + return epsf + + def _resample_residual(self, star, epsf): + + #max_dist = .5 / np.sqrt(np.sum(np.power(epsf.oversampling, 2))) + + #star_points = list(zip(star._xidx_centered, star._yidx_centered)) + #epsf_points = list(zip(*map(np.ravel, self._epsf_xy_grid))) + + #star_tree = cKDTree(star_points) + #dd, ii = star_tree.query(epsf_points, distance_upper_bound=max_dist) + #mask = np.isfinite(dd) + + #star_data = np.full_like(epsf.data, np.nan) + #star_data.ravel()[mask] = star._data_values_normalized[ii[mask]] + + star_points = list(zip(star._xidx_centered, star._yidx_centered)) + star_data = griddata(star_points, star._data_values_normalized, self._epsf_xy_grid) + + return star_data - epsf._data + +# def _resample_residuals(self, stars, epsf): +# +# residuals = super()._resample_residuals(stars, epsf) +# +# import matplotlib.pyplot as plt +# +# if epsf.data.any(): +# +# #i = np.isfinite(residuals).sum(axis=0, dtype=bool) +# #data = epsf.data - gaussian_filter(epsf.data, sigma=1) +# #factor = np.nanstd(data) / np.std(np.nanmedian(residuals[:,i], axis=0)) +# #residuals *= min(max(factor, 0.1), 1.0) +# +# data = np.nanmedian(residuals, axis=0) +# self._.set_data(data) +# self._.set_clim([np.nanmin(data), np.nanmax(data)]) +# self._.cb.draw_all() +# +# else: +# +# data = np.nanmedian(residuals, axis=0) +# self._ = plt.imshow(data) +# self._.set_clim([np.nanmin(data), np.nanmax(data)]) +# self._.cb = plt.gcf().colorbar(self._) +# plt.show(block=False) +# +# plt.gcf().canvas.draw() +# time.sleep(1) +# +# return residuals + +# def _recenter_epsf(self, epsf, *args, **kwargs): +# +# if not hasattr(self, 'dx_total'): +# +# self.dx_total = [] +# self.dy_total = [] +# +# def profile(frame, event, arg): +# +# global x, y, xcenter, ycenter +# +# if event == "return" and frame.f_code.co_name == '_recenter_epsf': +# +# x, xcenter = frame.f_locals['x'], frame.f_locals['xcenter'] +# y, ycenter = frame.f_locals['y'], frame.f_locals['ycenter'] +# +# self.dx_total.append(frame.f_locals['dx_total']) +# self.dy_total.append(frame.f_locals['dy_total']) +# +# return profile +# +# sys.setprofile(profile) +# super()._recenter_epsf(epsf, *args, **kwargs) +# sys.setprofile(None) +# +# dx_total = np.mean(self.dx_total[-2:], axis=0) +# dy_total = np.mean(self.dy_total[-2:], axis=0) +# +# epsf_data = epsf.evaluate(x=x, y=y, flux=1.0, +# x_0=xcenter - .5*dx_total, +# y_0=ycenter - .5*dy_total) +# +# return epsf_data + + def __call__(self, *args, **kwargs): + + #import matplotlib + #_backend = matplotlib.get_backend() + #matplotlib.pyplot.switch_backend('TkAgg') + + t0 = time.time() + + epsf, stars = super().__call__(*args, **kwargs) + + epsf.fit_info = dict( + n_iter = len(self._epsf), + max_iters = self.maxiters, + time = time.time() - t0, + ) + + #matplotlib.pyplot.switch_backend(_backend) + + return epsf, stars diff --git a/flows/epsfbuilder/gaussian_kernel.py b/flows/epsfbuilder/gaussian_kernel.py new file mode 100644 index 0000000..0d9cf47 --- /dev/null +++ b/flows/epsfbuilder/gaussian_kernel.py @@ -0,0 +1,12 @@ +import numpy as np + +from scipy.stats import norm + +def gaussian_kernel(kernlen, nsig): + """Returns a 2D Gaussian kernel.""" + + x = np.linspace(-nsig, nsig, kernlen+1) + kern1d = np.diff(norm.cdf(x)) + kern2d = np.outer(kern1d, kern1d) + + return kern2d / kern2d.sum() diff --git a/flows/fitscmd.py b/flows/fitscmd.py index c0d04a5..8c8bd68 100644 --- a/flows/fitscmd.py +++ b/flows/fitscmd.py @@ -7,17 +7,17 @@ COMMANDS = { 'maskstar': ( # mask star for galaxy subtracted psf photometry - '(\d+(?:\.\d*)?),\s?([+-]?\d+(?:\.\d*)?)', # parse + r'(\d+(?:\.\d*)?),\s?([+-]?\d+(?:\.\d*)?)', # parse lambda ra, dec: (float(ra), float(dec)), # convert lambda ra, dec, hdul: 0 <= ra < 360 and -90 <= dec <= 90 # validate ), 'localseq': ( # specifiy hdu name with custom local sequence - '(.+)', # parse + r'(.+)', # parse lambda lsqhdu: (lsqhdu.upper(),), # convert lambda lsqhdu, hdul: lsqhdu in [hdu.name for hdu in hdul], # validate ), 'colorterm': ( # color term for moving references to another system - '([+-]?\d+(?:\.\d*)?)\s?\((.+)-(.+)\)', # parse + r'\\?([+-]?\d+(?:\.\d*)?)\s?\((.+)-(.+)\)', # parse lambda cterm, A_mag, B_mag: (float(cterm), A_mag, B_mag), # convert lambda cterm, A_mag, B_mag, hdul: True, # validate ) @@ -31,11 +31,11 @@ def maskstar(data, wcs, stars, fwhm): data = data.copy() if not hasattr(data, 'mask'): data = np.ma.array(data, mask=np.zeros_like(data)) - + X, Y = np.meshgrid(*map(np.arange, data.shape[::-1])) for x, y in wcs.all_world2pix(stars, 0): - i = np.where(((X-x)**2 + (Y-y)**2 < fwhm**2)) + i = np.where(((X-x)**2 + (Y-y)**2 < (1.5*fwhm)**2)) data.mask[i] = True return data @@ -64,6 +64,7 @@ def colorterm(ref_filter, colorterms, references): return colorterm, A_mag, B_mag = colorterms[-1] + A_mag, B_mag = A_mag + '_mag', B_mag + '_mag' # XXX remove after 2016adj if not {A_mag, B_mag} <= set(references.colnames): missing = ', '.join({A_mag, B_mag} - set(references.colnames)) diff --git a/flows/image/image.py b/flows/image/image.py index e8f6cbb..41e67d0 100644 --- a/flows/image/image.py +++ b/flows/image/image.py @@ -69,6 +69,11 @@ def set_instrument(self, instrument): def add_mask(self, mask): + assert callable(mask) or np.shape(mask) = self._data.shape + + if not callable(mask): + mask = lambda img: mask + i = max(self._lmasks.keys()) + 1 if len(self._lmasks) else 0 self._lmasks[i] = mask diff --git a/flows/load_image.py b/flows/load_image.py index 09bdbb0..f03b947 100644 --- a/flows/load_image.py +++ b/flows/load_image.py @@ -82,7 +82,7 @@ def load_image(FILENAME): telescope = hdr.get('TELESCOP') instrument = hdr.get('INSTRUME') - image.image = np.asarray(hdul[0].data) + image.image = np.asarray(hdul[0].data, dtype=np.float64) image.shape = image.image.shape image.head = hdr @@ -111,7 +111,8 @@ def load_image(FILENAME): image.site = site_keywords.get(hdr['SITE'], None) observatory = coords.EarthLocation.from_geodetic(lat=hdr['LATITUDE'], lon=hdr['LONGITUD'], height=hdr['HEIGHT']) - image.obstime = Time(hdr['MJD-OBS'], format='mjd', scale='utc', location=observatory) + image.obstime = Time(hdr['DATE-OBS'], format='isot', scale='utc', location=observatory) + image.obstime += 0.5*image.exptime * u.second # Make time centre of exposure image.photfilter = hdr['FILTER'] @@ -300,11 +301,19 @@ def load_image(FILENAME): }.get(hdr['FILTER'], hdr['FILTER']) image.exptime = float(hdr['FULL_EXP']) + elif instrument == 'OMEGACAM' and (origin == 'ESO' or origin.startswith('NOAO-IRAF')): + image.site = api.get_site(18) # Hard-coded the siteid for ESO VLT Survey telescope + image.obstime = Time(hdr['MJD-OBS'], format='mjd', scale='utc', location=image.site['EarthLocation']) + image.obstime += 0.5*image.exptime * u.second # Make time centre of exposure + image.photfilter = { + 'i_SDSS': 'ip' + }.get(hdr['ESO INS FILT1 NAME'], hdr['ESO INS FILT1 NAME']) + else: raise Exception("Could not determine origin of image") # Create masked version of image: - image.image[image.mask] = np.NaN + #image.image[image.mask] = np.NaN image.clean = np.ma.masked_array(image.image, image.mask) return image diff --git a/flows/photometry.py b/flows/photometry.py index 36da5ed..3ee625d 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -15,7 +15,7 @@ import logging import warnings -from astropy.utils.exceptions import AstropyDeprecationWarning +from astropy.utils.exceptions import AstropyDeprecationWarning, AstropyUserWarning import astropy.units as u import astropy.coordinates as coords from astropy.stats import sigma_clip, SigmaClip @@ -27,7 +27,7 @@ warnings.simplefilter('ignore', category=AstropyDeprecationWarning) from photutils import CircularAperture, CircularAnnulus, aperture_photometry -from photutils.psf import EPSFBuilder, EPSFFitter, BasicPSFPhotometry, DAOGroup, extract_stars +from photutils.psf import EPSFFitter, BasicPSFPhotometry, DAOGroup, extract_stars from photutils import Background2D, SExtractorBackground, MedianBackground from photutils.utils import calc_total_error from photutils.centroids import centroid_com @@ -44,6 +44,7 @@ from .wcs import force_reject_g2d, clean_with_rsq_and_get_fwhm, get_clean_references from .coordinatematch import CoordinateMatch, WCS from .fitscmd import get_fitscmd, maskstar, localseq, colorterm +from .epsfbuilder import EPSFBuilder, gaussian_kernel __version__ = get_version(pep440=False) @@ -122,6 +123,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi 'zp': 'z_mag', 'B': 'B_mag', 'V': 'V_mag', + #'Y': 'Y_mag', 'J': 'J_mag', 'H': 'H_mag', 'K': 'K_mag', @@ -165,35 +167,31 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Estimate image background: # Not using image.clean here, since we are redefining the mask anyway - bkg = Background2D(image.clean, (128, 128), filter_size=(5, 5), + background = Background2D(image.clean, (128, 128), filter_size=(5, 5), sigma_clip=SigmaClip(sigma=3.0), bkg_estimator=SExtractorBackground(), exclude_percentile=50.0) - image.background = bkg.background - image.std = bkg.background_rms_median # Create background-subtracted image: - image.subclean = image.clean - image.background + image.subclean = image.clean - background.background # Plot background estimation: fig, ax = plt.subplots(1, 3, figsize=(20, 6)) plot_image(image.clean, ax=ax[0], scale='log', title='Original') - plot_image(image.background, ax=ax[1], scale='log', title='Background') + plot_image(background.background, ax=ax[1], scale='log', title='Background') plot_image(image.subclean, ax=ax[2], scale='log', title='Background subtracted') fig.savefig(os.path.join(output_folder, 'background.png'), bbox_inches='tight') plt.close(fig) # TODO: Is this correct?! - image.error = calc_total_error(image.clean, bkg.background_rms, 1.0) + image.error = calc_total_error(image.clean, background.background_rms, 1.0) # Use sep to for soure extraction - image.sepdata = image.image.byteswap().newbyteorder() - image.sepbkg = sep.Background(image.sepdata, mask=image.mask) - image.sepsub = image.sepdata - image.sepbkg - logger.debug('sub: {} bkg_rms: {} mask: {}'.format(np.shape(image.sepsub), np.shape(image.sepbkg.globalrms), - np.shape(image.mask))) - objects = sep.extract(image.sepsub, thresh=5., err=image.sepbkg.globalrms, mask=image.mask, - deblend_cont=0.1, minarea=9, clean_param=2.0) + sep_background = sep.Background(image.clean.data, mask=image.mask) + logger.debug('sub: {} bkg_rms: {} mask: {}'.format(image.shape, np.shape(sep_background.globalrms), + image.shape)) + objects = sep.extract(image.clean.data - sep_background, thresh=5., err=sep_background.globalrms, + mask=image.mask, deblend_cont=0.1, minarea=9, clean_param=2.0) # ============================================================================================== # DETECTION OF STARS AND MATCHING WITH CATALOG @@ -208,28 +206,56 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi unit='deg', frame='icrs', obstime=Time(2015.5, format='decimalyear')) refs_coord = refs_coord.apply_space_motion(image.obstime) + # @TODO: These need to be based on the instrument! + radius = 10 + fwhm_guess = 6.0 + fwhm_min = 3.5 + fwhm_max = 18.0 + + # Clean extracted stars + masked_sep_xy, sep_mask, masked_sep_rsqs = force_reject_g2d(objects['x'], objects['y'], image, get_fwhm=False, + radius=radius, fwhm_guess=fwhm_guess, rsq_min=0.3, + fwhm_max=fwhm_max, fwhm_min=fwhm_min) + + # XXX +# try: +# _references = catalog['references'] +# _references.sort(ref_filter) +# replace(_references['pm_ra'], np.NaN, 0) +# replace(_references['pm_dec'], np.NaN, 0) +# _refs_coord = coords.SkyCoord(ra=_references['ra'], dec=_references['decl'], +# pm_ra_cosdec=_references['pm_ra'], pm_dec=_references['pm_dec'], +# unit='deg', frame='icrs', obstime=Time(2015.5, format='decimalyear')) +# _refs_coord = _refs_coord.apply_space_motion(image.obstime) +# if allnan(_references[ref_filter]): +# raise ValueError("No _reference stars found in current photfilter.") +# except Exception as e: +# logging.warning(e) +# _refs_coord = refs_coord + # XXX + head_wcs = str(WCS.from_astropy_wcs(image.wcs)) logging.debug('Head WCS: %s', head_wcs) references.meta['head_wcs'] = head_wcs # Solve for new WCS cm = CoordinateMatch( - xy = list(zip(objects['x'], objects['y'])), + xy = list(masked_sep_xy[sep_mask]), rd = list(zip(refs_coord.ra.deg, refs_coord.dec.deg)), - xy_order = np.argsort(-2.5 * np.log10(objects['flux'])), + xy_order = np.argsort(np.power(masked_sep_xy[sep_mask] - np.array(image.shape[::-1])/2, 2).sum(axis=1)), rd_order = np.argsort(target_coord.separation(refs_coord)), - xy_nmax = 200, rd_nmax = 200, + xy_nmax = 100, rd_nmax = 100, maximum_angle_distance = 0.002, ) try: i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=float('inf')))) except TimeoutError: - logging.warning('TimeoutError: No new WCS solution found') + logger.warning('TimeoutError: No new WCS solution found') except StopIteration: - logging.warning('StopIterationError: No new WCS solution found') + logger.warning('StopIterationError: No new WCS solution found') else: - logging.info('Found new WCS') + logger.info('Found new WCS') image.wcs = fit_wcs_from_points( np.array(list(zip(*cm.xy[i_xy]))), coords.SkyCoord(*map(list, zip(*cm.rd[i_rd])), unit='deg') @@ -240,33 +266,19 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi references.meta['used_wcs'] = used_wcs # Calculate pixel-coordinates of references: - row_col_coords = image.wcs.all_world2pix(np.array([[ref.ra.deg, ref.dec.deg] for ref in refs_coord]), 0) - references['pixel_column'] = row_col_coords[:, 0] - references['pixel_row'] = row_col_coords[:, 1] - - # Calculate the targets position in the image: - target_pixel_pos = image.wcs.all_world2pix([[target['ra'], target['decl']]], 0)[0] + xy = image.wcs.all_world2pix(list(zip(refs_coord.ra.deg, refs_coord.dec.deg)), 0) + references['pixel_column'], references['pixel_row'] = x, y = list(map(np.array, zip(*xy))) # Clean out the references: hsize = 10 - x = references['pixel_column'] - y = references['pixel_row'] clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] # & (references[ref_filter] < ref_mag_limit) - assert len(clean_references), 'No references in field' + assert len(clean_references), 'No clean references in field' - # @TODO: These need to be based on the instrument! - radius = 10 - fwhm_guess = 6.0 - fwhm_min = 3.5 - fwhm_max = 18.0 - - # Clean extracted stars - masked_sep_xy, sep_mask, masked_sep_rsqs = force_reject_g2d(objects['x'], objects['y'], image, get_fwhm=False, - radius=radius, fwhm_guess=fwhm_guess, rsq_min=0.3, - fwhm_max=fwhm_max, fwhm_min=fwhm_min) + # Calculate the targets position in the image: + target_pixel_pos = image.wcs.all_world2pix([(target['ra'], target['decl'])], 0)[0] # Clean reference star locations masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], @@ -300,7 +312,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Final clean of wcs corrected references logger.info("Number of references before final cleaning: %d", len(clean_references)) - logger.info('masked R^2 values: {}'.format(masked_rsqs[rsq_mask])) + logger.debug('masked R^2 values: {}'.format(masked_rsqs[rsq_mask])) references = get_clean_references(clean_references, masked_rsqs, rsq_ideal=0.8) logger.info("Number of references after final cleaning: %d", len(references)) @@ -320,58 +332,51 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Make cutouts of stars using extract_stars: # Scales with FWHM size = int(np.round(29 * fwhm / 6)) - if size % 2 == 0: - size += 1 # Make sure it's a uneven number + size += 0 if size % 2 else 1 # Make sure it's a uneven number size = max(size, 15) # Never go below 15 pixels - hsize = (size - 1) / 2 # higher hsize than before to do more aggressive edge masking. - - x = references['pixel_column'] - y = references['pixel_row'] - mask_near_edge = ((x > hsize) & (x < (image.shape[1] - 1 - hsize)) - & (y > hsize) & (y < (image.shape[0] - 1 - hsize))) - stars_for_epsf = Table() - stars_for_epsf['x'] = x[mask_near_edge] - stars_for_epsf['y'] = y[mask_near_edge] + # Extract stars sub-images: + xy = [tuple(masked_ref_xys[clean_references['starid'] == ref['starid']].data[0]) for ref in references] # FIXME !!! + with warnings.catch_warnings(): + warnings.simplefilter('ignore', AstropyUserWarning) + stars = extract_stars( + NDData(data=image.subclean, mask=image.mask), + Table(np.array(xy), names=('x', 'y')), + size = size + 6#2*size+1 # +6 for edge buffer + ) # Store which stars were used in ePSF in the table: - logger.info("Number of stars used for ePSF: %d", len(stars_for_epsf)) - references['used_for_epsf'] = mask_near_edge - - # Extract stars sub-images: - stars = extract_stars( - NDData(data=image.subclean, mask=image.mask), - stars_for_epsf, - size=size - ) + references['used_for_epsf'] = False + references['used_for_epsf'][[star.id_label-1 for star in stars]] = True + logger.info("Number of stars used for ePSF: %d", len(stars)) # Plot the stars being used for ePSF: - nrows = 5 - ncols = 5 imgnr = 0 - for k in range(int(np.ceil(len(stars_for_epsf) / (nrows * ncols)))): + nrows, ncols = 5, 5 + for k in range(int(np.ceil(len(stars) / (nrows * ncols)))): fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True) ax = ax.ravel() for i in range(nrows * ncols): - if imgnr > len(stars_for_epsf) - 1: + if imgnr > len(stars) - 1: ax[i].axis('off') else: - plot_image(stars[imgnr], ax=ax[i], scale='log', cmap='viridis') + offset_axes = stars[imgnr].bbox.ixmin, stars[imgnr].bbox.iymin + plot_image(stars[imgnr], ax=ax[i], scale='log', cmap='viridis')#, offset_axes=offset_axes) imgnr += 1 fig.savefig(os.path.join(output_folder, 'epsf_stars%02d.png' % (k + 1)), bbox_inches='tight') plt.close(fig) # Build the ePSF: - epsf = EPSFBuilder( - oversampling=1.0, - maxiters=500, - fitter=EPSFFitter(fit_boxsize=np.round(2 * fwhm, 0)), - progress_bar=True, - recentering_func=centroid_com - )(stars)[0] - - logger.info('Successfully built PSF model') + epsf, stars = EPSFBuilder( + oversampling = 1, + shape = 1 * size, + fitter = EPSFFitter(fit_boxsize=max(np.round(1.5*fwhm).astype(int), 5)), + recentering_boxsize = max(np.round(2*fwhm).astype(int), 5), + norm_radius = max(fwhm, 5), + maxiters = 100, + )(stars) + logger.info('Built PSF model ({n_iter}/{max_iters}) in {time:.1f}s'.format(**epsf.fit_info)) fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 15)) plot_image(epsf.data, ax=ax1, cmap='viridis') @@ -442,16 +447,13 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi apphot_tbl = aperture_photometry(image.subclean, [apertures, annuli], mask=image.mask, error=image.error) + logger.info('Aperture Photometry Success') logger.debug("Aperture Photometry Table:\n%s", apphot_tbl) - logger.info('Apperature Photometry Success') # ============================================================================================== # PSF PHOTOMETRY # ============================================================================================== - # Are we fixing the postions? - epsf.fixed.update({'x_0': False, 'y_0': False}) - # Create photometry object: photometry_obj = BasicPSFPhotometry( group_maker=DAOGroup(fwhm), @@ -467,8 +469,8 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi init_guesses=Table(coordinates, names=['x_0', 'y_0']) ) - logger.debug("PSF Photometry Table:\n%s", psfphot_tbl) logger.info('PSF Photometry Success') + logger.debug("PSF Photometry Table:\n%s", psfphot_tbl) # ============================================================================================== # TEMPLATE SUBTRACTION AND TARGET PHOTOMETRY @@ -501,6 +503,12 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi apertures = CircularAperture(target_pixel_pos, r=fwhm) annuli = CircularAnnulus(target_pixel_pos, r_in=1.5 * fwhm, r_out=2.5 * fwhm) + # XXX +# stars = get_fitscmd(diffimg, 'maskstar') +# masked_diffimage = maskstar(diffimage, image.wcs, stars, image.fwhm) +# _img = masked_diffimage.data * ~masked_diffimage.mask + # XXX + # Create two plots of the difference image: fig, ax = plt.subplots(1, 1, squeeze=True, figsize=(20, 20)) plot_image(diffimage, ax=ax, cbar='right', title=target_name) @@ -551,12 +559,10 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Build results table: tab = references.copy() - extkeys = {'pm_ra', 'pm_dec', 'gaia_mag', 'gaia_bp_mag', 'gaia_rp_mag', 'B_mag', 'V_mag', 'H_mag', 'J_mag', 'K_mag', - 'u_mag', 'g_mag', 'r_mag', 'i_mag', 'z_mag'} row = {'starid': 0, 'ra': target['ra'], 'decl': target['decl'], 'pixel_column': target_pixel_pos[0], 'pixel_row': target_pixel_pos[1]} - row.update([(k, np.NaN) for k in extkeys & set(tab.keys())]) + row.update([(k, np.NaN) for k in set(tab.keys()) - set(row) - {'gaia_variability'}]) tab.insert_row(0, row) if diffimage is not None: @@ -608,6 +614,8 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi yerr = mag_inst_err[use_for_calibration] weights = 1.0 / yerr ** 2 + assert any(use_for_calibration), "No calibration stars" + # Fit linear function with fixed slope, using sigma-clipping: model = models.Linear1D(slope=1, fixed={'slope': True}) fitter = fitting.FittingWithOutlierRemoval(fitting.LinearLSQFitter(), sigma_clip, sigma=3.0) @@ -712,6 +720,6 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi logger.info("------------------------------------------------------") logger.info("Success!") logger.info("Main target: %f +/- %f", tab[0]['mag'], tab[0]['mag_error']) - logger.info("Photometry took: %f seconds", toc - tic) + logger.info("Photometry took: %.1f seconds", toc - tic) return photometry_output diff --git a/flows/wcs.py b/flows/wcs.py index 7ee7d51..7b6926d 100644 --- a/flows/wcs.py +++ b/flows/wcs.py @@ -72,7 +72,12 @@ def force_reject_g2d(xarray, yarray, image, get_fwhm=True, rsq_min=0.5, radius=1 masked_xys = np.ma.masked_array(xys, ~np.isfinite(xys)) masked_rsqs = np.ma.masked_array(rsqs, ~np.isfinite(rsqs)) mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) # Reject Rsq < rsq_min - masked_xys = masked_xys[mask] # Clean extracted array. + # changed + #masked_xys = masked_xys[mask] # Clean extracted array. + # to + masked_xys.mask[~mask] = True + # don't know if it breaks anything, but it doesn't make sence if + # len(masked_xys) != len(masked_rsqs) FIXME masked_fwhms = np.ma.masked_array(fwhms, ~np.isfinite(fwhms)) if get_fwhm: return masked_fwhms,masked_xys,mask,masked_rsqs diff --git a/run_fitscmd.py b/run_fitscmd.py index 65fd0fc..b4d44e0 100644 --- a/run_fitscmd.py +++ b/run_fitscmd.py @@ -38,6 +38,8 @@ def _get(): with fits.open(args.fitsfile) as hdul: + print('# %s' % args.fitsfile) + if not '' in hdul[0].header: return diff --git a/run_localweb.py b/run_localweb.py index 72c1052..88a8abf 100644 --- a/run_localweb.py +++ b/run_localweb.py @@ -1,47 +1,179 @@ +import os, json +from copy import deepcopy +from base64 import b64encode as b64 + import numpy as np -from flask import Flask, request -from web import sites, datafiles, reference_stars +from astropy.io import ascii +from astropy.coordinates import SkyCoord -import json +from flask import Flask, request, render_template -app = Flask(__name__) +from flows import load_config +from flows.api import get_targets, get_target +from flows.api import get_datafiles, get_datafile +from flows.api import get_site -@app.route('/api/sites.php') -def api_sites(): - if 'siteid' in request.args: - siteid = int(request.args['siteid']) - return sites[siteid] - for site in sites: - sites[site] = {**sites.site, **sites[site]} - return json.dumps(list(sites.values())) +from web.api import targets, catalogs, datafiles, sites + +from functools import lru_cache +get_datafile = lru_cache(maxsize=1000)(get_datafile.__wrapped__) + +app = Flask(__name__, template_folder='web', static_folder='web/static') + +@app.route('/api/targets.php') +def api_targets(): + if 'target' in request.args: + target = request.args['target'] + if target in targets: + target = targets[target] + else: + if (target := get_target_by_id(target)) is None: + return '""' + return json.dumps(target) + return json.dumps(list(targets.values())) @app.route('/api/datafiles.php') def api_datafiles(): if 'fileid' in request.args: fileid = int(request.args['fileid']) - for f in (f for target in datafiles for f in datafiles[target]): - if f['fileid'] != fileid: - continue - f.update({**datafiles.image, **f}) - if not f['diffimg'] is None: - f['diffimg'] = {**datafiles.diffimg, **f['diffimg']} - return f + for name in targets: + for datafile in datafiles[name]: + if datafile['fileid'] == fileid: + return datafile elif 'targetid' in request.args: targetid = int(request.args['targetid']) - return str([f['fileid'] for f in datafiles[targetid]]) + if (target := get_target_by_id(targetid)) is None: + return '""' + return str([datafile['fileid'] for datafile in datafiles[target['target_name']]]) + return '""' @app.route('/api/reference_stars.php') -def api_reference_stars(): - targetid = int(request.args['target']) - reference_stars[targetid]['target'] = { - **reference_stars.target, **reference_stars[targetid]['target'] - } - for reference in reference_stars[targetid]['references']: - reference.update({**reference_stars.references, **reference}) - for avoid in reference_stars[targetid]['avoid']: - avoid.update({**reference_stars.avoid, **avoid}) - return reference_stars[targetid] +def api_catalogs(): + targetid = target_name = request.args['target'] + if (target := get_target_by_id(targetid)) is None and \ + (target := get_target_by_name(target_name)) is None: + return '""' + catalog = catalogs[target["target_name"]] + catalog['target'] = targets[target["target_name"]] + catalog['avoid'] = None + return json.dumps(catalog) + +@app.route('/api/sites.php') +def api_sites(): + if 'siteid' in request.args: + siteid = int(request.args['siteid']) + site = [s for s in sites if siteid == s['siteid']] + site = site[0] if site else '' + return json.dumps(site) + return json.dumps(sites) + +@app.route('/') +def index(): + targets = sorted(deepcopy(get_targets()), key=lambda t: t['target_name'])[::-1] + local_targets = os.listdir(load_config().get('photometry', 'archive_local', fallback='/')) + for target in targets: + c = SkyCoord(target['ra'], target['decl'], unit='deg') + target['ra'], target['decl'] = c.to_string('hmsdms').split() + target['inserted'] = target['inserted'].strftime('%Y-%m-%d %H:%M:%S') + target['local'] = target['target_name'] in local_targets + return render_template('index.html', targets=targets) + +@app.route('/') +def target(target): + if not (target := get_target_by_name(target)): + return '' + datafiles = [deepcopy(get_datafile(datafile)) for datafile in get_datafiles(target['targetid'], 'all')] + datafiles = sorted(datafiles, key=lambda f: f['obstime']) + archive = '%s/%s' % (load_config().get('photometry', 'archive_local', fallback=''), target['target_name']) + output = '%s/%s' % (load_config().get('photometry', 'output', fallback=''), target['target_name']) + fileids = {int(fileid): fileid for fileid in os.listdir(output)} + for datafile in datafiles: + fileid = fileids.get(datafile['fileid'], '') + datafile['filename'] = filename = datafile['path'].split('/')[-1] + datafile['sitename'] = get_site(datafile['site'])['sitename'] if not datafile['site'] is None else 'None' + datafile['exptime'] = '%.2f' % datafile['exptime'] if not datafile['exptime'] is None else 'None' + datafile['inserted'] = datafile['inserted'].strftime('%Y-%m-%d %H:%M:%S') + datafile['is_local'] = os.path.isfile(f'{archive}/{filename}') + datafile['has_phot'] = os.path.isfile(f'{output}/{fileid}/photometry.ecsv') + return render_template('target.html', target=target, datafiles=datafiles) + +@app.route('//photometry.js') +def photometry(target): + if not (target := get_target_by_name(target)): + return '' + output = load_config().get('photometry', 'output', fallback='') + fileids = get_datafiles(target['targetid']) + photometry = dict() + for fileid in os.listdir(f'{output}/%s' % target['target_name']): + if not int(fileid) in fileids: + continue + try: + table = ascii.read(f'{output}/%s/{fileid}/photometry.ecsv' % target['target_name']) + except FileNotFoundError: + continue + filt, mjd = table.meta['photfilter'], table.meta['obstime-bmjd'] + for i in np.where(table['starid'] <= 0)[0]: + mag, err = table[i]['mag'], table[i]['mag_error'] + _filt = 's_' + filt if table[i]['starid'] else filt + if not _filt in photometry: + photometry[_filt] = [] + photometry[_filt].append((mjd, mag, err, 'fileid: %d' % int(fileid))) + photometry = {filt: list(map(list, zip(*photometry[filt]))) for filt in sorted(photometry)} + return render_template('photometry.js', photometry=photometry) + +@app.route('//') +def datafile(target, fileid): + if not (target := get_target_by_name(target)): + return '' + output = '%s/%s' % (load_config().get('photometry', 'output', fallback=''), target['target_name']) + fileids = {int(fileid): fileid for fileid in os.listdir(output)} + try: + photometry = ascii.read(f'{output}/{fileids[fileid]}/photometry.ecsv') + except (FileNotFoundError, KeyError): + photometry = [{'starid': 0, 'ra': target['ra'], 'decl': target['decl'], 'distance': 0}] + try: + with open(f'{output}/{fileids[fileid]}/photometry.log', 'r') as fd: + log = fd.read() + except (FileNotFoundError, KeyError): + log = '' + try: + images = [] + for f in sorted(os.listdir(f'{output}/{fileids[fileid]}')): + if f.split('.')[-1] != 'png': + continue + with open(f'{output}/{fileids[fileid]}/{f}', 'rb') as fd: + images.append(b64(fd.read()).decode('utf-8')) + except KeyError: + pass + return render_template('datafile.html', target=target, fileid=fileid, photometry=photometry, log=log, images=images) + +@app.route('//.fits') +def fits(target, fileid): + archive = load_config().get('photometry', 'archive_local', fallback='/') + try: + datafile = get_datafile(fileid) + except: + return '' + if 'subtracted' in request.args: + try: + path = f'{archive}/' + datafile['diffimg']['path'] + except: + return '' + else: + path = f'{archive}/' + datafile['path'] + with open(path, 'rb') as fd: + return fd.read() + +def get_target_by_id(targetid): + for target in get_targets(): + if str(target['targetid']) == str(targetid): + return target + +def get_target_by_name(target_name): + for target in get_targets(): + if target['target_name'] == target_name: + return target if __name__ == '__main__': - app.run(debug=True) + app.run(host='0.0.0.0', debug=True) diff --git a/run_photometry.py b/run_photometry.py index b3c66f0..85a8667 100644 --- a/run_photometry.py +++ b/run_photometry.py @@ -20,7 +20,7 @@ def process_fileid(fid, output_folder_root=None, attempt_imagematch=True, autoup logger = logging.getLogger('flows') logging.captureWarnings(True) logger_warn = logging.getLogger('py.warnings') - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', "%Y-%m-%d %H:%M:%S") datafile = api.get_datafile(fid) target_name = datafile['target_name'] @@ -118,12 +118,13 @@ def process_fileid(fid, output_folder_root=None, attempt_imagematch=True, autoup threads = multiprocessing.cpu_count() # Setup logging: - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', "%Y-%m-%d %H:%M:%S") console = logging.StreamHandler() console.setFormatter(formatter) logger = logging.getLogger('flows') if not logger.hasHandlers(): logger.addHandler(console) + logger.propagate = False logger.setLevel(logging_level) if args.fileid is not None: diff --git a/web/__init__.py b/web/__init__.py deleted file mode 100644 index ea7298a..0000000 --- a/web/__init__.py +++ /dev/null @@ -1,83 +0,0 @@ -from .reference_stars import reference_stars -reference_stars = type('reference_stars', (dict,), { - 'target' : { - 'targetid' : -1, - 'target_name' : None, - 'target_status' : None, - 'ra' : None, - 'decl' : None, - 'redshift' : None, - 'redshift_error' : None, - 'discovery_mag' : None, - 'catalog_downloaded' : None, - 'pointing_model_created' : '1970-01-01 00:00:00.0', - 'inserted' : '1970-01-01 00:00:00.0', - 'discovery_date' : '1970-01-01 00:00:00.0', - 'project' : None, - 'host_galaxy' : None, - 'ztf_id' : None - }, - 'references' : { - 'starid' : -1, - 'ra' : None, - 'decl' : None, - 'pm_ra' : 0, - 'pm_dec' : 0, - 'gaia_mag' : None, - 'gaia_bp_mag' : None, - 'gaia_rp_mag' : None, - 'J_mag' : None, - 'H_mag' : None, - 'K_mag' : None, - 'g_mag' : None, - 'r_mag' : None, - 'i_mag' : None, - 'z_mag' : None, - 'gaia_variability' : 0, - 'V_mag' : None, - 'B_mag' : None, - 'u_mag' : None, - 'distance' : None - } -})(reference_stars) -reference_stars.avoid = reference_stars.references - -from .datafiles import datafiles -datafiles = type('datafiles', (dict,), { - 'image' : { - 'fileid' : None, - 'path' : None, - 'targetid' : None, - 'site' : None, - 'filesize' : None, - 'filehash' : None, - 'inserted' : '1970-01-01 00:00:00.0', - 'lastmodified' : '1970-01-01 00:00:00.0', - 'photfilter' : None, - 'obstime' : None, - 'exptime' : None, - 'version' : None, - 'archive_path' : None, - 'target_name' : None, - 'template' : None, - 'diffimg' : None - }, - 'diffimg' : { - 'fileid' : None, - 'path' : None, - 'filehash' : None, - 'filesize' : None - } -})(datafiles) - -from .sites import sites -sites = type('sites', (dict,), { - 'site' : { - 'siteid' : -1, - 'sitename' : None, - 'longitude': None, - 'latitude' : None, - 'elevation' : None, - 'site_keyword' : None, - } -})(sites) diff --git a/web/api/__init__.py b/web/api/__init__.py new file mode 100644 index 0000000..e19ae42 --- /dev/null +++ b/web/api/__init__.py @@ -0,0 +1,86 @@ +import os, json + +DIR = os.path.dirname(os.path.abspath(__file__)) + +with open("%s/targets.json" % DIR, 'r') as fd: + targets = json.load(fd) + +for target in targets: + targets[target] = {**{ + 'target_status' : None, + 'redshift' : None, + 'redshift_error' : None, + 'discovery_mag' : None, + 'catalog_downloaded' : None, + 'pointing_model_created' : '1970-01-01 00:00:00.0', + 'inserted' : '1970-01-01 00:00:00.0', + 'discovery_date' : '1970-01-01 00:00:00.0', + 'project' : None, + 'host_galaxy' : None, + 'ztf_id' : None + }, **targets[target]} + +with open("%s/catalogs.json" % DIR, 'r') as fd: + catalogs = json.load(fd) + +for target in catalogs: + catalogs[target] = { + 'references': [ + {**{ + 'pm_ra' : 0, + 'pm_dec' : 0, + 'gaia_mag' : None, + 'gaia_bp_mag' : None, + 'gaia_rp_mag' : None, + 'J_mag' : None, + 'H_mag' : None, + 'K_mag' : None, + 'g_mag' : None, + 'r_mag' : None, + 'i_mag' : None, + 'z_mag' : None, + 'gaia_variability' : 0, + 'V_mag' : None, + 'B_mag' : None, + 'u_mag' : None, + 'distance' : None + }, **reference} + for reference in catalogs[target]] + } + +with open("%s/datafiles.json" % DIR, 'r') as fd: + datafiles = json.load(fd) + +for target in datafiles: + for i, datafile in enumerate(datafiles[target]): + if "diffimg" in datafile and datafile["diffimg"]: + datafile["diffimg"] = {**{ + 'filehash' : None, + 'filesize' : None + }, **datafile["diffimg"]} + datafiles[target][i] = {**{ + 'site' : None, + 'filesize' : None, + 'filehash' : None, + 'inserted' : '1970-01-01 00:00:00.0', + 'lastmodified' : '1970-01-01 00:00:00.0', + 'obstime' : None, + 'exptime' : None, + 'version' : None, + 'archive_path' : None, + 'template' : None, + 'diffimg' : None + }, **datafile} + +with open("%s/sites.json" % DIR, 'r') as fd: + sites = json.load(fd) + +for i, site in enumerate(sites): + sites[i] = {**{ + 'siteid' : -1, + 'sitename' : None, + 'longitude': None, + 'latitude' : None, + 'elevation' : None, + 'site_keyword' : None, + }, **site} diff --git a/web/api/catalogs.json b/web/api/catalogs.json new file mode 100644 index 0000000..2a62250 --- /dev/null +++ b/web/api/catalogs.json @@ -0,0 +1,45 @@ +{ + "2019yvr": [ + { + "starid": 107481912845754726, + "ra": 191.28457586, + "decl": -0.42975258, + "i_mag": 14.282 + }, { + "starid": 107411912811133651, + "ra": 191.28111312, + "decl": -0.48898194, + "i_mag": 15.234 + }, { + "starid": 107391913152190877, + "ra": 191.31521935, + "decl": -0.50796039, + "i_mag": 15.564 + }, { + "starid": 107401912890621513, + "ra": 191.2890621, + "decl": -0.49909695, + "i_mag": 15.966 + }, { + "starid": 107401912782199891, + "ra": 191.27821937, + "decl": -0.49211498, + "i_mag": 16.631 + }, { + "starid": 107411912729499005, + "ra": 191.27294953, + "decl": -0.48452017, + "i_mag": 16.71 + }, { + "starid": 107431912569317799, + "ra": 191.25693153, + "decl": -0.46885893, + "i_mag": 16.938 + }, { + "starid": 107421912971816909, + "ra": 191.29718197, + "decl": -0.47793321, + "i_mag": 17.435 + } + ] +} diff --git a/web/api/datafiles.json b/web/api/datafiles.json new file mode 100644 index 0000000..efc0bf8 --- /dev/null +++ b/web/api/datafiles.json @@ -0,0 +1,15 @@ +{ + "2019yvr": [ + { + "fileid": 441, + "targetid": 2, + "target_name": "2019yvr", + "photfilter": "ip", + "path": "2019yvr/SN2019yvr_i01_NOT_ALFOSC_20200104.fits.gz", + "diffimg" : { + "fileid" : 627, + "path" : "2019yvr/subtracted/SN2019yvr_i01_NOT_ALFOSC_20200104diff.fits.gz" + } + } + ] +} diff --git a/web/api/sites.json b/web/api/sites.json new file mode 100644 index 0000000..6df26de --- /dev/null +++ b/web/api/sites.json @@ -0,0 +1,7 @@ +[{ + "siteid": 5, + "sitename": "NOT", + "longitude": -17.88508, + "latitude": 28.75728, + "elevation": 2382 +}] diff --git a/web/api/targets.json b/web/api/targets.json new file mode 100644 index 0000000..2f78599 --- /dev/null +++ b/web/api/targets.json @@ -0,0 +1,8 @@ +{ + "2019yvr": { + "target_name": "2019yvr", + "targetid": 2, + "ra": 191.283890127, + "decl": -0.45909033652 + } +} diff --git a/web/datafile.html b/web/datafile.html new file mode 100644 index 0000000..bba64e7 --- /dev/null +++ b/web/datafile.html @@ -0,0 +1,152 @@ +{% extends 'index.html' %} + +{% block head %} + + + + + + + + + + +{% endblock %} + +{% block title %} +flows.localweb / +{{ target['target_name'] }} / +{{ fileid }} + +
+
{{ log }}
+
+ +
+
+
+{% for img in images %} +
+{% endfor %} +
+
+{% endblock %} + +{% block body %} +
+
+
+
+
+
+ +
+
+ + + +
+
+ +
+
Star ID
+
Right Ascension
+
Declination
+
Distance
+
Magnitude
+
Error
+
+ +{% for phot in photometry %} +
+
{{ phot['starid'] }}
+
{{ phot['ra'] }}
+
{{ phot['decl'] }}
+
{{ phot['distance'] }}
+
{{ phot['mag'] }}
+
{{ phot['mag_error'] }}
+
+{% endfor %} + +{% endblock %} diff --git a/web/datafiles.py b/web/datafiles.py deleted file mode 100644 index b65211a..0000000 --- a/web/datafiles.py +++ /dev/null @@ -1,12 +0,0 @@ -datafiles = { - 248 : [ - { - 'fileid' : 0, - 'path' : '2016adj/AT2016adj_r_SWO_NC_2016_04_30.fits.gz', - 'diffimg' : { - 'fileid' : 1, - 'path' : '2016adj/subtracted/AT2016adj_r_SWO_NC_2016_04_30diff.fits.gz', - } - } - ] -} diff --git a/web/index.html b/web/index.html new file mode 100644 index 0000000..67a8c62 --- /dev/null +++ b/web/index.html @@ -0,0 +1,100 @@ + + + + + + + + + + + +{% block head %} + + + +{% endblock %} + +flows.localweb + + + +
+ +
+{% block title %} +flows.localweb +{% endblock %} +
+ +{% block body %} + +
+
Target
+
Target ID
+
Right Ascension
+
Declination
+
Redshift
+
Host galaxy
+
Inserted
+
+ +{% for target in targets %} +
+
{{ target['target_name'] }}
+
{{ target['targetid'] }}
+
{{ target['ra'] }}
+
{{ target['decl'] }}
+
{{ target['redshift'] }}
+
{{ target['host_galaxy'] }}
+
{{ target['inserted'] }}
+
+{% endfor %} + +{% endblock %} + +
+ +
+ + + diff --git a/web/photometry.js b/web/photometry.js new file mode 100644 index 0000000..34d10f9 --- /dev/null +++ b/web/photometry.js @@ -0,0 +1,31 @@ +var photometry = [ +{% for f in photometry %} + { + x: {{ photometry[f][0] }}, + y: {{ photometry[f][1] }}, + error_y: { + type: 'data', + array: {{ photometry[f][2] }}, + visible: true + }, + name: '{{ f }}', + text: {{ photometry[f][3] }}, + mode: 'markers', + type: 'scatter', +{% if f[:2] == 's_' %} + marker: { + size: 10, + symbol: 'circle', + }, + opacity: .8, +{% else %} + marker: { + size: 10, + symbol: 'square', + }, + opacity: .5, + visible: 'legendonly', +{% endif %} + }, +{% endfor %} +] diff --git a/web/reference_stars.py b/web/reference_stars.py deleted file mode 100644 index 7683baf..0000000 --- a/web/reference_stars.py +++ /dev/null @@ -1,26 +0,0 @@ -reference_stars = { - 248 : { - 'target' : { - 'ra' : 201.350458, - 'decl' : -43.015972, - 'inserted': '2020-12-15 12:30:27.694425', - 'discovery_date': '2016-01-29 08:00:00', - }, - 'references' : [ - { - 'ra' : 201.35558532, - 'decl' : -43.02340329, - 'B_mag' : 0, - 'V_mag' : 0, - }, - ], - 'avoid' : [ - { - 'ra' : 201.35558532, - 'decl' : -43.02340329, - 'B_mag' : 0, - 'V_mag' : 0, - }, - ] - } -} diff --git a/web/sites.py b/web/sites.py deleted file mode 100644 index 5b6e0d8..0000000 --- a/web/sites.py +++ /dev/null @@ -1,10 +0,0 @@ -sites = { - 1 : { - 'siteid' : 1, - 'sitename' : 'LCOGT at McDonald', - 'longitude': -104.015173, - 'latitude' : 30.679833, - 'elevation' : 2030, - 'site_keyword' : 'LCOGT node at McDonald Observatory', - }, -} diff --git a/web/static/README.md b/web/static/README.md new file mode 100644 index 0000000..fa00a14 --- /dev/null +++ b/web/static/README.md @@ -0,0 +1 @@ +Put js9 in a folder called js9 diff --git a/web/target.html b/web/target.html new file mode 100644 index 0000000..2993f6e --- /dev/null +++ b/web/target.html @@ -0,0 +1,92 @@ +{% extends 'index.html' %} + +{% block head %} + + + + + +{% endblock %} + +{% block title %} +flows.localweb / +{{ target['target_name'] }} +{% endblock %} + +{% block body %} +
+ +
+ +
+
File name
+
File ID
+
Site
+
Obstime
+
Filter
+
Exp. time
+
Inserted
+
+ +{% for datafile in datafiles %} +
+
{{ datafile['filename'] }}
+
{{ datafile['fileid'] }}
+
{{ datafile['sitename'] }}
+
{{ datafile['obstime'] }}
+
{{ datafile['photfilter'] }}
+
{{ datafile['exptime'] }}
+
{{ datafile['inserted'] }}
+
+{% endfor %} + +{% endblock %} From 43283912b5acbb4e76963ade47576109f025d5c8 Mon Sep 17 00:00:00 2001 From: Simon Holmbo Date: Tue, 23 Mar 2021 19:40:50 +0100 Subject: [PATCH 37/43] Bunch of stuff --- flows/__init__.py | 1 - flows/epsfbuilder/epsfbuilder.py | 74 ----------------- flows/filters.py | 5 -- flows/fitscmd.py | 95 ---------------------- flows/image/__init__.py | 1 - flows/image/image.py | 135 ------------------------------- flows/instruments/__init__.py | 61 -------------- flows/instruments/instrument.py | 78 ------------------ flows/instruments/lcogt.py | 46 ----------- flows/instruments/liverpool.py | 29 ------- flows/photometry.py | 53 +++--------- run_fitscmd.py | 93 --------------------- 12 files changed, 12 insertions(+), 659 deletions(-) delete mode 100644 flows/filters.py delete mode 100644 flows/fitscmd.py delete mode 100644 flows/image/__init__.py delete mode 100644 flows/image/image.py delete mode 100644 flows/instruments/__init__.py delete mode 100644 flows/instruments/instrument.py delete mode 100644 flows/instruments/lcogt.py delete mode 100644 flows/instruments/liverpool.py delete mode 100644 run_fitscmd.py diff --git a/flows/__init__.py b/flows/__init__.py index 55ba1a9..f65aed0 100644 --- a/flows/__init__.py +++ b/flows/__init__.py @@ -6,7 +6,6 @@ from .download_catalog import download_catalog from .visibility import visibility from .config import load_config -from .filters import FILTERS from .version import get_version __version__ = get_version(pep440=False) diff --git a/flows/epsfbuilder/epsfbuilder.py b/flows/epsfbuilder/epsfbuilder.py index 21e758a..ff07486 100644 --- a/flows/epsfbuilder/epsfbuilder.py +++ b/flows/epsfbuilder/epsfbuilder.py @@ -1,10 +1,7 @@ -#import sys import time import numpy as np -#from scipy.ndimage import gaussian_filter - #from scipy.spatial import cKDTree from scipy.interpolate import griddata @@ -45,77 +42,8 @@ def _resample_residual(self, star, epsf): return star_data - epsf._data -# def _resample_residuals(self, stars, epsf): -# -# residuals = super()._resample_residuals(stars, epsf) -# -# import matplotlib.pyplot as plt -# -# if epsf.data.any(): -# -# #i = np.isfinite(residuals).sum(axis=0, dtype=bool) -# #data = epsf.data - gaussian_filter(epsf.data, sigma=1) -# #factor = np.nanstd(data) / np.std(np.nanmedian(residuals[:,i], axis=0)) -# #residuals *= min(max(factor, 0.1), 1.0) -# -# data = np.nanmedian(residuals, axis=0) -# self._.set_data(data) -# self._.set_clim([np.nanmin(data), np.nanmax(data)]) -# self._.cb.draw_all() -# -# else: -# -# data = np.nanmedian(residuals, axis=0) -# self._ = plt.imshow(data) -# self._.set_clim([np.nanmin(data), np.nanmax(data)]) -# self._.cb = plt.gcf().colorbar(self._) -# plt.show(block=False) -# -# plt.gcf().canvas.draw() -# time.sleep(1) -# -# return residuals - -# def _recenter_epsf(self, epsf, *args, **kwargs): -# -# if not hasattr(self, 'dx_total'): -# -# self.dx_total = [] -# self.dy_total = [] -# -# def profile(frame, event, arg): -# -# global x, y, xcenter, ycenter -# -# if event == "return" and frame.f_code.co_name == '_recenter_epsf': -# -# x, xcenter = frame.f_locals['x'], frame.f_locals['xcenter'] -# y, ycenter = frame.f_locals['y'], frame.f_locals['ycenter'] -# -# self.dx_total.append(frame.f_locals['dx_total']) -# self.dy_total.append(frame.f_locals['dy_total']) -# -# return profile -# -# sys.setprofile(profile) -# super()._recenter_epsf(epsf, *args, **kwargs) -# sys.setprofile(None) -# -# dx_total = np.mean(self.dx_total[-2:], axis=0) -# dy_total = np.mean(self.dy_total[-2:], axis=0) -# -# epsf_data = epsf.evaluate(x=x, y=y, flux=1.0, -# x_0=xcenter - .5*dx_total, -# y_0=ycenter - .5*dy_total) -# -# return epsf_data - def __call__(self, *args, **kwargs): - #import matplotlib - #_backend = matplotlib.get_backend() - #matplotlib.pyplot.switch_backend('TkAgg') - t0 = time.time() epsf, stars = super().__call__(*args, **kwargs) @@ -126,6 +54,4 @@ def __call__(self, *args, **kwargs): time = time.time() - t0, ) - #matplotlib.pyplot.switch_backend(_backend) - return epsf, stars diff --git a/flows/filters.py b/flows/filters.py deleted file mode 100644 index f42aa1a..0000000 --- a/flows/filters.py +++ /dev/null @@ -1,5 +0,0 @@ -FILTERS = { - 'u', 'g', 'r', 'i', 'z', - 'B', 'V', 'R', 'I', - 'J', 'H', 'K', -} diff --git a/flows/fitscmd.py b/flows/fitscmd.py deleted file mode 100644 index 8c8bd68..0000000 --- a/flows/fitscmd.py +++ /dev/null @@ -1,95 +0,0 @@ -import re, logging - -import numpy as np -import astropy.units as u - -from astropy.table import Table - -COMMANDS = { - 'maskstar': ( # mask star for galaxy subtracted psf photometry - r'(\d+(?:\.\d*)?),\s?([+-]?\d+(?:\.\d*)?)', # parse - lambda ra, dec: (float(ra), float(dec)), # convert - lambda ra, dec, hdul: 0 <= ra < 360 and -90 <= dec <= 90 # validate - ), - 'localseq': ( # specifiy hdu name with custom local sequence - r'(.+)', # parse - lambda lsqhdu: (lsqhdu.upper(),), # convert - lambda lsqhdu, hdul: lsqhdu in [hdu.name for hdu in hdul], # validate - ), - 'colorterm': ( # color term for moving references to another system - r'\\?([+-]?\d+(?:\.\d*)?)\s?\((.+)-(.+)\)', # parse - lambda cterm, A_mag, B_mag: (float(cterm), A_mag, B_mag), # convert - lambda cterm, A_mag, B_mag, hdul: True, # validate - ) -} - -def maskstar(data, wcs, stars, fwhm): - - if not stars: - return - - data = data.copy() - if not hasattr(data, 'mask'): - data = np.ma.array(data, mask=np.zeros_like(data)) - - X, Y = np.meshgrid(*map(np.arange, data.shape[::-1])) - - for x, y in wcs.all_world2pix(stars, 0): - i = np.where(((X-x)**2 + (Y-y)**2 < (1.5*fwhm)**2)) - data.mask[i] = True - - return data - -def localseq(lsqhdus, hdul): - - if not lsqhdus: - return - - hdu = hdul[[hdu.name for hdu in hdul].index(lsqhdus[-1][0])] - - references = Table(hdu.data) - n = len(references) - - references.add_column(np.arange(n) + 1, name='starid', index=0) - references['pm_ra'] = np.zeros(n) * u.deg / u.yr - references['pm_dec'] = np.zeros(n) * u.deg / u.yr - - references.meta['localseq'] = hdu.name - - return references - -def colorterm(ref_filter, colorterms, references): - - if not colorterms: - return - - colorterm, A_mag, B_mag = colorterms[-1] - A_mag, B_mag = A_mag + '_mag', B_mag + '_mag' # XXX remove after 2016adj - - if not {A_mag, B_mag} <= set(references.colnames): - missing = ', '.join({A_mag, B_mag} - set(references.colnames)) - logging.warning('%s not in references', missing) - return - - references[ref_filter] += colorterm * (references[A_mag] - references[B_mag]) - - return references - -def get_fitscmd(image, command): - - if not command in COMMANDS: - logging.warning('fitscmd %s unknown', command) - return - - if not '' in image.head: - return - - parameters = [ - c[13+len(command):] for c in image.head[''] - if c.startswith('FLOWSCMD: %s' % command) - ] - - return [ - COMMANDS[command][1](*re.match(COMMANDS[command][0], p).groups()) - for p in parameters - ] diff --git a/flows/image/__init__.py b/flows/image/__init__.py deleted file mode 100644 index a0c1ffb..0000000 --- a/flows/image/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .image import Image diff --git a/flows/image/image.py b/flows/image/image.py deleted file mode 100644 index 41e67d0..0000000 --- a/flows/image/image.py +++ /dev/null @@ -1,135 +0,0 @@ -import pickle, warnings - -import numpy as np - -from astropy.io import fits -from astropy.wcs import WCS, FITSFixedWarning - -class Image: - - MASK_PARAMETERS = 'wcs', - - def __init__(self, data, hdr, wcs, exthdus=dict(), subtracted=None): - - self._data = data = np.asarray(data) - self.x, self.y = np.meshgrid(*map(np.arange, data.shape[::-1])) - - self.hdr, self.wcs = hdr, wcs - self.exthdus = exthdus - - self.set_subtracted(subtracted) - - self._lmasks = dict() - self._mask, self._mask_hash = None, None - self._data_mask = None - - self.instrument = None - - @classmethod - def from_fits(cls, filename, subtracted=None): - - with fits.open(filename, mode='readonly') as hdul: - - data = hdul[0].data - hdr = hdul[0].header - exthdus = {hdu.name:hdu.copy() for hdu in hdul[1:]} - - with warnings.catch_warnings(): - - warnings.simplefilter('ignore', category=FITSFixedWarning) - wcs = WCS(hdr) - - if not subtracted is None: - - with fits.open(subtracted, mode='readonly') as hdul: - subtracted = hdul[0].data - - else: - - subtracted = None - - return cls(data, hdr, wcs, exthdus, subtracted) - - def set_subtracted(self, subtracted): - - assert subtracted is None or np.shape(subtracted) == self._data.shape - - self._subtracted = np.asarray(subtracted) if not subtracted is None else None - self._subtracted_mask = None - - def set_instrument(self, instrument): - - self.instrument = instrument - self.filter = instrument.get_filter(self) - self.obstime = instrument.get_obstime(self) - self.exptime = instrument.get_exptime(self) - - if not (mask := instrument.get_mask(self)) is None: - self.add_mask(mask) - - def add_mask(self, mask): - - assert callable(mask) or np.shape(mask) = self._data.shape - - if not callable(mask): - mask = lambda img: mask - - i = max(self._lmasks.keys()) + 1 if len(self._lmasks) else 0 - self._lmasks[i] = mask - - return i - - def del_mask(self, i): - - assert i in self._lmasks, 'mask id does not exist' - - del self._lmasks[i] - - def _update_mask(self): - - mask_parameters = sorted([hash(mask) for mask in self._lmasks.values()]) - mask_parameters += [getattr(self, p) for p in self.MASK_PARAMETERS] - mask_hash = hash(pickle.dumps(mask_parameters)) - - print(self._mask_hash == mask_hash) - if self._mask_hash == mask_hash: - return False - - self._mask_hash = mask_hash - - masks = [~mask(self).astype(bool) for mask in self._lmasks.values()] - self._mask = ~np.prod(masks, axis=0, dtype=bool) \ - if len(masks) else np.zeros_like(self._data, dtype=bool) - - return True - - @property - def data(self): - - if self._update_mask() or self._data_mask is None: - - self._data_mask = self._mask.copy() - self._data_mask |= ~np.isfinite(self._data) - - return np.ma.array(self._data, mask=self._data_mask) - - @property - def subtracted(self): - - if self._subtracted is None: - raise AttributeError('no subtracted image') - - if self._update_mask() or self._subtracted_mask is None: - - self._subtracted_mask = self._mask.copy() - self._subtracted_mask |= ~np.isfinite(self._subtracted) - - return np.ma.array(self._subtracted, mask=self._subtracted_mask) - - def __getitem__(self, item): - - if item in ('filter', 'obstime', 'exptime') and \ - self.instrument is None: - raise AttributeError('can not get %s without instrument' % item) - - return super().__getitem__(item) diff --git a/flows/instruments/__init__.py b/flows/instruments/__init__.py deleted file mode 100644 index d18fc46..0000000 --- a/flows/instruments/__init__.py +++ /dev/null @@ -1,61 +0,0 @@ -import os, logging - -from inspect import getmro -from traceback import format_exc - -from importlib import import_module - -from .instrument import Instrument - -INSTRUMENTS = list() - -for instrument_file in os.listdir(__file__.rsplit('/',1)[0]): - - if not '.' in instrument_file: - continue - - instrument_filename, file_ext = instrument_file.rsplit('.', 1) - - if instrument_filename[0] == '_' or file_ext != 'py': - continue - - instrument = import_module('.' + instrument_filename, 'flows.instruments') - - for attribute in dir(instrument): - - if attribute[0] == '_': - continue - - attribute = getattr(instrument, attribute) - - if not hasattr(attribute, '__bases__'): - continue - - all_bases = [base for _base in attribute.__bases__ for base in getmro(_base)] - if not Instrument in all_bases or not hasattr(attribute, 'siteid'): - continue - - INSTRUMENTS.append(attribute) - -def get_instrument(image): - - container = list() - - for instrument in INSTRUMENTS: - try: - instrument.verify(image) - except Exception as e: - if not str(e): - e = format_exc().strip().split('\n')[-2].strip() - logging.debug(f'{instrument} : {e}') - else: - container.append(instrument) - - assert container, 'Instrument was not identified' - - if len(container) > 1: - msg = 'Data matched multiple instruments; ' - msg += ', '.join(map(str, container)) - logging.error(msg) - - return container[0] diff --git a/flows/instruments/instrument.py b/flows/instruments/instrument.py deleted file mode 100644 index 4281ebd..0000000 --- a/flows/instruments/instrument.py +++ /dev/null @@ -1,78 +0,0 @@ -from inspect import getmro - -from astropy.time import Time -from astropy import units as u - -from .. import FILTERS -from ..api import get_site - -class MetaInstrument(type): - - def __new__(mcls, name, bases, attrs): - - all_attrs = dict() - for base in bases: - all_attrs.update(base.__dict__) - all_attrs.update(attrs) - - if not 'siteid' in all_attrs: - return super().__new__(mcls, name, bases, attrs) - - assert 'filters' in all_attrs, 'no filters variable' - assert not set(all_attrs['filters'].values()) - FILTERS, 'unknown filter(s)' - - assert 'verify' in all_attrs, 'no verify classmethod' - - return super().__new__(mcls, name, bases, attrs) - -class Instrument(metaclass=MetaInstrument): - - filter_keyword = 'filter' - exptime_keyword = 'exptime' - - peakmax = None - - scale = None - mirror = None - angle = None - - def __init__(self): - - site = get_site(self.siteid) - - self.name = site['sitename'] - self.longitude = site['longitude'] - self.latitude = site['latitude'] - self.elevation = site['elevation'] - self.location = site['EarthLocation'] - - def get_exptime(self, image): - - return float(image.hdr[self.exptime_keyword]) - - def get_obstime(self, image): - - if 'mjd-obs' in image.hdr: - obstime = Time(image.hdr['mjd-obs'], format='mjd', scale='utc', location=self.location) - elif 'date-obs' in image.hdr: - obstime = Time(image.hdr['date-obs'], format='isot', scale='utc', location=self.location) - else: - raise KeyError('mjd-obs nor date-obs in header') - - return obstime - - def get_filter(self, image): - - f = image.hdr[self.filter_keyword] - - assert f in self.filters, f'filter {f} not recognized' - - return self.filters[f] - - def get_mask(self, image): - - return None - - def __repr__(self): - - return self.name diff --git a/flows/instruments/lcogt.py b/flows/instruments/lcogt.py deleted file mode 100644 index 75c04d6..0000000 --- a/flows/instruments/lcogt.py +++ /dev/null @@ -1,46 +0,0 @@ -from astropy.time import Time -from astropy.coordinates import EarthLocation - -from .instrument import Instrument - -class LCOGT(Instrument): - - filters = { - 'B' : 'B', - } - - def get_obstime(self, image): - - lat, lon, height = image.hdr['latitude'], image.hdr['longitud'], image.hdr['height'] - location = EarthLocation.from_geodetic(lat, lon, height) - - return Time(image.hdr['mjd-obs'], format='mjd', scale='utc', location=location) - - def get_mask(self, image): - - return np.asarray(image.exthdus['BPM'].data, dtype=bool) - - @classmethod - def verify(cls, image): - - assert image.hdr['origin'] == 'LCOGT' - -class LCOGT_SAAO(LCOGT): - - siteid = 3 - - @classmethod - def verify(cls, image): - - LCOGT.verify(image) - assert image.hdr['site'] == 'LCOGT node at SAAO' - -class LCOGT_SSO(LCOGT): - - siteid = 6 - - @classmethod - def verify(cls, image): - - LCOGT.verify(image) - assert image.hdr['site'] == 'LCOGT node at Siding Spring Observatory' diff --git a/flows/instruments/liverpool.py b/flows/instruments/liverpool.py deleted file mode 100644 index 663733e..0000000 --- a/flows/instruments/liverpool.py +++ /dev/null @@ -1,29 +0,0 @@ -from astropy.time import Time -from astropy import units as u - -from .instrument import Instrument - -class Liverpool(Instrument): - - siteid = 8 - - filters = { - 'Bessel-B' : 'B', - 'Bessell-B' : 'B', - 'V' : 'V' # XXX - } - - filter_keyword = 'filter'#1' XXX - - def get_obstime(self, image): - - obstime = super().get_obstime(image) - obstime += self.get_exptime(image) / 2 * u.second # Make time centre of exposure - - return obstime - - @classmethod - def verify(cls, image): - - return - assert image.hdr['telescop'] == 'Liverpool Telescope' diff --git a/flows/photometry.py b/flows/photometry.py index a224283..f916a00 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -15,7 +15,7 @@ import logging import warnings -from astropy.utils.exceptions import AstropyDeprecationWarning, AstropyUserWarning +from astropy.utils.exceptions import AstropyDeprecationWarning, AstropyUserWarning, ErfaWarning import astropy.units as u import astropy.coordinates as coords from astropy.stats import sigma_clip, SigmaClip @@ -42,7 +42,6 @@ from .zeropoint import bootstrap_outlier, sigma_from_Chauvenet from .wcs import force_reject_g2d, clean_with_rsq_and_get_fwhm, get_clean_references from .coordinatematch import CoordinateMatch, WCS -from .fitscmd import get_fitscmd, maskstar, localseq, colorterm from .epsfbuilder import EPSFBuilder, gaussian_kernel __version__ = get_version(pep440=False) @@ -133,14 +132,10 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi ref_filter = 'g_mag' # Load the image from the FITS file: + logger.info("Load image '%s'", filepath) image = load_image(filepath) - lsqhdus = get_fitscmd(image, 'localseq') # look for local sequence in fits table - references = catalog['references'] if not lsqhdus else localseq(lsqhdus, image.exthdu) - - colorterms = get_fitscmd(image, 'colorterm') - references = colorterm(ref_filter, colorterms, references) if colorterms else references - + references = catalog['references'] references.sort(ref_filter) # Check that there actually are reference stars in that filter: @@ -203,7 +198,10 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi refs_coord = coords.SkyCoord(ra=references['ra'], dec=references['decl'], pm_ra_cosdec=references['pm_ra'], pm_dec=references['pm_dec'], unit='deg', frame='icrs', obstime=Time(2015.5, format='decimalyear')) - refs_coord = refs_coord.apply_space_motion(image.obstime) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ErfaWarning) + refs_coord = refs_coord.apply_space_motion(image.obstime) # @TODO: These need to be based on the instrument! radius = 10 @@ -216,23 +214,6 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi radius=radius, fwhm_guess=fwhm_guess, rsq_min=0.3, fwhm_max=fwhm_max, fwhm_min=fwhm_min) - # XXX -# try: -# _references = catalog['references'] -# _references.sort(ref_filter) -# replace(_references['pm_ra'], np.NaN, 0) -# replace(_references['pm_dec'], np.NaN, 0) -# _refs_coord = coords.SkyCoord(ra=_references['ra'], dec=_references['decl'], -# pm_ra_cosdec=_references['pm_ra'], pm_dec=_references['pm_dec'], -# unit='deg', frame='icrs', obstime=Time(2015.5, format='decimalyear')) -# _refs_coord = _refs_coord.apply_space_motion(image.obstime) -# if allnan(_references[ref_filter]): -# raise ValueError("No _reference stars found in current photfilter.") -# except Exception as e: -# logging.warning(e) -# _refs_coord = refs_coord - # XXX - head_wcs = str(WCS.from_astropy_wcs(image.wcs)) logger.debug('Head WCS: %s', head_wcs) references.meta['head_wcs'] = head_wcs @@ -335,13 +316,13 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi size = max(size, 15) # Never go below 15 pixels # Extract stars sub-images: - xy = [tuple(masked_ref_xys[clean_references['starid'] == ref['starid']].data[0]) for ref in references] # FIXME !!! + xy = [tuple(masked_ref_xys[clean_references['starid'] == ref['starid']].data[0]) for ref in references] # FIXME with warnings.catch_warnings(): warnings.simplefilter('ignore', AstropyUserWarning) stars = extract_stars( - NDData(data=image.subclean, mask=image.mask), + NDData(data=image.subclean.data, mask=image.mask), Table(np.array(xy), names=('x', 'y')), - size = size + 6#2*size+1 # +6 for edge buffer + size = size + 6 # +6 for edge buffer ) # Store which stars were used in ePSF in the table: @@ -360,7 +341,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi ax[i].axis('off') else: offset_axes = stars[imgnr].bbox.ixmin, stars[imgnr].bbox.iymin - plot_image(stars[imgnr], ax=ax[i], scale='log', cmap='viridis')#, offset_axes=offset_axes) + plot_image(stars[imgnr], ax=ax[i], scale='log', cmap='viridis')#, offset_axes=offset_axes) FIXME (no x-ticks) imgnr += 1 fig.savefig(os.path.join(output_folder, 'epsf_stars%02d.png' % (k + 1)), bbox_inches='tight') @@ -502,12 +483,6 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi apertures = CircularAperture(target_pixel_pos, r=fwhm) annuli = CircularAnnulus(target_pixel_pos, r_in=1.5 * fwhm, r_out=2.5 * fwhm) - # XXX -# stars = get_fitscmd(diffimg, 'maskstar') -# masked_diffimage = maskstar(diffimage, image.wcs, stars, image.fwhm) -# _img = masked_diffimage.data * ~masked_diffimage.mask - # XXX - # Create two plots of the difference image: fig, ax = plt.subplots(1, 1, squeeze=True, figsize=(20, 20)) plot_image(diffimage, ax=ax, cbar='right', title=target_name) @@ -538,13 +513,9 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi aperture_radius=fwhm ) - # Mask stars from FITS header - stars = get_fitscmd(diffimg, 'maskstar') - masked_diffimage = maskstar(diffimage, image.wcs, stars, image.fwhm) - # Run PSF photometry on template subtracted image: target_psfphot_tbl = photometry_obj( - diffimage if masked_diffimage is None else masked_diffimage, + diffimage, init_guesses=Table(target_pixel_pos, names=['x_0', 'y_0']) ) diff --git a/run_fitscmd.py b/run_fitscmd.py deleted file mode 100644 index b4d44e0..0000000 --- a/run_fitscmd.py +++ /dev/null @@ -1,93 +0,0 @@ -import os, sys, re -import argparse, logging - -from astropy.io import fits - -from flows.fitscmd import COMMANDS - -def _add(): - - parser = argparse.ArgumentParser(description='Add FITS specific command.') - parser.add_argument('command', choices=list(COMMANDS.keys()), help='command') - parser.add_argument('parameters', help='command parameters') - parser.add_argument('fitsfiles', nargs='+', help='FITS files') - args = parser.parse_args() - - try: - parameters = COMMANDS[args.command][1](*re.match(COMMANDS[args.command][0], args.parameters).groups()) - except: - logging.critical('can not parse %s', args.parameters) - quit(1) - - for f in args.fitsfiles: - try: - with fits.open(f, mode='update') as hdul: - if not COMMANDS[args.command][2](*parameters, hdul): - logging.error('%s %s is not valid for %s', args.command, args.parameters, f) - continue - hdul[0].header[''] = 'FLOWSCMD: %s = %s' % (args.command, args.parameters) - except Exception as e: - logging.error('could not open %s', f) - continue - -def _get(): - - parser = argparse.ArgumentParser(description='Get FITS specific commands.') - parser.add_argument('fitsfile', help='FITS file') - args = parser.parse_args() - - with fits.open(args.fitsfile) as hdul: - - print('# %s' % args.fitsfile) - - if not '' in hdul[0].header: - return - - commands = [(i, c[10:]) for i, c in enumerate(hdul[0].header['']) if c.startswith('FLOWSCMD: ')] - for i, c in commands: - print('%%%dd) %%s' % (len(str(len(commands))),) % (i, c)) - -def _del(): - - parser = argparse.ArgumentParser(description='Delete FITS specific command.') - parser.add_argument('command', type=int, help='Command ID') - parser.add_argument('fitsfile', help='FITS file') - args = parser.parse_args() - - with fits.open(args.fitsfile, mode='update') as hdul: - - if not '' in hdul[0].header: - logging.critical('no commands in fits') - quit(1) - - commands = [i for i, c in enumerate(hdul[0].header['']) if c.startswith('FLOWSCMD: ')] - if not args.command in commands: - logging.critical('command id does not exist') - quit(1) - - head = list(hdul[0].header['']) - head.pop(args.command) - del hdul[0].header[''] - - for l in head: - hdul[0].header[''] = l - -if __name__ == '__main__': - - if len(sys.argv) > 1: - - method = sys.argv.pop(1) - - if not { - 'add' : _add, - 'get' : _get, - 'del' : _del, - }.get(method, lambda *a: 1)(): - - quit(0) - - sys.argv.insert(1, method) - - parser = argparse.ArgumentParser(description='Control FITS specific commands.') - parser.add_argument('method', choices=('add', 'get', 'del'), help='method') - args = parser.parse_args() From d18dd4366f2d6edd11af8e40c1fa1c5ba8fa3814 Mon Sep 17 00:00:00 2001 From: Simon Holmbo Date: Wed, 14 Apr 2021 16:50:06 +0200 Subject: [PATCH 38/43] -web +pep8 --- .gitignore | 1 - flows/api/catalogs.py | 10 +- flows/api/datafiles.py | 10 +- flows/api/sites.py | 10 +- flows/api/targets.py | 6 +- flows/coordinatematch/__init__.py | 4 +- flows/coordinatematch/coordinatematch.py | 253 +++++----- flows/coordinatematch/wcs.py | 56 ++- flows/epsfbuilder/__init__.py | 3 +- flows/epsfbuilder/epsfbuilder.py | 17 +- flows/epsfbuilder/gaussian_kernel.py | 12 - flows/load_image.py | 4 +- flows/photometry.py | 435 +++++++++++------ flows/references.py | 75 --- flows/wcs.py | 563 +++++++++++++---------- requirements.txt | 6 +- run_localweb.py | 179 ------- web/api/__init__.py | 86 ---- web/api/catalogs.json | 45 -- web/api/datafiles.json | 15 - web/api/sites.json | 7 - web/api/targets.json | 8 - web/datafile.html | 152 ------ web/index.html | 100 ---- web/photometry.js | 31 -- web/static/README.md | 1 - web/target.html | 92 ---- 27 files changed, 806 insertions(+), 1375 deletions(-) delete mode 100644 flows/epsfbuilder/gaussian_kernel.py delete mode 100644 flows/references.py delete mode 100644 run_localweb.py delete mode 100644 web/api/__init__.py delete mode 100644 web/api/catalogs.json delete mode 100644 web/api/datafiles.json delete mode 100644 web/api/sites.json delete mode 100644 web/api/targets.json delete mode 100644 web/datafile.html delete mode 100644 web/index.html delete mode 100644 web/photometry.js delete mode 100644 web/static/README.md delete mode 100644 web/target.html diff --git a/.gitignore b/.gitignore index 0803d48..56d7440 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ flows/config.ini flows/casjobs/CasJobs.config -web/static/js9 # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/flows/api/catalogs.py b/flows/api/catalogs.py index c8461b0..97ab13e 100644 --- a/flows/api/catalogs.py +++ b/flows/api/catalogs.py @@ -36,15 +36,12 @@ def get_catalog(target, radius=None, output='table'): # Get API token from config file: config = load_config() - address = config.get('api', 'catalog', fallback=None) token = config.get('api', 'token', fallback=None) - if address is None: - raise Exception("No API catalog address has been defined") if token is None: raise Exception("No API token has been defined") # - r = requests.get('%s/reference_stars.php' % address, + r = requests.get('https://flows.phys.au.dk/api/reference_stars.php', params={'target': target}, headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() @@ -116,15 +113,12 @@ def get_catalog_missing(): # Get API token from config file: config = load_config() - address = config.get('api', 'catalog', fallback=None) token = config.get('api', 'token', fallback=None) - if address is None: - raise Exception("No API catalog address has been defined") if token is None: raise Exception("No API token has been defined") # - r = requests.get('%s/catalog_missing.php' % address, + r = requests.get('https://flows.phys.au.dk/api/catalog_missing.php', headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() return r.json() diff --git a/flows/api/datafiles.py b/flows/api/datafiles.py index 5650ded..6cd8843 100644 --- a/flows/api/datafiles.py +++ b/flows/api/datafiles.py @@ -15,14 +15,11 @@ def get_datafile(fileid): # Get API token from config file: config = load_config() - address = config.get('api', 'address', fallback=None) token = config.get('api', 'token', fallback=None) - if address is None: - raise Exception("No API catalog address has been defined") if token is None: raise Exception("No API token has been defined") - r = requests.get('%s/datafiles.php' % address, + r = requests.get('https://flows.phys.au.dk/api/datafiles.php', params={'fileid': fileid}, headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() @@ -59,10 +56,7 @@ def get_datafiles(targetid=None, filt=None): # Get API token from config file: config = load_config() - address = config.get('api', 'address', fallback=None) token = config.get('api', 'token', fallback=None) - if address is None: - raise Exception("No API catalog address has been defined") if token is None: raise Exception("No API token has been defined") @@ -71,7 +65,7 @@ def get_datafiles(targetid=None, filt=None): params['targetid'] = targetid params['filter'] = filt - r = requests.get('%s/datafiles.php' % address, + r = requests.get('https://flows.phys.au.dk/api/datafiles.php', params=params, headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() diff --git a/flows/api/sites.py b/flows/api/sites.py index f6854c7..96dd67f 100644 --- a/flows/api/sites.py +++ b/flows/api/sites.py @@ -16,14 +16,11 @@ def get_site(siteid): # Get API token from config file: config = load_config() - address = config.get('api', 'address', fallback=None) token = config.get('api', 'token', fallback=None) - if address is None: - raise Exception("No API catalog address has been defined") if token is None: raise Exception("No API token has been defined") - r = requests.get('%s/sites.php' % address, + r = requests.get('https://flows.phys.au.dk/api/sites.php', params={'siteid': siteid}, headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() @@ -40,14 +37,11 @@ def get_all_sites(): # Get API token from config file: config = load_config() - address = config.get('api', 'address', fallback=None) token = config.get('api', 'token', fallback=None) - if address is None: - raise Exception("No API catalog address has been defined") if token is None: raise Exception("No API token has been defined") - r = requests.get('%s/sites.php' % address, + r = requests.get('https://flows.phys.au.dk/api/sites.php', headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() jsn = r.json() diff --git a/flows/api/targets.py b/flows/api/targets.py index 8afffa2..be8f5e4 100644 --- a/flows/api/targets.py +++ b/flows/api/targets.py @@ -20,12 +20,11 @@ def get_target(target): # Get API token from config file: config = load_config() - address = config.get('api', 'address', fallback=None) token = config.get('api', 'token', fallback=None) if token is None: raise Exception("No API token has been defined") - r = requests.get('%s/targets.php' % address, + r = requests.get('https://flows.phys.au.dk/api/targets.php', params={'target': target}, headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() @@ -44,12 +43,11 @@ def get_targets(): # Get API token from config file: config = load_config() - address = config.get('api', 'address', fallback=None) token = config.get('api', 'token', fallback=None) if token is None: raise Exception("No API token has been defined") - r = requests.get('%s/targets.php' % address, + r = requests.get('https://flows.phys.au.dk/api/targets.php', headers={'Authorization': 'Bearer ' + token}) r.raise_for_status() jsn = r.json() diff --git a/flows/coordinatematch/__init__.py b/flows/coordinatematch/__init__.py index ce2eb3c..93e91ab 100644 --- a/flows/coordinatematch/__init__.py +++ b/flows/coordinatematch/__init__.py @@ -1,2 +1,2 @@ -from .coordinatematch import CoordinateMatch -from .wcs import WCS +from .coordinatematch import CoordinateMatch # noqa +from .wcs import WCS2 # noqa diff --git a/flows/coordinatematch/coordinatematch.py b/flows/coordinatematch/coordinatematch.py index f6bb2cb..a3acc95 100644 --- a/flows/coordinatematch/coordinatematch.py +++ b/flows/coordinatematch/coordinatematch.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +""" +Match two sets of coordinates + +.. codeauthor:: Simon Holmbo +""" import time from itertools import count, islice, chain, product, zip_longest @@ -8,29 +14,34 @@ from scipy.spatial import cKDTree as KDTree from networkx import Graph, connected_components -from .wcs import WCS +from .wcs import WCS2 -class CoordinateMatch () : - def __init__(self, xy, rd, - xy_order=None, rd_order=None, - xy_nmax=None, rd_nmax=None, - n_triangle_packages = 10, - triangle_package_size = 10000, - maximum_angle_distance = 0.001, - distance_factor = 1 - ): +class CoordinateMatch(): + def __init__(self, + xy, + rd, + xy_order=None, + rd_order=None, + xy_nmax=None, + rd_nmax=None, + n_triangle_packages=10, + triangle_package_size=10000, + maximum_angle_distance=0.001, + distance_factor=1): self.xy, self.rd = np.array(xy), np.array(rd) self._xy = xy - np.mean(xy, axis=0) self._rd = rd - np.mean(rd, axis=0) - self._rd[:,0] *= np.cos(np.deg2rad(self.rd[:,1])) + self._rd[:, 0] *= np.cos(np.deg2rad(self.rd[:, 1])) xy_n, rd_n = min(xy_nmax, len(xy)), min(rd_nmax, len(rd)) - self.i_xy = xy_order[:xy_n] if not xy_order is None else np.arange(xy_n) - self.i_rd = rd_order[:rd_n] if not rd_order is None else np.arange(rd_n) + self.i_xy = xy_order[:xy_n] if xy_order is not None else np.arange( + xy_n) + self.i_rd = rd_order[:rd_n] if rd_order is not None else np.arange( + rd_n) self.n_triangle_packages = n_triangle_packages self.triangle_package_size = triangle_package_size @@ -45,14 +56,17 @@ def __init__(self, xy, rd, self.parameters = None self.neighbours = Graph() - self.normalizations = type('Normalizations', (object,), dict( - ra = 0.0001, dec = 0.0001, scale = 0.002, angle = 0.002 - )) + self.normalizations = type( + 'Normalizations', (object, ), + dict(ra=0.0001, dec=0.0001, scale=0.002, angle=0.002)) - self.bounds = type('Bounds', (object,), dict( - xy = self.xy.mean(axis=0), rd = None, radius = None, - scale = None, angle = None - )) + self.bounds = type( + 'Bounds', (object, ), + dict(xy=self.xy.mean(axis=0), + rd=None, + radius=None, + scale=None, + angle=None)) def set_normalizations(self, ra=None, dec=None, scale=None, angle=None): '''Set normalization factors in the (ra, dec, scale, angle) space. @@ -64,21 +78,29 @@ def set_normalizations(self, ra=None, dec=None, scale=None, angle=None): angle = 0.002 radians ''' - if not self.parameters is None: + if self.parameters is not None: - raise Exception('can\'t change normalization after matching is started') + raise Exception( + 'can\'t change normalization after matching is started') assert ra is None or 0 < ra assert dec is None or 0 < dec assert scale is None or 0 < scale assert angle is None or 0 < angle - self.normalizations.ra = ra if not ra is None else self.normalizations.ra - self.normalizations.dec = dec if not dec is None else self.normalizations.dec - self.normalizations.scale = scale if not scale is None else self.normalizations.scale - self.normalizations.angle = angle if not ra is None else self.normalizations.angle - - def set_bounds(self, x=None, y=None, ra=None, dec=None, radius=None, scale=None, angle=None): + self.normalizations.ra = ra if ra is not None else self.normalizations.ra + self.normalizations.dec = dec if dec is not None else self.normalizations.dec + self.normalizations.scale = scale if scale is not None else self.normalizations.scale + self.normalizations.angle = angle if ra is not None else self.normalizations.angle + + def set_bounds(self, + x=None, + y=None, + ra=None, + dec=None, + radius=None, + scale=None, + angle=None): '''Set bounds for what are valid results. Set x, y, ra, dec and radius to specify that the x, y coordinates must be no @@ -88,7 +110,7 @@ def set_bounds(self, x=None, y=None, ra=None, dec=None, radius=None, scale=None, same system. ''' - if not self.parameters is None: + if self.parameters is not None: raise Exception('can\'t change bounds after matching is started') @@ -109,8 +131,8 @@ def set_bounds(self, x=None, y=None, ra=None, dec=None, radius=None, scale=None, assert scale is None or 0 < scale[0] < scale[1] assert angle is None or -np.pi <= angle[0] < angle[1] <= np.pi - self.bounds.scale = scale if not scale is None else self.bounds.scale - self.bounds.angle = angle if not angle is None else self.bounds.angle + self.bounds.scale = scale if scale is not None else self.bounds.scale + self.bounds.angle = angle if angle is not None else self.bounds.angle def _sorted_triangles(self, pool): @@ -125,7 +147,8 @@ def _sorted_product_pairs(self, p, q): i_p = np.argsort(np.arange(len(p))) i_q = np.argsort(np.arange(len(q))) - for _i_p, _i_q in sorted(product(i_p, i_q), key=lambda idxs: sum(idxs)): + for _i_p, _i_q in sorted(product(i_p, i_q), + key=lambda idxs: sum(idxs)): yield p[_i_p], q[_i_q] @@ -134,23 +157,23 @@ def _sorted_triangle_packages(self): i_xy_triangle_generator = self._sorted_triangles(self.i_xy) i_rd_triangle_generator = self._sorted_triangles(self.i_rd) - i_xy_triangle_slice_generator = ( - tuple(islice(i_xy_triangle_generator, self.triangle_package_size)) - for _ in count() - ) - i_rd_triangle_slice_generator = ( - list(islice(i_rd_triangle_generator, self.triangle_package_size)) - for _ in count() - ) + i_xy_triangle_slice_generator = (tuple( + islice(i_xy_triangle_generator, self.triangle_package_size)) for _ in count()) + i_rd_triangle_slice_generator = (list( + islice(i_rd_triangle_generator, self.triangle_package_size)) for _ in count()) for n in count(step=self.n_triangle_packages): - i_xy_triangle_slice = tuple(filter(None, - islice(i_xy_triangle_slice_generator, self.n_triangle_packages) - )) - i_rd_triangle_slice = tuple(filter(None, - islice(i_rd_triangle_slice_generator, self.n_triangle_packages) - )) + i_xy_triangle_slice = tuple( + filter( + None, + islice(i_xy_triangle_slice_generator, + self.n_triangle_packages))) + i_rd_triangle_slice = tuple( + filter( + None, + islice(i_rd_triangle_slice_generator, + self.n_triangle_packages))) if not len(i_xy_triangle_slice) and not len(i_rd_triangle_slice): return @@ -158,32 +181,32 @@ def _sorted_triangle_packages(self): i_xy_triangle_generator2 = self._sorted_triangles(self.i_xy) i_rd_triangle_generator2 = self._sorted_triangles(self.i_rd) - i_xy_triangle_cum = filter(None, ( - tuple(islice(i_xy_triangle_generator2, self.triangle_package_size)) - for _ in range(n) - )) - i_rd_triangle_cum = filter(None, ( - tuple(islice(i_rd_triangle_generator2, self.triangle_package_size)) - for _ in range(n) - )) + i_xy_triangle_cum = filter(None, (tuple( + islice(i_xy_triangle_generator2, self.triangle_package_size)) for _ in range(n))) + i_rd_triangle_cum = filter(None, (tuple( + islice(i_rd_triangle_generator2, self.triangle_package_size)) for _ in range(n))) for i_xy_triangles, i_rd_triangles in chain( - filter(None, chain(*zip_longest( # alternating chain + filter( + None, + chain(*zip_longest( # alternating chain product(i_xy_triangle_slice, i_rd_triangle_cum), - product(i_xy_triangle_cum, i_rd_triangle_slice) - ))), - self._sorted_product_pairs(i_xy_triangle_slice, i_rd_triangle_slice) - ): + product(i_xy_triangle_cum, i_rd_triangle_slice)))), + self._sorted_product_pairs(i_xy_triangle_slice, + i_rd_triangle_slice)): yield np.array(i_xy_triangles), np.array(i_rd_triangles) def _get_triangle_angles(self, triangles): - sidelengths = np.sqrt(np.power(triangles[:,(1,0,0)] - triangles[:,(2,2,1)], 2).sum(axis=2)) + sidelengths = np.sqrt( + np.power(triangles[:, (1, 0, 0)] - triangles[:, (2, 2, 1)], + 2).sum(axis=2)) # law of cosines - angles = np.power(sidelengths[:,((1,2),(0,2),(0,1))], 2).sum(axis=2) - angles -= np.power(sidelengths[:,(0,1,2)], 2) - angles /= 2 * sidelengths[:,((1,2),(0,2),(0,1))].prod(axis=2) + angles = np.power(sidelengths[:, ((1, 2), (0, 2), (0, 1))], + 2).sum(axis=2) + angles -= np.power(sidelengths[:, (0, 1, 2)], 2) + angles /= 2 * sidelengths[:, ((1, 2), (0, 2), (0, 1))].prod(axis=2) return np.arccos(angles) @@ -191,10 +214,12 @@ def _solve_for_matrices(self, xy_triangles, rd_triangles): n = len(xy_triangles) - A = xy_triangles - np.mean(xy_triangles, axis=1).reshape(n,1,2) - b = rd_triangles - np.mean(rd_triangles, axis=1).reshape(n,1,2) + A = xy_triangles - np.mean(xy_triangles, axis=1).reshape(n, 1, 2) + b = rd_triangles - np.mean(rd_triangles, axis=1).reshape(n, 1, 2) - matrices = [np.linalg.lstsq(Ai, bi, rcond=None)[0].T for Ai, bi in zip(A, b)] + matrices = [ + np.linalg.lstsq(Ai, bi, rcond=None)[0].T for Ai, bi in zip(A, b) + ] return np.array(matrices) @@ -202,21 +227,18 @@ def _extract_parameters(self, xy_triangles, rd_triangles, matrices): parameters = [] - for xy_com, rd_com, matrix in zip( # com -> center-of-mass - xy_triangles.mean(axis=1), - rd_triangles.mean(axis=1), - matrices - ): + for xy_com, rd_com, matrix in zip( # com -> center-of-mass + xy_triangles.mean(axis=1), rd_triangles.mean(axis=1), + matrices): - cos_dec = np.cos(np.deg2rad(rd_com[1])) - coordinates = (self.bounds.xy - xy_com).dot(matrix) - coordinates = coordinates / np.array([cos_dec, 1]) + rd_com + cos_dec = np.cos(np.deg2rad(rd_com[1])) + coordinates = (self.bounds.xy - xy_com).dot(matrix) + coordinates = coordinates / np.array([cos_dec, 1]) + rd_com - wcs = WCS.from_matrix(*xy_com, *rd_com, matrix) + wcs = WCS2.from_matrix(*xy_com, *rd_com, matrix) - parameters.append( - (*coordinates, np.log(wcs.scale), np.deg2rad(wcs.angle)) - ) + parameters.append( + (*coordinates, np.log(wcs.scale), np.deg2rad(wcs.angle))) return parameters @@ -225,30 +247,26 @@ def _get_bounds_mask(self, parameters): i = np.ones(len(parameters), dtype=bool) parameters = np.array(parameters) - if not self.bounds.radius is None: + if self.bounds.radius is not None: i *= angular_separation( - *np.deg2rad(self.bounds.rd), - *zip(*np.deg2rad(parameters[:,(0,1)])) - ) <= np.deg2rad(self.bounds.radius) + *np.deg2rad(self.bounds.rd), + *zip(*np.deg2rad(parameters[:, (0, 1)]))) <= np.deg2rad( + self.bounds.radius) - if not self.bounds.scale is None: + if self.bounds.scale is not None: - i *= self.bounds.scale[0] <= parameters[:,2] - i *= parameters[:,2] <= self.bounds.scale[1] + i *= self.bounds.scale[0] <= parameters[:, 2] + i *= parameters[:, 2] <= self.bounds.scale[1] - if not self.bounds.angle is None: + if self.bounds.angle is not None: - i *= self.bounds.angle[0] <= parameters[:,3] - i *= parameters[:,3] <= self.bounds.angle[1] + i *= self.bounds.angle[0] <= parameters[:, 3] + i *= parameters[:, 3] <= self.bounds.angle[1] return i - def __call__(self, - minimum_matches = 4, - ratio_superiority = 1, - timeout = 60 - ): + def __call__(self, minimum_matches=4, ratio_superiority=1, timeout=60): '''Start the alogrithm. Can be run multiple times with different arguments to relax the @@ -279,7 +297,8 @@ def __call__(self, print('Failed, timeout.') ''' - self.parameters = list() if self.parameters is None else self.parameters + self.parameters = list( + ) if self.parameters is None else self.parameters t0 = time.time() @@ -287,7 +306,8 @@ def __call__(self, # get triangles and derive angles - i_xy_triangles, i_rd_triangles = next(self.triangle_package_generator) + i_xy_triangles, i_rd_triangles = next( + self.triangle_package_generator) xy_angles = self._get_triangle_angles(self._xy[i_xy_triangles]) rd_angles = self._get_triangle_angles(self._rd[i_rd_triangles]) @@ -304,27 +324,27 @@ def __call__(self, # match triangles - matches = KDTree(xy_angles).query_ball_tree(KDTree(rd_angles), r=self.maximum_angle_distance) - matches = np.array([(_i_xy, _i_rd) for _i_xy, _li_rd in enumerate(matches) for _i_rd in _li_rd]) + matches = KDTree(xy_angles).query_ball_tree( + KDTree(rd_angles), r=self.maximum_angle_distance) + matches = np.array([(_i_xy, _i_rd) + for _i_xy, _li_rd in enumerate(matches) + for _i_rd in _li_rd]) if not len(matches): continue - i_xy_triangles = list(i_xy_triangles[matches[:,0]]) - i_rd_triangles = list(i_rd_triangles[matches[:,1]]) + i_xy_triangles = list(i_xy_triangles[matches[:, 0]]) + i_rd_triangles = list(i_rd_triangles[matches[:, 1]]) # get parameters of wcs solutions matrices = self._solve_for_matrices( - self._xy[np.array(i_xy_triangles)], - self._rd[np.array(i_rd_triangles)] - ) + self._xy[np.array(i_xy_triangles)], + self._rd[np.array(i_rd_triangles)]) parameters = self._extract_parameters( - self.xy[np.array(i_xy_triangles)], - self.rd[np.array(i_rd_triangles)], - matrices - ) + self.xy[np.array(i_xy_triangles)], + self.rd[np.array(i_rd_triangles)], matrices) # apply bounds if any @@ -338,15 +358,23 @@ def __call__(self, # normalize parameters - normalization = [getattr(self.normalizations, v) for v in ('ra', 'dec', 'scale', 'angle')] - normalization[0] *= np.cos(np.deg2rad(self.rd[:,1].mean(axis=0))) + normalization = [ + getattr(self.normalizations, v) + for v in ('ra', 'dec', 'scale', 'angle') + ] + normalization[0] *= np.cos(np.deg2rad(self.rd[:, 1].mean(axis=0))) parameters = list(parameters / np.array(normalization)) # match parameters - neighbours = KDTree(parameters).query_ball_tree(KDTree(self.parameters + parameters), r=self.distance_factor) - neighbours = np.array([(i, j) for i, lj in enumerate(neighbours, len(self.parameters)) for j in lj]) - neighbours = list(neighbours[(np.diff(neighbours, axis=1) < 0).flatten()]) + neighbours = KDTree(parameters).query_ball_tree( + KDTree(self.parameters + parameters), r=self.distance_factor) + neighbours = np.array([ + (i, j) for i, lj in enumerate(neighbours, len(self.parameters)) + for j in lj + ]) + neighbours = list( + neighbours[(np.diff(neighbours, axis=1) < 0).flatten()]) if not len(neighbours): continue @@ -360,7 +388,8 @@ def __call__(self, communities = list(connected_components(self.neighbours)) c1 = np.array(list(max(communities, key=len))) - i = np.unique(np.array(self.i_xy_triangles)[c1].flatten(), return_index=True)[1] + i = np.unique(np.array(self.i_xy_triangles)[c1].flatten(), + return_index=True)[1] if ratio_superiority > 1 and len(communities) > 1: diff --git a/flows/coordinatematch/wcs.py b/flows/coordinatematch/wcs.py index e3e5c07..6d16c85 100644 --- a/flows/coordinatematch/wcs.py +++ b/flows/coordinatematch/wcs.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +""" +WCS tools + +.. codeauthor:: Simon Holmbo +""" from copy import deepcopy import numpy as np @@ -6,15 +12,16 @@ from scipy.optimize import minimize from scipy.spatial.transform import Rotation -class WCS () : + +class WCS2(): '''Manipulate WCS solution. Initialize ---------- - wcs = WCS(x, y, ra, dec, scale, mirror, angle) - wcs = WCS.from_matrix(x, y, ra, dec, matrix) - wcs = WCS.from_points(list(zip(x, y)), list(zip(ra, dec))) - wcs = WCS.from_astropy_wcs(astropy.wcs.WCS()) + wcs = WCS2(x, y, ra, dec, scale, mirror, angle) + wcs = WCS2.from_matrix(x, y, ra, dec, matrix) + wcs = WCS2.from_points(list(zip(x, y)), list(zip(ra, dec))) + wcs = WCS2.from_astropy_wcs(astropy.wcs.WCS()) ra, dec and angle should be in degrees scale should be in arcsec/pixel @@ -30,12 +37,11 @@ class WCS () : print(wcs.scale, wcs.angle) Change an astropy.wcs.WCS (wcs) angle - wcs = WCS(wcs)(angle=new_angle).astropy_wcs + wcs = WCS2(wcs)(angle=new_angle).astropy_wcs Adjust solution with points wcs.adjust_with_points(list(zip(x, y)), list(zip(ra, dec))) ''' - def __init__(self, x, y, ra, dec, scale, mirror, angle): self.x, self.y = x, y @@ -77,7 +83,8 @@ def from_astropy_wcs(cls, astropy_wcs): 'Must be astropy.wcs.WCS' (x, y), (ra, dec) = astropy_wcs.wcs.crpix, astropy_wcs.wcs.crval - scale, mirror, angle = cls._decompose_matrix(astropy_wcs.pixel_scale_matrix) + scale, mirror, angle = cls._decompose_matrix( + astropy_wcs.pixel_scale_matrix) return cls(x, y, ra, dec, scale, mirror, angle) @@ -98,20 +105,26 @@ def adjust_with_points(self, xy, rd): self.ra, self.dec = rd.mean(axis=0) A, b = xy - xy.mean(axis=0), rd - rd.mean(axis=0) - b[:,0] *= np.cos(np.deg2rad(rd[:,1])) + b[:, 0] *= np.cos(np.deg2rad(rd[:, 1])) if len(xy) == 2: M = np.diag([[-1, 1][self.mirror], 1]) - R = lambda t: np.array([[np.cos(t), -np.sin(t)], [np.sin(t), np.cos(t)]]) - chi2 = lambda x: np.power(A.dot(x[1]/60/60*R(x[0]).dot(M).T) - b, 2).sum() + def R(t): + return np.array([[np.cos(t), -np.sin(t)], + [np.sin(t), np.cos(t)]]) + + def chi2(x): + return np.power( + A.dot(x[1] / 60 / 60 * R(x[0]).dot(M).T) - b, 2).sum() self.angle, self.scale = minimize(chi2, [self.angle, self.scale]).x elif len(xy) > 2: matrix = np.linalg.lstsq(A, b, rcond=None)[0].T - self.scale, self.mirror, self.angle = self._decompose_matrix(matrix) + self.scale, self.mirror, self.angle = self._decompose_matrix( + matrix) @property def matrix(self): @@ -120,10 +133,8 @@ def matrix(self): mirror = np.diag([[-1, 1][self.mirror], 1]) angle = np.deg2rad(self.angle) - matrix = np.array([ - [np.cos(angle), -np.sin(angle)], - [np.sin(angle), np.cos(angle)] - ]) + matrix = np.array([[np.cos(angle), -np.sin(angle)], + [np.sin(angle), np.cos(angle)]]) return scale * matrix @ mirror @@ -143,7 +154,7 @@ def _solve_from_points(xy, rd): (x, y), (ra, dec) = xy.mean(axis=0), rd.mean(axis=0) A, b = xy - xy.mean(axis=0), rd - rd.mean(axis=0) - b[:,0] *= np.cos(np.deg2rad(rd[:,1])) + b[:, 0] *= np.cos(np.deg2rad(rd[:, 1])) matrix = np.linalg.lstsq(A, b, rcond=None)[0].T @@ -155,13 +166,16 @@ def _decompose_matrix(matrix): scale = np.sqrt(np.power(matrix, 2).sum() / 2) * 60 * 60 if np.argmax(np.power(matrix[0], 2)): - mirror = True if np.sign(matrix[0,1]) != np.sign(matrix[1,0]) else False + mirror = True if np.sign(matrix[0, 1]) != np.sign( + matrix[1, 0]) else False else: - mirror = True if np.sign(matrix[0,0]) == np.sign(matrix[1,1]) else False + mirror = True if np.sign(matrix[0, 0]) == np.sign( + matrix[1, 1]) else False matrix = matrix if mirror else matrix.dot(np.diag([-1, 1])) - matrix3d = np.eye(3); matrix3d[:2,:2] = matrix / (scale / 60 / 60) + matrix3d = np.eye(3) + matrix3d[:2, :2] = matrix / (scale / 60 / 60) angle = Rotation.from_matrix(matrix3d).as_euler('xyz', degrees=True)[2] return scale, mirror, angle @@ -211,4 +225,4 @@ def __repr__(self): ra, dec = self.astropy_wcs.wcs_pix2world([(0, 0)], 0)[0] - return f'WCS(0, 0, {ra:.4f}, {dec:.4f}, {self.scale:.2f}, {self.mirror}, {self.angle:.2f})' + return f'WCS2(0, 0, {ra:.4f}, {dec:.4f}, {self.scale:.2f}, {self.mirror}, {self.angle:.2f})' diff --git a/flows/epsfbuilder/__init__.py b/flows/epsfbuilder/__init__.py index 6a86ded..a7b2ccf 100644 --- a/flows/epsfbuilder/__init__.py +++ b/flows/epsfbuilder/__init__.py @@ -1,2 +1 @@ -from .epsfbuilder import EPSFBuilder -from .gaussian_kernel import gaussian_kernel +from .epsfbuilder import EPSFBuilder # noqa diff --git a/flows/epsfbuilder/epsfbuilder.py b/flows/epsfbuilder/epsfbuilder.py index ff07486..a9fd2af 100644 --- a/flows/epsfbuilder/epsfbuilder.py +++ b/flows/epsfbuilder/epsfbuilder.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +""" +Photutils hack for EPSF building + +.. codeauthor:: Simon Holmbo +""" import time import numpy as np @@ -7,8 +13,8 @@ import photutils.psf -class EPSFBuilder(photutils.psf.EPSFBuilder): +class EPSFBuilder(photutils.psf.EPSFBuilder): def _create_initial_epsf(self, stars): epsf = super()._create_initial_epsf(stars) @@ -38,7 +44,8 @@ def _resample_residual(self, star, epsf): #star_data.ravel()[mask] = star._data_values_normalized[ii[mask]] star_points = list(zip(star._xidx_centered, star._yidx_centered)) - star_data = griddata(star_points, star._data_values_normalized, self._epsf_xy_grid) + star_data = griddata(star_points, star._data_values_normalized, + self._epsf_xy_grid) return star_data - epsf._data @@ -49,9 +56,9 @@ def __call__(self, *args, **kwargs): epsf, stars = super().__call__(*args, **kwargs) epsf.fit_info = dict( - n_iter = len(self._epsf), - max_iters = self.maxiters, - time = time.time() - t0, + n_iter=len(self._epsf), + max_iters=self.maxiters, + time=time.time() - t0, ) return epsf, stars diff --git a/flows/epsfbuilder/gaussian_kernel.py b/flows/epsfbuilder/gaussian_kernel.py deleted file mode 100644 index 0d9cf47..0000000 --- a/flows/epsfbuilder/gaussian_kernel.py +++ /dev/null @@ -1,12 +0,0 @@ -import numpy as np - -from scipy.stats import norm - -def gaussian_kernel(kernlen, nsig): - """Returns a 2D Gaussian kernel.""" - - x = np.linspace(-nsig, nsig, kernlen+1) - kern1d = np.diff(norm.cdf(x)) - kern2d = np.outer(kern1d, kern1d) - - return kern2d / kern2d.sum() diff --git a/flows/load_image.py b/flows/load_image.py index f03b947..579e875 100644 --- a/flows/load_image.py +++ b/flows/load_image.py @@ -82,7 +82,7 @@ def load_image(FILENAME): telescope = hdr.get('TELESCOP') instrument = hdr.get('INSTRUME') - image.image = np.asarray(hdul[0].data, dtype=np.float64) + image.image = np.asarray(hdul[0].data, dtype='float64') image.shape = image.image.shape image.head = hdr @@ -313,7 +313,7 @@ def load_image(FILENAME): raise Exception("Could not determine origin of image") # Create masked version of image: - #image.image[image.mask] = np.NaN + image.image[image.mask] = np.NaN image.clean = np.ma.masked_array(image.image, image.mask) return image diff --git a/flows/photometry.py b/flows/photometry.py index f916a00..66569eb 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -26,30 +26,35 @@ from astropy.time import Time warnings.simplefilter('ignore', category=AstropyDeprecationWarning) -from photutils import CircularAperture, CircularAnnulus, aperture_photometry -from photutils.psf import EPSFFitter, BasicPSFPhotometry, DAOGroup, extract_stars -from photutils import Background2D, SExtractorBackground, MedianBackground -from photutils.utils import calc_total_error - -from scipy.interpolate import UnivariateSpline - -from . import api -from .config import load_config -from .plots import plt, plot_image -from .version import get_version -from .load_image import load_image -from .run_imagematch import run_imagematch -from .zeropoint import bootstrap_outlier, sigma_from_Chauvenet -from .wcs import force_reject_g2d, clean_with_rsq_and_get_fwhm, get_clean_references -from .coordinatematch import CoordinateMatch, WCS -from .epsfbuilder import EPSFBuilder, gaussian_kernel +from photutils import CircularAperture, CircularAnnulus, aperture_photometry # noqa +from photutils.psf import EPSFFitter, BasicPSFPhotometry, DAOGroup, extract_stars # noqa +from photutils import Background2D, SExtractorBackground, MedianBackground # noqa +from photutils.utils import calc_total_error # noqa + +from scipy.interpolate import UnivariateSpline # noqa + +from . import api # noqa +from .config import load_config # noqa +from .plots import plt, plot_image # noqa +from .version import get_version # noqa +from .load_image import load_image # noqa +from .run_imagematch import run_imagematch # noqa +from .zeropoint import bootstrap_outlier, sigma_from_Chauvenet # noqa +from .wcs import force_reject_g2d, clean_with_rsq_and_get_fwhm, get_clean_references # noqa +from .coordinatematch import CoordinateMatch, WCS2 # noqa +from .epsfbuilder import EPSFBuilder # noqa __version__ = get_version(pep440=False) warnings.simplefilter('ignore', category=AstropyDeprecationWarning) + # -------------------------------------------------------------------------------------------------- -def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fixed=False, timeoutpar=10): +def photometry(fileid, + output_folder=None, + attempt_imagematch=True, + keep_diff_fixed=False, + timeoutpar=10): """ Run photometry. @@ -95,12 +100,16 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # TODO: Include proper-motion to the time of observation catalog = api.get_catalog(targetid, output='table') target = catalog['target'][0] - target_coord = coords.SkyCoord(ra=target['ra'], dec=target['decl'], unit='deg', frame='icrs') + target_coord = coords.SkyCoord(ra=target['ra'], + dec=target['decl'], + unit='deg', + frame='icrs') # Folder to save output: if output_folder is None: output_folder_root = config.get('photometry', 'output', fallback='.') - output_folder = os.path.join(output_folder_root, target_name, '%05d' % fileid) + output_folder = os.path.join(output_folder_root, target_name, + '%05d' % fileid) logger.info("Placing output in '%s'", output_folder) os.makedirs(output_folder, exist_ok=True) @@ -128,7 +137,9 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi }.get(photfilter, None) if ref_filter is None: - logger.warning("Could not find filter '%s' in catalogs. Using default gp filter.", photfilter) + logger.warning( + "Could not find filter '%s' in catalogs. Using default gp filter.", + photfilter) ref_filter = 'g_mag' # Load the image from the FITS file: @@ -155,16 +166,22 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi fig, ax = plt.subplots(1, 2, figsize=(20, 18)) plot_image(image.clean, ax=ax[0], scale='log', cbar='right', title='Image') - plot_image(image.mask, ax=ax[1], scale='linear', cbar='right', title='Mask') - fig.savefig(os.path.join(output_folder, 'original.png'), bbox_inches='tight') + plot_image(image.mask, + ax=ax[1], + scale='linear', + cbar='right', + title='Mask') + fig.savefig(os.path.join(output_folder, 'original.png'), + bbox_inches='tight') plt.close(fig) # Estimate image background: # Not using image.clean here, since we are redefining the mask anyway - background = Background2D(image.clean, (128, 128), filter_size=(5, 5), - sigma_clip=SigmaClip(sigma=3.0), - bkg_estimator=SExtractorBackground(), - exclude_percentile=50.0) + background = Background2D(image.clean, (128, 128), + filter_size=(5, 5), + sigma_clip=SigmaClip(sigma=3.0), + bkg_estimator=SExtractorBackground(), + exclude_percentile=50.0) # Create background-subtracted image: image.subclean = image.clean - background.background @@ -172,9 +189,16 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Plot background estimation: fig, ax = plt.subplots(1, 3, figsize=(20, 6)) plot_image(image.clean, ax=ax[0], scale='log', title='Original') - plot_image(background.background, ax=ax[1], scale='log', title='Background') - plot_image(image.subclean, ax=ax[2], scale='log', title='Background subtracted') - fig.savefig(os.path.join(output_folder, 'background.png'), bbox_inches='tight') + plot_image(background.background, + ax=ax[1], + scale='log', + title='Background') + plot_image(image.subclean, + ax=ax[2], + scale='log', + title='Background subtracted') + fig.savefig(os.path.join(output_folder, 'background.png'), + bbox_inches='tight') plt.close(fig) # TODO: Is this correct?! @@ -182,10 +206,15 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Use sep to for soure extraction sep_background = sep.Background(image.clean.data, mask=image.mask) - logger.debug('sub: {} bkg_rms: {} mask: {}'.format(image.shape, np.shape(sep_background.globalrms), - image.shape)) - objects = sep.extract(image.clean.data - sep_background, thresh=5., err=sep_background.globalrms, - mask=image.mask, deblend_cont=0.1, minarea=9, clean_param=2.0) + msg = 'sub: {} bkg_rms: {} mask: {}'.format(image.shape, np.shape(sep_background.globalrms), image.shape) + logger.debug(msg) + objects = sep.extract(image.clean.data - sep_background, + thresh=5., + err=sep_background.globalrms, + mask=image.mask, + deblend_cont=0.1, + minarea=9, + clean_param=2.0) # ============================================================================================== # DETECTION OF STARS AND MATCHING WITH CATALOG @@ -195,9 +224,13 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # TODO: Are catalog RA-proper motions including cosdec? replace(references['pm_ra'], np.NaN, 0) replace(references['pm_dec'], np.NaN, 0) - refs_coord = coords.SkyCoord(ra=references['ra'], dec=references['decl'], - pm_ra_cosdec=references['pm_ra'], pm_dec=references['pm_dec'], - unit='deg', frame='icrs', obstime=Time(2015.5, format='decimalyear')) + refs_coord = coords.SkyCoord(ra=references['ra'], + dec=references['decl'], + pm_ra_cosdec=references['pm_ra'], + pm_dec=references['pm_dec'], + unit='deg', + frame='icrs', + obstime=Time(2015.5, format='decimalyear')) with warnings.catch_warnings(): warnings.simplefilter("ignore", ErfaWarning) @@ -210,22 +243,32 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi fwhm_max = 18.0 # Clean extracted stars - masked_sep_xy, sep_mask, masked_sep_rsqs = force_reject_g2d(objects['x'], objects['y'], image, get_fwhm=False, - radius=radius, fwhm_guess=fwhm_guess, rsq_min=0.3, - fwhm_max=fwhm_max, fwhm_min=fwhm_min) - - head_wcs = str(WCS.from_astropy_wcs(image.wcs)) + masked_sep_xy, sep_mask, masked_sep_rsqs = force_reject_g2d( + objects['x'], + objects['y'], + image, + get_fwhm=False, + radius=radius, + fwhm_guess=fwhm_guess, + rsq_min=0.3, + fwhm_max=fwhm_max, + fwhm_min=fwhm_min) + + head_wcs = str(WCS2.from_astropy_wcs(image.wcs)) logger.debug('Head WCS: %s', head_wcs) references.meta['head_wcs'] = head_wcs # Solve for new WCS cm = CoordinateMatch( - xy = list(masked_sep_xy[sep_mask]), - rd = list(zip(refs_coord.ra.deg, refs_coord.dec.deg)), - xy_order = np.argsort(np.power(masked_sep_xy[sep_mask] - np.array(image.shape[::-1])/2, 2).sum(axis=1)), - rd_order = np.argsort(target_coord.separation(refs_coord)), - xy_nmax = 100, rd_nmax = 100, - maximum_angle_distance = 0.002, + xy=list(masked_sep_xy[sep_mask]), + rd=list(zip(refs_coord.ra.deg, refs_coord.dec.deg)), + xy_order=np.argsort( + np.power(masked_sep_xy[sep_mask] - np.array(image.shape[::-1]) / 2, + 2).sum(axis=1)), + rd_order=np.argsort(target_coord.separation(refs_coord)), + xy_nmax=100, + rd_nmax=100, + maximum_angle_distance=0.002, ) try: @@ -238,53 +281,81 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi logger.info('Found new WCS') image.wcs = fit_wcs_from_points( np.array(list(zip(*cm.xy[i_xy]))), - coords.SkyCoord(*map(list, zip(*cm.rd[i_rd])), unit='deg') - ) + coords.SkyCoord(*map(list, zip(*cm.rd[i_rd])), unit='deg')) - used_wcs = str(WCS.from_astropy_wcs(image.wcs)) + used_wcs = str(WCS2.from_astropy_wcs(image.wcs)) logger.debug('Used WCS: %s', used_wcs) references.meta['used_wcs'] = used_wcs # Calculate pixel-coordinates of references: - xy = image.wcs.all_world2pix(list(zip(refs_coord.ra.deg, refs_coord.dec.deg)), 0) - references['pixel_column'], references['pixel_row'] = x, y = list(map(np.array, zip(*xy))) + xy = image.wcs.all_world2pix( + list(zip(refs_coord.ra.deg, refs_coord.dec.deg)), 0) + references['pixel_column'], references['pixel_row'] = x, y = list( + map(np.array, zip(*xy))) # Clean out the references: hsize = 10 - clean_references = references[(target_coord.separation(refs_coord) > ref_target_dist_limit) - & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) - & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] + clean_references = references[ + (target_coord.separation(refs_coord) > ref_target_dist_limit) + & (x > hsize) & (x < (image.shape[1] - 1 - hsize)) + & (y > hsize) & (y < (image.shape[0] - 1 - hsize))] # & (references[ref_filter] < ref_mag_limit) assert len(clean_references), 'No clean references in field' # Calculate the targets position in the image: - target_pixel_pos = image.wcs.all_world2pix([(target['ra'], target['decl'])], 0)[0] + target_pixel_pos = image.wcs.all_world2pix( + [(target['ra'], target['decl'])], 0)[0] # Clean reference star locations - masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d(clean_references['pixel_column'], - clean_references['pixel_row'], - image, - get_fwhm=True, - radius=radius, - fwhm_guess=fwhm_guess, - fwhm_max=fwhm_max, - fwhm_min=fwhm_min, - rsq_min=0.15) + masked_fwhms, masked_ref_xys, rsq_mask, masked_rsqs = force_reject_g2d( + clean_references['pixel_column'], + clean_references['pixel_row'], + image, + get_fwhm=True, + radius=radius, + fwhm_guess=fwhm_guess, + fwhm_max=fwhm_max, + fwhm_min=fwhm_min, + rsq_min=0.15) # Use R^2 to more robustly determine initial FWHM guess. # This cleaning is good when we have FEW references. - fwhm, fwhm_clean_references = clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, clean_references, - min_fwhm_references=2, min_references=6, rsq_min=0.15) - logger.info('Initial FWHM guess is {} pixels'.format(fwhm)) + fwhm, fwhm_clean_references = clean_with_rsq_and_get_fwhm( + masked_fwhms, + masked_rsqs, + clean_references, + min_fwhm_references=2, + min_references=6, + rsq_min=0.15) + msg = 'Initial FWHM guess is {} pixels'.format(fwhm) + logger.info(msg) image.fwhm = fwhm # Create plot of target and reference star positions from 2D Gaussian fits. fig, ax = plt.subplots(1, 1, figsize=(20, 18)) - plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) - ax.scatter(fwhm_clean_references['pixel_column'], fwhm_clean_references['pixel_row'], c='r', marker='o', alpha=0.3) - ax.scatter(masked_sep_xy[:, 0], masked_sep_xy[:, 1], marker='s', alpha=1.0, edgecolors='green', facecolors='none') - ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') - fig.savefig(os.path.join(output_folder, 'positions_g2d.png'), bbox_inches='tight') + plot_image(image.subclean, + ax=ax, + scale='log', + cbar='right', + title=target_name) + ax.scatter(fwhm_clean_references['pixel_column'], + fwhm_clean_references['pixel_row'], + c='r', + marker='o', + alpha=0.3) + ax.scatter(masked_sep_xy[:, 0], + masked_sep_xy[:, 1], + marker='s', + alpha=1.0, + edgecolors='green', + facecolors='none') + ax.scatter(target_pixel_pos[0], + target_pixel_pos[1], + marker='+', + s=20, + c='r') + fig.savefig(os.path.join(output_folder, 'positions_g2d.png'), + bbox_inches='tight') plt.close(fig) # Uncomment For Debugging @@ -292,17 +363,39 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Final clean of wcs corrected references logger.info("Number of references before final cleaning: %d", len(clean_references)) - logger.debug('masked R^2 values: {}'.format(masked_rsqs[rsq_mask])) - references = get_clean_references(clean_references, masked_rsqs, rsq_ideal=0.8) - logger.info("Number of references after final cleaning: %d", len(references)) + msg = 'masked R^2 values: {}'.format(masked_rsqs[rsq_mask]) + logger.debug(msg) + references = get_clean_references(clean_references, + masked_rsqs, + rsq_ideal=0.8) + logger.info("Number of references after final cleaning: %d", + len(references)) # Create plot of target and reference star positions: fig, ax = plt.subplots(1, 1, figsize=(20, 18)) - plot_image(image.subclean, ax=ax, scale='log', cbar='right', title=target_name) - ax.scatter(references['pixel_column'], references['pixel_row'], c='r', marker='o', alpha=0.6) - ax.scatter(masked_sep_xy[:, 0], masked_sep_xy[:, 1], marker='s', alpha=0.6, edgecolors='green', facecolors='none') - ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', s=20, c='r') - fig.savefig(os.path.join(output_folder, 'positions.png'), bbox_inches='tight') + plot_image(image.subclean, + ax=ax, + scale='log', + cbar='right', + title=target_name) + ax.scatter(references['pixel_column'], + references['pixel_row'], + c='r', + marker='o', + alpha=0.6) + ax.scatter(masked_sep_xy[:, 0], + masked_sep_xy[:, 1], + marker='s', + alpha=0.6, + edgecolors='green', + facecolors='none') + ax.scatter(target_pixel_pos[0], + target_pixel_pos[1], + marker='+', + s=20, + c='r') + fig.savefig(os.path.join(output_folder, 'positions.png'), + bbox_inches='tight') plt.close(fig) # ============================================================================================== @@ -312,7 +405,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Make cutouts of stars using extract_stars: # Scales with FWHM size = int(np.round(29 * fwhm / 6)) - size += 0 if size % 2 else 1 # Make sure it's a uneven number + size += 0 if size % 2 else 1 # Make sure it's a uneven number size = max(size, 15) # Never go below 15 pixels # Extract stars sub-images: @@ -322,41 +415,47 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi stars = extract_stars( NDData(data=image.subclean.data, mask=image.mask), Table(np.array(xy), names=('x', 'y')), - size = size + 6 # +6 for edge buffer + size=size + 6 # +6 for edge buffer ) # Store which stars were used in ePSF in the table: references['used_for_epsf'] = False - references['used_for_epsf'][[star.id_label-1 for star in stars]] = True + references['used_for_epsf'][[star.id_label - 1 for star in stars]] = True logger.info("Number of stars used for ePSF: %d", len(stars)) # Plot the stars being used for ePSF: imgnr = 0 nrows, ncols = 5, 5 for k in range(int(np.ceil(len(stars) / (nrows * ncols)))): - fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True) + fig, ax = plt.subplots(nrows=nrows, + ncols=ncols, + figsize=(20, 20), + squeeze=True) ax = ax.ravel() for i in range(nrows * ncols): if imgnr > len(stars) - 1: ax[i].axis('off') else: - offset_axes = stars[imgnr].bbox.ixmin, stars[imgnr].bbox.iymin - plot_image(stars[imgnr], ax=ax[i], scale='log', cmap='viridis')#, offset_axes=offset_axes) FIXME (no x-ticks) + #offset_axes = stars[imgnr].bbox.ixmin, stars[imgnr].bbox.iymin + plot_image(stars[imgnr], ax=ax[i], scale='log', cmap='viridis') # , offset_axes=offset_axes) FIXME (no x-ticks) imgnr += 1 - fig.savefig(os.path.join(output_folder, 'epsf_stars%02d.png' % (k + 1)), bbox_inches='tight') + fig.savefig(os.path.join(output_folder, + 'epsf_stars%02d.png' % (k + 1)), + bbox_inches='tight') plt.close(fig) # Build the ePSF: epsf, stars = EPSFBuilder( - oversampling = 1, - shape = 1 * size, - fitter = EPSFFitter(fit_boxsize=max(np.round(1.5*fwhm).astype(int), 5)), - recentering_boxsize = max(np.round(2*fwhm).astype(int), 5), - norm_radius = max(fwhm, 5), - maxiters = 100, + oversampling=1, + shape=1 * size, + fitter=EPSFFitter(fit_boxsize=max(np.round(1.5 * fwhm).astype(int), 5)), + recentering_boxsize=max(np.round(2 * fwhm).astype(int), 5), + norm_radius=max(fwhm, 5), + maxiters=100, )(stars) - logger.info('Built PSF model ({n_iter}/{max_iters}) in {time:.1f}s'.format(**epsf.fit_info)) + msg = 'Built PSF model ({n_iter}/{max_iters}) in {time:.1f}s'.format(**epsf.fit_info) + logger.info(msg) fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 15)) plot_image(epsf.data, ax=ax1, cmap='viridis') @@ -372,7 +471,11 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Run a spline through the points, but subtract half of the peak value, and find the roots: # We have to use a cubic spline, since roots() is not supported for other splines # for some reason - profile_intp = UnivariateSpline(np.arange(0, len(profile)), profile - poffset, k=3, s=0, ext=3) + profile_intp = UnivariateSpline(np.arange(0, len(profile)), + profile - poffset, + k=3, + s=0, + ext=3) lr = profile_intp.roots() # Plot the profile and spline: @@ -411,7 +514,8 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # COORDINATES TO DO PHOTOMETRY AT # ============================================================================================== - coordinates = np.array([[ref['pixel_column'], ref['pixel_row']] for ref in references]) + coordinates = np.array([[ref['pixel_column'], ref['pixel_row']] + for ref in references]) # Add the main target position as the first entry for doing photometry directly in the # science image: @@ -425,7 +529,9 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi apertures = CircularAperture(coordinates, r=fwhm) annuli = CircularAnnulus(coordinates, r_in=1.5 * fwhm, r_out=2.5 * fwhm) - apphot_tbl = aperture_photometry(image.subclean, [apertures, annuli], mask=image.mask, error=image.error) + apphot_tbl = aperture_photometry(image.subclean, [apertures, annuli], + mask=image.mask, + error=image.error) logger.info('Aperture Photometry Success') logger.debug("Aperture Photometry Table:\n%s", apphot_tbl) @@ -435,19 +541,16 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # ============================================================================================== # Create photometry object: - photometry_obj = BasicPSFPhotometry( - group_maker=DAOGroup(fwhm), - bkg_estimator=MedianBackground(), - psf_model=epsf, - fitter=fitting.LevMarLSQFitter(), - fitshape=size, - aperture_radius=fwhm - ) + photometry_obj = BasicPSFPhotometry(group_maker=DAOGroup(fwhm), + bkg_estimator=MedianBackground(), + psf_model=epsf, + fitter=fitting.LevMarLSQFitter(), + fitshape=size, + aperture_radius=fwhm) - psfphot_tbl = photometry_obj( - image=image.subclean, - init_guesses=Table(coordinates, names=['x_0', 'y_0']) - ) + psfphot_tbl = photometry_obj(image=image.subclean, + init_guesses=Table(coordinates, + names=['x_0', 'y_0'])) logger.info('PSF Photometry Success') logger.debug("PSF Photometry Table:\n%s", psfphot_tbl) @@ -465,14 +568,19 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi diffimage = None if datafile.get('diffimg') is not None: - diffimg_path = os.path.join(datafile['archive_path'], datafile['diffimg']['path']) + diffimg_path = os.path.join(datafile['archive_path'], + datafile['diffimg']['path']) diffimg = load_image(diffimg_path) diffimage = diffimg.image elif attempt_imagematch and datafile.get('template') is not None: # Run the template subtraction, and get back # the science image where the template has been subtracted: - diffimage = run_imagematch(datafile, target, star_coord=coordinates, fwhm=fwhm, pixel_scale=pixel_scale) + diffimage = run_imagematch(datafile, + target, + star_coord=coordinates, + fwhm=fwhm, + pixel_scale=pixel_scale) # We have a diff image, so let's do photometry of the target using this: if diffimage is not None: @@ -481,22 +589,31 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Create apertures around the target: apertures = CircularAperture(target_pixel_pos, r=fwhm) - annuli = CircularAnnulus(target_pixel_pos, r_in=1.5 * fwhm, r_out=2.5 * fwhm) + annuli = CircularAnnulus(target_pixel_pos, + r_in=1.5 * fwhm, + r_out=2.5 * fwhm) # Create two plots of the difference image: fig, ax = plt.subplots(1, 1, squeeze=True, figsize=(20, 20)) plot_image(diffimage, ax=ax, cbar='right', title=target_name) - ax.plot(target_pixel_pos[0], target_pixel_pos[1], marker='+', color='r') - fig.savefig(os.path.join(output_folder, 'diffimg.png'), bbox_inches='tight') + ax.plot(target_pixel_pos[0], + target_pixel_pos[1], + marker='+', + color='r') + fig.savefig(os.path.join(output_folder, 'diffimg.png'), + bbox_inches='tight') apertures.plot(color='r') annuli.plot(color='k') ax.set_xlim(target_pixel_pos[0] - 50, target_pixel_pos[0] + 50) ax.set_ylim(target_pixel_pos[1] - 50, target_pixel_pos[1] + 50) - fig.savefig(os.path.join(output_folder, 'diffimg_zoom.png'), bbox_inches='tight') + fig.savefig(os.path.join(output_folder, 'diffimg_zoom.png'), + bbox_inches='tight') plt.close(fig) # Run aperture photometry on subtracted image: - target_apphot_tbl = aperture_photometry(diffimage, [apertures, annuli], mask=image.mask, error=image.error) + target_apphot_tbl = aperture_photometry(diffimage, [apertures, annuli], + mask=image.mask, + error=image.error) # Make target only photometry object if keep_diff_fixed = True if keep_diff_fixed: @@ -510,14 +627,13 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi psf_model=epsf, fitter=fitting.LevMarLSQFitter(), fitshape=size, - aperture_radius=fwhm - ) + aperture_radius=fwhm) # Run PSF photometry on template subtracted image: - target_psfphot_tbl = photometry_obj( - diffimage, - init_guesses=Table(target_pixel_pos, names=['x_0', 'y_0']) - ) + target_psfphot_tbl = photometry_obj(diffimage, + init_guesses=Table( + target_pixel_pos, + names=['x_0', 'y_0'])) if keep_diff_fixed: # Need to adjust table columns if x_0 and y_0 were fixed target_psfphot_tbl['x_0_unc'] = 0.0 @@ -525,14 +641,21 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Combine the output tables from the target and the reference stars into one: apphot_tbl = vstack([target_apphot_tbl, apphot_tbl], join_type='exact') - psfphot_tbl = vstack([target_psfphot_tbl, psfphot_tbl], join_type='exact') + psfphot_tbl = vstack([target_psfphot_tbl, psfphot_tbl], + join_type='exact') # Build results table: tab = references.copy() - row = {'starid': 0, 'ra': target['ra'], 'decl': target['decl'], 'pixel_column': target_pixel_pos[0], - 'pixel_row': target_pixel_pos[1]} - row.update([(k, np.NaN) for k in set(tab.keys()) - set(row) - {'gaia_variability'}]) + row = { + 'starid': 0, + 'ra': target['ra'], + 'decl': target['decl'], + 'pixel_column': target_pixel_pos[0], + 'pixel_row': target_pixel_pos[1] + } + row.update([(k, np.NaN) + for k in set(tab.keys()) - set(row) - {'gaia_variability'}]) tab.insert_row(0, row) if diffimage is not None: @@ -542,9 +665,9 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi indx_main_target = tab['starid'] <= 0 # Subtract background estimated from annuli: - flux_aperture = apphot_tbl['aperture_sum_0'] - (apphot_tbl['aperture_sum_1'] / annuli.area) * apertures.area - flux_aperture_error = np.sqrt( - apphot_tbl['aperture_sum_err_0'] ** 2 + (apphot_tbl['aperture_sum_err_1'] / annuli.area * apertures.area) ** 2) + flux_aperture = apphot_tbl['aperture_sum_0'] - ( + apphot_tbl['aperture_sum_1'] / annuli.area) * apertures.area + flux_aperture_error = np.sqrt(apphot_tbl['aperture_sum_err_0']**2 + (apphot_tbl['aperture_sum_err_1'] / annuli.area * apertures.area)**2) # Add table columns with results: tab['flux_aperture'] = flux_aperture / image.exptime @@ -575,20 +698,24 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Mask out things that should not be used in calibration: use_for_calibration = np.ones_like(mag_catalog, dtype='bool') - use_for_calibration[indx_main_target] = False # Do not use target for calibration - use_for_calibration[~np.isfinite(mag_inst) | ~np.isfinite(mag_catalog)] = False + use_for_calibration[ + indx_main_target] = False # Do not use target for calibration + use_for_calibration[~np.isfinite(mag_inst) + | ~np.isfinite(mag_catalog)] = False # Just creating some short-hands: x = mag_catalog[use_for_calibration] y = mag_inst[use_for_calibration] yerr = mag_inst_err[use_for_calibration] - weights = 1.0 / yerr ** 2 + weights = 1.0 / yerr**2 assert any(use_for_calibration), "No calibration stars" # Fit linear function with fixed slope, using sigma-clipping: model = models.Linear1D(slope=1, fixed={'slope': True}) - fitter = fitting.FittingWithOutlierRemoval(fitting.LinearLSQFitter(), sigma_clip, sigma=3.0) + fitter = fitting.FittingWithOutlierRemoval(fitting.LinearLSQFitter(), + sigma_clip, + sigma=3.0) best_fit, sigma_clipped = fitter(model, x, y, weights=weights) # Extract zero-point and estimate its error using a single weighted fit: @@ -598,7 +725,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi weights[sigma_clipped] = 0 # Trick to make following expression simpler n_weights = len(weights.nonzero()[0]) if n_weights > 1: - zp_error = np.sqrt(n_weights * nansum(weights * (y - best_fit(x)) ** 2) / nansum(weights) / (n_weights - 1)) + zp_error = np.sqrt(n_weights * nansum(weights * (y - best_fit(x))**2) / nansum(weights) / (n_weights - 1)) else: zp_error = np.NaN logger.info('Leastsquare ZP = %.3f, ZP_error = %.3f', zp, zp_error) @@ -612,10 +739,19 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Extract zero point and error using bootstrap method nboot = 1000 - logger.info('Running bootstrap with sigma = %.2f and n = %d', sig_chauv, nboot) - pars = bootstrap_outlier(x, y, yerr, n=nboot, model=model, fitter=fitting.LinearLSQFitter, - outlier=sigma_clip, outlier_kwargs={'sigma': sig_chauv}, summary='median', - error='bootstrap', return_vals=False) + logger.info('Running bootstrap with sigma = %.2f and n = %d', sig_chauv, + nboot) + pars = bootstrap_outlier(x, + y, + yerr, + n=nboot, + model=model, + fitter=fitting.LinearLSQFitter, + outlier=sigma_clip, + outlier_kwargs={'sigma': sig_chauv}, + summary='median', + error='bootstrap', + return_vals=False) zp_bs = pars['intercept'] * -1.0 zp_error_bs = pars['intercept_error'] @@ -625,12 +761,13 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Check that difference is not large zp_diff = 0.4 if np.abs(zp_bs - zp) >= zp_diff: - logger.warning("Bootstrap and weighted LSQ ZPs differ by %.2f, \ + logger.warning( + "Bootstrap and weighted LSQ ZPs differ by %.2f, \ which is more than the allowed %.2f mag.", np.abs(zp_bs - zp), zp_diff) # Add calibrated magnitudes to the photometry table: tab['mag'] = mag_inst + zp_bs - tab['mag_error'] = np.sqrt(mag_inst_err ** 2 + zp_error_bs ** 2) + tab['mag_error'] = np.sqrt(mag_inst_err**2 + zp_error_bs**2) fig, ax = plt.subplots(1, 1) ax.errorbar(x, y, yerr=yerr, fmt='k.') @@ -638,7 +775,8 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi ax.plot(x, best_fit(x), color='g', linewidth=3) ax.set_xlabel('Catalog magnitude') ax.set_ylabel('Instrumental magnitude') - fig.savefig(os.path.join(output_folder, 'calibration.png'), bbox_inches='tight') + fig.savefig(os.path.join(output_folder, 'calibration.png'), + bbox_inches='tight') plt.close(fig) # Check that we got valid photometry: @@ -667,8 +805,10 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi # Meta-data: tab.meta['version'] = __version__ tab.meta['fileid'] = fileid - tab.meta['template'] = None if datafile.get('template') is None else datafile['template']['fileid'] - tab.meta['diffimg'] = None if datafile.get('diffimg') is None else datafile['diffimg']['fileid'] + tab.meta['template'] = None if datafile.get( + 'template') is None else datafile['template']['fileid'] + tab.meta['diffimg'] = None if datafile.get( + 'diffimg') is None else datafile['diffimg']['fileid'] tab.meta['photfilter'] = photfilter tab.meta['fwhm'] = fwhm * u.pixel tab.meta['pixel_scale'] = pixel_scale * u.arcsec / u.pixel @@ -683,7 +823,10 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fi photometry_output = os.path.join(output_folder, 'photometry.ecsv') # Write the final table to file: - tab.write(photometry_output, format='ascii.ecsv', delimiter=',', overwrite=True) + tab.write(photometry_output, + format='ascii.ecsv', + delimiter=',', + overwrite=True) toc = default_timer() diff --git a/flows/references.py b/flows/references.py deleted file mode 100644 index 385989d..0000000 --- a/flows/references.py +++ /dev/null @@ -1,75 +0,0 @@ -from collections import OrderedDict - -import numpy as np - -from astropy.table import Table, TableColumns, Column - -class ReferenceColumns(TableColumns): - - def _set_image(self, image): - - self.image = image - - def keys(self): - - keys = list(super().keys()) - - if not hasattr(self, 'image'): - return keys - - keys += ['x'] if not 'x' in keys else [] - keys += ['y'] if not 'y' in keys else [] - - return keys - - def values(self): - - return [self[key] for key in self.keys()] - - def __len__(self): - - return len(self.keys()) - - def __iter__(self): - - return iter(key for key in self.keys()) - - def __getitem__(self, item): - - if item in ('x', 'y') and \ - {'ra', 'decl'} <= set(self.keys()) and \ - hasattr(self, 'image'): - - rd = list(zip(self['ra'], self['decl'])) - x, y = zip(*self.image.wcs.all_world2pix(rd, 0)) - - return Column({'x': x, 'y': y}[item], item) - - return super().__getitem__(item) - -class References(Table): - - def __init__(self, *args, **kwargs): - - super().__init__(*args, **kwargs) - - self.columns = ReferenceColumns(self.columns) - - def set_image(self, image): - - self.columns._set_image(image) - - def __getitem__(self, item): - - if item == 'xy': - return list(zip(self['x'], self['y'])) - - return super().__getitem__(item) - - def copy_with_image(self): - - table = self.copy() - del table['x'], table['y'] - table.set_image(self.columns.image) - - return table diff --git a/flows/wcs.py b/flows/wcs.py index 7b6926d..6fd1e9a 100644 --- a/flows/wcs.py +++ b/flows/wcs.py @@ -7,274 +7,339 @@ """ import numpy as np -from astropy.table import Table import astroalign as aa -import astropy.units as u import astropy.coordinates as coords import astropy.wcs as wcs -import astropy.io.fits as fits -from astropy.stats import sigma_clip, SigmaClip, gaussian_fwhm_to_sigma +from astropy.stats import sigma_clip, gaussian_fwhm_to_sigma from astropy.modeling import models, fitting from copy import deepcopy from bottleneck import nanmedian + class MinStarError(RuntimeError): - pass - -def force_reject_g2d(xarray, yarray, image, get_fwhm=True, rsq_min=0.5, radius=10, fwhm_guess=6.0, fwhm_min=3.5, - fwhm_max=18.0): - '''xarray, yarray, image, get_fwhm=True, rsq_min=0.5, radius=10, fwhm_guess=6.0, fwhm_min=3.5, - fwhm_max=18.0''' - # Set up 2D Gaussian model for fitting to reference stars: - g2d = models.Gaussian2D(amplitude=1.0, x_mean=radius, y_mean=radius, x_stddev=fwhm_guess * gaussian_fwhm_to_sigma) - g2d.amplitude.bounds = (0.1, 2.0) - g2d.x_mean.bounds = (0.5 * radius, 1.5 * radius) - g2d.y_mean.bounds = (0.5 * radius, 1.5 * radius) - g2d.x_stddev.bounds = (fwhm_min * gaussian_fwhm_to_sigma, fwhm_max * gaussian_fwhm_to_sigma) - g2d.y_stddev.tied = lambda model: model.x_stddev - g2d.theta.fixed = True - - gfitter = fitting.LevMarLSQFitter() - - # Stars reject - N = len(xarray) - fwhms = np.full((N, 2), np.NaN) - xys = np.full((N, 2), np.NaN) - rsqs = np.full(N, np.NaN) - for i, (x, y) in enumerate(zip(xarray, yarray)): - x = int(np.round(x)) - y = int(np.round(y)) - xmin = max(x - radius, 0) - xmax = min(x + radius + 1, image.shape[1]) - ymin = max(y - radius, 0) - ymax = min(y + radius + 1, image.shape[0]) - - curr_star = deepcopy(image.subclean[ymin:ymax, xmin:xmax]) - - edge = np.zeros_like(curr_star, dtype='bool') - edge[(0, -1), :] = True - edge[:, (0, -1)] = True - curr_star -= nanmedian(curr_star[edge]) - curr_star /= np.max(curr_star) - - ypos, xpos = np.mgrid[:curr_star.shape[0], :curr_star.shape[1]] - gfit = gfitter(g2d, x=xpos, y=ypos, z=curr_star) - - # Center - xys[i] = np.array([gfit.x_mean + x - radius, gfit.y_mean + y - radius], dtype=np.float64) - # Calculate rsq - sstot = ((curr_star - curr_star.mean()) ** 2).sum() - sserr = (gfitter.fit_info['fvec'] ** 2).sum() - rsqs[i] = 1. - (sserr / sstot) - # FWHM - fwhms[i] = gfit.x_fwhm - - masked_xys = np.ma.masked_array(xys, ~np.isfinite(xys)) - masked_rsqs = np.ma.masked_array(rsqs, ~np.isfinite(rsqs)) - mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) # Reject Rsq < rsq_min + pass + + +def force_reject_g2d(xarray, + yarray, + image, + get_fwhm=True, + rsq_min=0.5, + radius=10, + fwhm_guess=6.0, + fwhm_min=3.5, + fwhm_max=18.0): + '''xarray, yarray, image, get_fwhm=True, rsq_min=0.5, radius=10, fwhm_guess=6.0, fwhm_min=3.5, + fwhm_max=18.0''' + # Set up 2D Gaussian model for fitting to reference stars: + g2d = models.Gaussian2D(amplitude=1.0, + x_mean=radius, + y_mean=radius, + x_stddev=fwhm_guess * gaussian_fwhm_to_sigma) + g2d.amplitude.bounds = (0.1, 2.0) + g2d.x_mean.bounds = (0.5 * radius, 1.5 * radius) + g2d.y_mean.bounds = (0.5 * radius, 1.5 * radius) + g2d.x_stddev.bounds = (fwhm_min * gaussian_fwhm_to_sigma, + fwhm_max * gaussian_fwhm_to_sigma) + g2d.y_stddev.tied = lambda model: model.x_stddev + g2d.theta.fixed = True + + gfitter = fitting.LevMarLSQFitter() + + # Stars reject + N = len(xarray) + fwhms = np.full((N, 2), np.NaN) + xys = np.full((N, 2), np.NaN) + rsqs = np.full(N, np.NaN) + for i, (x, y) in enumerate(zip(xarray, yarray)): + x = int(np.round(x)) + y = int(np.round(y)) + xmin = max(x - radius, 0) + xmax = min(x + radius + 1, image.shape[1]) + ymin = max(y - radius, 0) + ymax = min(y + radius + 1, image.shape[0]) + + curr_star = deepcopy(image.subclean[ymin:ymax, xmin:xmax]) + + edge = np.zeros_like(curr_star, dtype='bool') + edge[(0, -1), :] = True + edge[:, (0, -1)] = True + curr_star -= nanmedian(curr_star[edge]) + curr_star /= np.max(curr_star) + + ypos, xpos = np.mgrid[:curr_star.shape[0], :curr_star.shape[1]] + gfit = gfitter(g2d, x=xpos, y=ypos, z=curr_star) + + # Center + xys[i] = np.array([gfit.x_mean + x - radius, gfit.y_mean + y - radius], + dtype=np.float64) + # Calculate rsq + sstot = ((curr_star - curr_star.mean())**2).sum() + sserr = (gfitter.fit_info['fvec']**2).sum() + rsqs[i] = 1. - (sserr / sstot) + # FWHM + fwhms[i] = gfit.x_fwhm + + masked_xys = np.ma.masked_array(xys, ~np.isfinite(xys)) + masked_rsqs = np.ma.masked_array(rsqs, ~np.isfinite(rsqs)) + mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0 + ) # Reject Rsq < rsq_min # changed #masked_xys = masked_xys[mask] # Clean extracted array. # to - masked_xys.mask[~mask] = True + masked_xys.mask[~mask] = True # don't know if it breaks anything, but it doesn't make sence if # len(masked_xys) != len(masked_rsqs) FIXME - masked_fwhms = np.ma.masked_array(fwhms, ~np.isfinite(fwhms)) - - if get_fwhm: return masked_fwhms,masked_xys,mask,masked_rsqs - return masked_xys,mask,masked_rsqs - - -def clean_with_rsq_and_get_fwhm(masked_fwhms, masked_rsqs, references, - min_fwhm_references = 2, min_references = 6, rsq_min = 0.15): - """ - Clean references and obtain fwhm using RSQ values. - Args: - masked_fwhms (np.ma.maskedarray): array of fwhms - masked_rsqs (np.ma.maskedarray): array of rsq values - references (astropy.table.Table): table or reference stars - min_fwhm_references: (Default 2) min stars to get a fwhm - min_references: (Default 6) min stars to aim for when cutting by R2 - rsq_min: (Default 0.15) min rsq value - """ - min_references_now = min_references - rsqvals = np.arange(rsq_min, 0.95, 0.15)[::-1] - fwhm_found = False - min_references_achieved = False - - # Clean based on R^2 Value - while not min_references_achieved: - for rsqval in rsqvals: - mask = (masked_rsqs >= rsqval) & (masked_rsqs < 1.0) - nreferences = np.sum(np.isfinite(masked_fwhms[mask])) - if nreferences >= min_fwhm_references: - _fwhms_cut_ = np.nanmean(sigma_clip(masked_fwhms[mask], maxiters=100, sigma=2.0)) - #logger.info('R^2 >= ' + str(rsqval) + ': ' + str( - # np.sum(np.isfinite(masked_fwhms[mask]))) + ' stars w/ mean FWHM = ' + str(np.round(_fwhms_cut_, 1))) - if not fwhm_found: - fwhm = _fwhms_cut_ - fwhm_found = True - if nreferences >= min_references_now: - references = references[mask] - min_references_achieved = True - break - if min_references_achieved: break - min_references_now = min_references_now - 2 - if (min_references_now < 2) and fwhm_found: - break - elif not fwhm_found: - raise Exception("Could not estimate FWHM") - #logger.debug('{} {} {}'.format(min_references_now, min_fwhm_references, nreferences)) - - #logger.info("FWHM: %f", fwhm) - if np.isnan(fwhm): - raise Exception("Could not estimate FWHM") - - # if minimum references not found, then take what we can get with even a weaker cut. - # TODO: Is this right, or should we grab rsq_min (or even weaker?) - min_references_now = min_references - 2 - while not min_references_achieved: - mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) - nreferences = np.sum(np.isfinite(masked_fwhms[mask])) - if nreferences >= min_references_now: - references = references[mask] - min_references_achieved = True - rsq_min = rsq_min - 0.07 - min_references_now = min_references_now - 1 - - # Check len of references as this is a destructive cleaning. - # if len(references) == 2: logger.info('2 reference stars remaining, check WCS and image quality') - if len(references) < 2: - raise Exception("{} References remaining; could not clean.".format(len(references))) - return fwhm, references + masked_fwhms = np.ma.masked_array(fwhms, ~np.isfinite(fwhms)) + + if get_fwhm: return masked_fwhms, masked_xys, mask, masked_rsqs + return masked_xys, mask, masked_rsqs + + +def clean_with_rsq_and_get_fwhm(masked_fwhms, + masked_rsqs, + references, + min_fwhm_references=2, + min_references=6, + rsq_min=0.15): + """ + Clean references and obtain fwhm using RSQ values. + Args: + masked_fwhms (np.ma.maskedarray): array of fwhms + masked_rsqs (np.ma.maskedarray): array of rsq values + references (astropy.table.Table): table or reference stars + min_fwhm_references: (Default 2) min stars to get a fwhm + min_references: (Default 6) min stars to aim for when cutting by R2 + rsq_min: (Default 0.15) min rsq value + """ + min_references_now = min_references + rsqvals = np.arange(rsq_min, 0.95, 0.15)[::-1] + fwhm_found = False + min_references_achieved = False + + # Clean based on R^2 Value + while not min_references_achieved: + for rsqval in rsqvals: + mask = (masked_rsqs >= rsqval) & (masked_rsqs < 1.0) + nreferences = np.sum(np.isfinite(masked_fwhms[mask])) + if nreferences >= min_fwhm_references: + _fwhms_cut_ = np.nanmean( + sigma_clip(masked_fwhms[mask], maxiters=100, sigma=2.0)) + #logger.info('R^2 >= ' + str(rsqval) + ': ' + str( + # np.sum(np.isfinite(masked_fwhms[mask]))) + ' stars w/ mean FWHM = ' + str(np.round(_fwhms_cut_, 1))) + if not fwhm_found: + fwhm = _fwhms_cut_ + fwhm_found = True + if nreferences >= min_references_now: + references = references[mask] + min_references_achieved = True + break + if min_references_achieved: break + min_references_now = min_references_now - 2 + if (min_references_now < 2) and fwhm_found: + break + elif not fwhm_found: + raise Exception("Could not estimate FWHM") + #logger.debug('{} {} {}'.format(min_references_now, min_fwhm_references, nreferences)) + + #logger.info("FWHM: %f", fwhm) + if np.isnan(fwhm): + raise Exception("Could not estimate FWHM") + + # if minimum references not found, then take what we can get with even a weaker cut. + # TODO: Is this right, or should we grab rsq_min (or even weaker?) + min_references_now = min_references - 2 + while not min_references_achieved: + mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) + nreferences = np.sum(np.isfinite(masked_fwhms[mask])) + if nreferences >= min_references_now: + references = references[mask] + min_references_achieved = True + rsq_min = rsq_min - 0.07 + min_references_now = min_references_now - 1 + + # Check len of references as this is a destructive cleaning. + # if len(references) == 2: logger.info('2 reference stars remaining, check WCS and image quality') + if len(references) < 2: + raise Exception("{} References remaining; could not clean.".format( + len(references))) + return fwhm, references def mkposxy(posx, posy): - '''Make 2D np array for astroalign''' - img_posxy = np.array([[x, y] for x, y in zip(posx, posy)], dtype="float64") - return img_posxy + '''Make 2D np array for astroalign''' + img_posxy = np.array([[x, y] for x, y in zip(posx, posy)], dtype="float64") + return img_posxy + def try_transform(source, target, pixeltol=2, nnearest=5, max_stars=50): - aa.NUM_NEAREST_NEIGHBORS = nnearest - aa.PIXEL_TOL = pixeltol - transform,(sourcestars, targetstars) = aa.find_transform(source, target, max_control_points=max_stars) - return sourcestars, targetstars + aa.NUM_NEAREST_NEIGHBORS = nnearest + aa.PIXEL_TOL = pixeltol + transform, (sourcestars, + targetstars) = aa.find_transform(source, + target, + max_control_points=max_stars) + return sourcestars, targetstars + def try_astroalign(source, target, pixeltol=2, nnearest=5, max_stars_n=50): - # Get indexes of matched stars - success = False - try: - source_stars, target_stars = try_transform(source, target, - pixeltol=pixeltol, nnearest=nnearest, max_stars=max_stars_n) - source_ind = np.argwhere(np.in1d(source, source_stars)[::2]).flatten() - target_ind = np.argwhere(np.in1d(target, target_stars)[::2]).flatten() - success = True - except aa.MaxIterError: - source_ind, target_ind = 'None', 'None' - return source_ind,target_ind,success - - -def min_to_max_astroalign(source, target, fwhm=5, fwhm_min=1, fwhm_max=4, knn_min=5, knn_max=20, - max_stars=100, min_matches=3): - '''Try to find matches using astroalign asterisms by stepping through some parameters.''' - # Set max_control_points par based on number of stars and max_stars. - nstars = max(len(source), len(source)) - if max_stars >= nstars : max_stars_list = 'None' - else: - if max_stars > 60: max_stars_list = (max_stars,50,4,3) - else: max_stars_list = (max_stars,6,4,3) - - # Create max_stars step-through list if not given - if max_stars_list == 'None': - if nstars > 6: - max_stars_list = (nstars, 5, 3) - elif nstars > 3: - max_stars_list = (nstars, 3) - - pixeltols = np.linspace(int(fwhm*fwhm_min), int(fwhm*fwhm_max), 4, dtype=int) - nearest_neighbors = np.linspace(knn_min, min(knn_max,nstars), 4, dtype=int) - - for max_stars_n in max_stars_list: - for pixeltol in pixeltols: - for nnearest in nearest_neighbors: - source_ind,target_ind,success = try_astroalign(source, - target, - pixeltol=pixeltol, - nnearest=nnearest, - max_stars_n=max_stars_n) - if success: - if len(source_ind) >= min_matches: - return source_ind, target_ind, success - else: success = False - return 'None', 'None', success + # Get indexes of matched stars + success = False + try: + source_stars, target_stars = try_transform(source, + target, + pixeltol=pixeltol, + nnearest=nnearest, + max_stars=max_stars_n) + source_ind = np.argwhere(np.in1d(source, source_stars)[::2]).flatten() + target_ind = np.argwhere(np.in1d(target, target_stars)[::2]).flatten() + success = True + except aa.MaxIterError: + source_ind, target_ind = 'None', 'None' + return source_ind, target_ind, success + + +def min_to_max_astroalign(source, + target, + fwhm=5, + fwhm_min=1, + fwhm_max=4, + knn_min=5, + knn_max=20, + max_stars=100, + min_matches=3): + '''Try to find matches using astroalign asterisms by stepping through some parameters.''' + # Set max_control_points par based on number of stars and max_stars. + nstars = max(len(source), len(source)) + if max_stars >= nstars: max_stars_list = 'None' + else: + if max_stars > 60: max_stars_list = (max_stars, 50, 4, 3) + else: max_stars_list = (max_stars, 6, 4, 3) + + # Create max_stars step-through list if not given + if max_stars_list == 'None': + if nstars > 6: + max_stars_list = (nstars, 5, 3) + elif nstars > 3: + max_stars_list = (nstars, 3) + + pixeltols = np.linspace(int(fwhm * fwhm_min), + int(fwhm * fwhm_max), + 4, + dtype=int) + nearest_neighbors = np.linspace(knn_min, + min(knn_max, nstars), + 4, + dtype=int) + + for max_stars_n in max_stars_list: + for pixeltol in pixeltols: + for nnearest in nearest_neighbors: + source_ind, target_ind, success = try_astroalign( + source, + target, + pixeltol=pixeltol, + nnearest=nnearest, + max_stars_n=max_stars_n) + if success: + if len(source_ind) >= min_matches: + return source_ind, target_ind, success + else: + success = False + return 'None', 'None', success + def kdtree(source, target, fwhm=5, fwhm_max=4, min_matches=3): - '''Use KDTree to get nearest neighbor matches within fwhm_max*fwhm distance''' - - # Use KDTree to rapidly efficiently query nearest neighbors - from scipy.spatial import KDTree - tt = KDTree(target) - st = KDTree(source) - matches_list = st.query_ball_tree(tt, r=fwhm*fwhm_max) - - #indx = [] - targets = [] - sources = [] - for j, (sstar, match) in enumerate(zip(source, matches_list)): - if np.array(target[match]).size != 0: - targets.append(match[0]) - sources.append(j) - sources = np.array(sources, dtype=int) - targets = np.array(targets, dtype=int) - # Return indexes of matches - return sources, targets, len(sources)>= min_matches - -def get_new_wcs(extracted_ind,extracted_stars,clean_references,ref_ind,obstime,rakey='ra_obs',deckey='decl_obs'): - targets = (extracted_stars[extracted_ind][:,0],extracted_stars[extracted_ind][:,1]) - c = coords.SkyCoord(clean_references[rakey][ref_ind],clean_references[deckey][ref_ind],obstime=obstime) - return wcs.utils.fit_wcs_from_points(targets,c) - -def get_clean_references(references, masked_rsqs, min_references_ideal=6, - min_references_abs=3, rsq_min=0.15, rsq_ideal=0.5, - keep_max=100, - rescue_bad: bool = True): - - # Greedy first try - mask = (masked_rsqs >= rsq_ideal) & (masked_rsqs < 1.0) - if np.sum(np.isfinite(masked_rsqs[mask])) >= min_references_ideal: - if len(references[mask]) <= keep_max: - return references[mask] - elif len(references[mask]) >= keep_max: - import pandas as pd # @TODO: Convert to pure numpy implementation - df = pd.DataFrame(masked_rsqs,columns=['rsq']) - masked_rsqs.mask = ~mask - nmasked_rsqs = df.sort_values('rsq',ascending=False).dropna().index._data - return references[nmasked_rsqs[:keep_max]] - - # Desperate second try - mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) - masked_rsqs.mask = ~mask - - # Switching to pandas for easier selection - import pandas as pd # @TODO: Convert to pure numpy implementation - df = pd.DataFrame(masked_rsqs,columns=['rsq']) - nmasked_rsqs = deepcopy(df.sort_values('rsq',ascending=False).dropna().index._data) - nmasked_rsqs = nmasked_rsqs[:min(min_references_ideal, len(nmasked_rsqs))] - if len(nmasked_rsqs) >= min_references_abs: - return references[nmasked_rsqs] - if not rescue_bad: - raise MinStarError('Less than {} clean stars and rescue_bad = False'.format(min_references_abs)) - - # Extremely desperate last ditch attempt i.e. "rescue bad" - elif rescue_bad: - mask = (masked_rsqs >= 0.02) & (masked_rsqs < 1.0) - masked_rsqs.mask = ~mask - - # Switch to pandas - df = pd.DataFrame(masked_rsqs,columns=['rsq']) - nmasked_rsqs = df.sort_values('rsq',ascending=False).dropna().index._data - nmasked_rsqs = nmasked_rsqs[:min(min_references_ideal, len(nmasked_rsqs))] - if len(nmasked_rsqs) < 2 : - raise MinStarError('Less than 2 clean stars.') - return references[nmasked_rsqs] # Return if len >= 2 - # Checks whether sensible input arrays and parameters were provided - raise ValueError('input parameters were wrong, you should not reach here. Check that rescue_bad is True or False.') + '''Use KDTree to get nearest neighbor matches within fwhm_max*fwhm distance''' + + # Use KDTree to rapidly efficiently query nearest neighbors + from scipy.spatial import KDTree + tt = KDTree(target) + st = KDTree(source) + matches_list = st.query_ball_tree(tt, r=fwhm * fwhm_max) + + #indx = [] + targets = [] + sources = [] + for j, (sstar, match) in enumerate(zip(source, matches_list)): + if np.array(target[match]).size != 0: + targets.append(match[0]) + sources.append(j) + sources = np.array(sources, dtype=int) + targets = np.array(targets, dtype=int) + # Return indexes of matches + return sources, targets, len(sources) >= min_matches + + +def get_new_wcs(extracted_ind, + extracted_stars, + clean_references, + ref_ind, + obstime, + rakey='ra_obs', + deckey='decl_obs'): + targets = (extracted_stars[extracted_ind][:, 0], + extracted_stars[extracted_ind][:, 1]) + c = coords.SkyCoord(clean_references[rakey][ref_ind], + clean_references[deckey][ref_ind], + obstime=obstime) + return wcs.utils.fit_wcs_from_points(targets, c) + + +def get_clean_references(references, + masked_rsqs, + min_references_ideal=6, + min_references_abs=3, + rsq_min=0.15, + rsq_ideal=0.5, + keep_max=100, + rescue_bad: bool = True): + + # Greedy first try + mask = (masked_rsqs >= rsq_ideal) & (masked_rsqs < 1.0) + if np.sum(np.isfinite(masked_rsqs[mask])) >= min_references_ideal: + if len(references[mask]) <= keep_max: + return references[mask] + elif len(references[mask]) >= keep_max: + import pandas as pd # @TODO: Convert to pure numpy implementation + df = pd.DataFrame(masked_rsqs, columns=['rsq']) + masked_rsqs.mask = ~mask + nmasked_rsqs = df.sort_values('rsq', + ascending=False).dropna().index._data + return references[nmasked_rsqs[:keep_max]] + + # Desperate second try + mask = (masked_rsqs >= rsq_min) & (masked_rsqs < 1.0) + masked_rsqs.mask = ~mask + + # Switching to pandas for easier selection + import pandas as pd # @TODO: Convert to pure numpy implementation + df = pd.DataFrame(masked_rsqs, columns=['rsq']) + nmasked_rsqs = deepcopy( + df.sort_values('rsq', ascending=False).dropna().index._data) + nmasked_rsqs = nmasked_rsqs[:min(min_references_ideal, len(nmasked_rsqs))] + if len(nmasked_rsqs) >= min_references_abs: + return references[nmasked_rsqs] + if not rescue_bad: + raise MinStarError( + 'Less than {} clean stars and rescue_bad = False'.format( + min_references_abs)) + + # Extremely desperate last ditch attempt i.e. "rescue bad" + elif rescue_bad: + mask = (masked_rsqs >= 0.02) & (masked_rsqs < 1.0) + masked_rsqs.mask = ~mask + + # Switch to pandas + df = pd.DataFrame(masked_rsqs, columns=['rsq']) + nmasked_rsqs = df.sort_values('rsq', + ascending=False).dropna().index._data + nmasked_rsqs = nmasked_rsqs[:min(min_references_ideal, len(nmasked_rsqs + ))] + if len(nmasked_rsqs) < 2: + raise MinStarError('Less than 2 clean stars.') + return references[nmasked_rsqs] # Return if len >= 2 + # Checks whether sensible input arrays and parameters were provided + raise ValueError( + 'input parameters were wrong, you should not reach here. Check that rescue_bad is True or False.' + ) diff --git a/requirements.txt b/requirements.txt index d304ccd..08f90dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,8 +10,8 @@ mplcursors == 0.3 seaborn pandas requests -astropy >=4.2 -photutils > 1.0 +astropy >= 4.1 +photutils >= 1.0.2 PyYAML psycopg2-binary jplephem @@ -21,8 +21,6 @@ tqdm pytz beautifulsoup4 git+https://github.com/obscode/imagematch.git@photutils#egg=imagematch -pandas sep astroalign > 2.3 networkx -flask diff --git a/run_localweb.py b/run_localweb.py deleted file mode 100644 index 88a8abf..0000000 --- a/run_localweb.py +++ /dev/null @@ -1,179 +0,0 @@ -import os, json -from copy import deepcopy -from base64 import b64encode as b64 - -import numpy as np - -from astropy.io import ascii -from astropy.coordinates import SkyCoord - -from flask import Flask, request, render_template - -from flows import load_config -from flows.api import get_targets, get_target -from flows.api import get_datafiles, get_datafile -from flows.api import get_site - -from web.api import targets, catalogs, datafiles, sites - -from functools import lru_cache -get_datafile = lru_cache(maxsize=1000)(get_datafile.__wrapped__) - -app = Flask(__name__, template_folder='web', static_folder='web/static') - -@app.route('/api/targets.php') -def api_targets(): - if 'target' in request.args: - target = request.args['target'] - if target in targets: - target = targets[target] - else: - if (target := get_target_by_id(target)) is None: - return '""' - return json.dumps(target) - return json.dumps(list(targets.values())) - -@app.route('/api/datafiles.php') -def api_datafiles(): - if 'fileid' in request.args: - fileid = int(request.args['fileid']) - for name in targets: - for datafile in datafiles[name]: - if datafile['fileid'] == fileid: - return datafile - elif 'targetid' in request.args: - targetid = int(request.args['targetid']) - if (target := get_target_by_id(targetid)) is None: - return '""' - return str([datafile['fileid'] for datafile in datafiles[target['target_name']]]) - return '""' - -@app.route('/api/reference_stars.php') -def api_catalogs(): - targetid = target_name = request.args['target'] - if (target := get_target_by_id(targetid)) is None and \ - (target := get_target_by_name(target_name)) is None: - return '""' - catalog = catalogs[target["target_name"]] - catalog['target'] = targets[target["target_name"]] - catalog['avoid'] = None - return json.dumps(catalog) - -@app.route('/api/sites.php') -def api_sites(): - if 'siteid' in request.args: - siteid = int(request.args['siteid']) - site = [s for s in sites if siteid == s['siteid']] - site = site[0] if site else '' - return json.dumps(site) - return json.dumps(sites) - -@app.route('/') -def index(): - targets = sorted(deepcopy(get_targets()), key=lambda t: t['target_name'])[::-1] - local_targets = os.listdir(load_config().get('photometry', 'archive_local', fallback='/')) - for target in targets: - c = SkyCoord(target['ra'], target['decl'], unit='deg') - target['ra'], target['decl'] = c.to_string('hmsdms').split() - target['inserted'] = target['inserted'].strftime('%Y-%m-%d %H:%M:%S') - target['local'] = target['target_name'] in local_targets - return render_template('index.html', targets=targets) - -@app.route('/') -def target(target): - if not (target := get_target_by_name(target)): - return '' - datafiles = [deepcopy(get_datafile(datafile)) for datafile in get_datafiles(target['targetid'], 'all')] - datafiles = sorted(datafiles, key=lambda f: f['obstime']) - archive = '%s/%s' % (load_config().get('photometry', 'archive_local', fallback=''), target['target_name']) - output = '%s/%s' % (load_config().get('photometry', 'output', fallback=''), target['target_name']) - fileids = {int(fileid): fileid for fileid in os.listdir(output)} - for datafile in datafiles: - fileid = fileids.get(datafile['fileid'], '') - datafile['filename'] = filename = datafile['path'].split('/')[-1] - datafile['sitename'] = get_site(datafile['site'])['sitename'] if not datafile['site'] is None else 'None' - datafile['exptime'] = '%.2f' % datafile['exptime'] if not datafile['exptime'] is None else 'None' - datafile['inserted'] = datafile['inserted'].strftime('%Y-%m-%d %H:%M:%S') - datafile['is_local'] = os.path.isfile(f'{archive}/{filename}') - datafile['has_phot'] = os.path.isfile(f'{output}/{fileid}/photometry.ecsv') - return render_template('target.html', target=target, datafiles=datafiles) - -@app.route('//photometry.js') -def photometry(target): - if not (target := get_target_by_name(target)): - return '' - output = load_config().get('photometry', 'output', fallback='') - fileids = get_datafiles(target['targetid']) - photometry = dict() - for fileid in os.listdir(f'{output}/%s' % target['target_name']): - if not int(fileid) in fileids: - continue - try: - table = ascii.read(f'{output}/%s/{fileid}/photometry.ecsv' % target['target_name']) - except FileNotFoundError: - continue - filt, mjd = table.meta['photfilter'], table.meta['obstime-bmjd'] - for i in np.where(table['starid'] <= 0)[0]: - mag, err = table[i]['mag'], table[i]['mag_error'] - _filt = 's_' + filt if table[i]['starid'] else filt - if not _filt in photometry: - photometry[_filt] = [] - photometry[_filt].append((mjd, mag, err, 'fileid: %d' % int(fileid))) - photometry = {filt: list(map(list, zip(*photometry[filt]))) for filt in sorted(photometry)} - return render_template('photometry.js', photometry=photometry) - -@app.route('//') -def datafile(target, fileid): - if not (target := get_target_by_name(target)): - return '' - output = '%s/%s' % (load_config().get('photometry', 'output', fallback=''), target['target_name']) - fileids = {int(fileid): fileid for fileid in os.listdir(output)} - try: - photometry = ascii.read(f'{output}/{fileids[fileid]}/photometry.ecsv') - except (FileNotFoundError, KeyError): - photometry = [{'starid': 0, 'ra': target['ra'], 'decl': target['decl'], 'distance': 0}] - try: - with open(f'{output}/{fileids[fileid]}/photometry.log', 'r') as fd: - log = fd.read() - except (FileNotFoundError, KeyError): - log = '' - try: - images = [] - for f in sorted(os.listdir(f'{output}/{fileids[fileid]}')): - if f.split('.')[-1] != 'png': - continue - with open(f'{output}/{fileids[fileid]}/{f}', 'rb') as fd: - images.append(b64(fd.read()).decode('utf-8')) - except KeyError: - pass - return render_template('datafile.html', target=target, fileid=fileid, photometry=photometry, log=log, images=images) - -@app.route('//.fits') -def fits(target, fileid): - archive = load_config().get('photometry', 'archive_local', fallback='/') - try: - datafile = get_datafile(fileid) - except: - return '' - if 'subtracted' in request.args: - try: - path = f'{archive}/' + datafile['diffimg']['path'] - except: - return '' - else: - path = f'{archive}/' + datafile['path'] - with open(path, 'rb') as fd: - return fd.read() - -def get_target_by_id(targetid): - for target in get_targets(): - if str(target['targetid']) == str(targetid): - return target - -def get_target_by_name(target_name): - for target in get_targets(): - if target['target_name'] == target_name: - return target - -if __name__ == '__main__': - app.run(host='0.0.0.0', debug=True) diff --git a/web/api/__init__.py b/web/api/__init__.py deleted file mode 100644 index e19ae42..0000000 --- a/web/api/__init__.py +++ /dev/null @@ -1,86 +0,0 @@ -import os, json - -DIR = os.path.dirname(os.path.abspath(__file__)) - -with open("%s/targets.json" % DIR, 'r') as fd: - targets = json.load(fd) - -for target in targets: - targets[target] = {**{ - 'target_status' : None, - 'redshift' : None, - 'redshift_error' : None, - 'discovery_mag' : None, - 'catalog_downloaded' : None, - 'pointing_model_created' : '1970-01-01 00:00:00.0', - 'inserted' : '1970-01-01 00:00:00.0', - 'discovery_date' : '1970-01-01 00:00:00.0', - 'project' : None, - 'host_galaxy' : None, - 'ztf_id' : None - }, **targets[target]} - -with open("%s/catalogs.json" % DIR, 'r') as fd: - catalogs = json.load(fd) - -for target in catalogs: - catalogs[target] = { - 'references': [ - {**{ - 'pm_ra' : 0, - 'pm_dec' : 0, - 'gaia_mag' : None, - 'gaia_bp_mag' : None, - 'gaia_rp_mag' : None, - 'J_mag' : None, - 'H_mag' : None, - 'K_mag' : None, - 'g_mag' : None, - 'r_mag' : None, - 'i_mag' : None, - 'z_mag' : None, - 'gaia_variability' : 0, - 'V_mag' : None, - 'B_mag' : None, - 'u_mag' : None, - 'distance' : None - }, **reference} - for reference in catalogs[target]] - } - -with open("%s/datafiles.json" % DIR, 'r') as fd: - datafiles = json.load(fd) - -for target in datafiles: - for i, datafile in enumerate(datafiles[target]): - if "diffimg" in datafile and datafile["diffimg"]: - datafile["diffimg"] = {**{ - 'filehash' : None, - 'filesize' : None - }, **datafile["diffimg"]} - datafiles[target][i] = {**{ - 'site' : None, - 'filesize' : None, - 'filehash' : None, - 'inserted' : '1970-01-01 00:00:00.0', - 'lastmodified' : '1970-01-01 00:00:00.0', - 'obstime' : None, - 'exptime' : None, - 'version' : None, - 'archive_path' : None, - 'template' : None, - 'diffimg' : None - }, **datafile} - -with open("%s/sites.json" % DIR, 'r') as fd: - sites = json.load(fd) - -for i, site in enumerate(sites): - sites[i] = {**{ - 'siteid' : -1, - 'sitename' : None, - 'longitude': None, - 'latitude' : None, - 'elevation' : None, - 'site_keyword' : None, - }, **site} diff --git a/web/api/catalogs.json b/web/api/catalogs.json deleted file mode 100644 index 2a62250..0000000 --- a/web/api/catalogs.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "2019yvr": [ - { - "starid": 107481912845754726, - "ra": 191.28457586, - "decl": -0.42975258, - "i_mag": 14.282 - }, { - "starid": 107411912811133651, - "ra": 191.28111312, - "decl": -0.48898194, - "i_mag": 15.234 - }, { - "starid": 107391913152190877, - "ra": 191.31521935, - "decl": -0.50796039, - "i_mag": 15.564 - }, { - "starid": 107401912890621513, - "ra": 191.2890621, - "decl": -0.49909695, - "i_mag": 15.966 - }, { - "starid": 107401912782199891, - "ra": 191.27821937, - "decl": -0.49211498, - "i_mag": 16.631 - }, { - "starid": 107411912729499005, - "ra": 191.27294953, - "decl": -0.48452017, - "i_mag": 16.71 - }, { - "starid": 107431912569317799, - "ra": 191.25693153, - "decl": -0.46885893, - "i_mag": 16.938 - }, { - "starid": 107421912971816909, - "ra": 191.29718197, - "decl": -0.47793321, - "i_mag": 17.435 - } - ] -} diff --git a/web/api/datafiles.json b/web/api/datafiles.json deleted file mode 100644 index efc0bf8..0000000 --- a/web/api/datafiles.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "2019yvr": [ - { - "fileid": 441, - "targetid": 2, - "target_name": "2019yvr", - "photfilter": "ip", - "path": "2019yvr/SN2019yvr_i01_NOT_ALFOSC_20200104.fits.gz", - "diffimg" : { - "fileid" : 627, - "path" : "2019yvr/subtracted/SN2019yvr_i01_NOT_ALFOSC_20200104diff.fits.gz" - } - } - ] -} diff --git a/web/api/sites.json b/web/api/sites.json deleted file mode 100644 index 6df26de..0000000 --- a/web/api/sites.json +++ /dev/null @@ -1,7 +0,0 @@ -[{ - "siteid": 5, - "sitename": "NOT", - "longitude": -17.88508, - "latitude": 28.75728, - "elevation": 2382 -}] diff --git a/web/api/targets.json b/web/api/targets.json deleted file mode 100644 index 2f78599..0000000 --- a/web/api/targets.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "2019yvr": { - "target_name": "2019yvr", - "targetid": 2, - "ra": 191.283890127, - "decl": -0.45909033652 - } -} diff --git a/web/datafile.html b/web/datafile.html deleted file mode 100644 index af14844..0000000 --- a/web/datafile.html +++ /dev/null @@ -1,152 +0,0 @@ -{% extends 'index.html' %} - -{% block head %} - - - - - - - - - - -{% endblock %} - -{% block title %} -flows.localweb / -{{ target['target_name'] }} / -{{ fileid }} - -
-
{{ log }}
-
- -
-
-
-{% for img in images %} -
-{% endfor %} -
-
-{% endblock %} - -{% block body %} -
-
-
-
-
-
- -
-
- - - -
-
- -
-
Star ID
-
Right Ascension
-
Declination
-
Distance
-
Magnitude
-
Error
-
- -{% for phot in photometry %} -
-
{{ phot['starid'] }}
-
{{ phot['ra'] }}
-
{{ phot['decl'] }}
-
{{ phot['distance'] }}
-
{{ phot['mag'] }}
-
{{ phot['mag_error'] }}
-
-{% endfor %} - -{% endblock %} diff --git a/web/index.html b/web/index.html deleted file mode 100644 index 67a8c62..0000000 --- a/web/index.html +++ /dev/null @@ -1,100 +0,0 @@ - - - - - - - - - - - -{% block head %} - - - -{% endblock %} - -flows.localweb - - - -
- -
-{% block title %} -flows.localweb -{% endblock %} -
- -{% block body %} - -
-
Target
-
Target ID
-
Right Ascension
-
Declination
-
Redshift
-
Host galaxy
-
Inserted
-
- -{% for target in targets %} -
-
{{ target['target_name'] }}
-
{{ target['targetid'] }}
-
{{ target['ra'] }}
-
{{ target['decl'] }}
-
{{ target['redshift'] }}
-
{{ target['host_galaxy'] }}
-
{{ target['inserted'] }}
-
-{% endfor %} - -{% endblock %} - -
- -
- - - diff --git a/web/photometry.js b/web/photometry.js deleted file mode 100644 index 34d10f9..0000000 --- a/web/photometry.js +++ /dev/null @@ -1,31 +0,0 @@ -var photometry = [ -{% for f in photometry %} - { - x: {{ photometry[f][0] }}, - y: {{ photometry[f][1] }}, - error_y: { - type: 'data', - array: {{ photometry[f][2] }}, - visible: true - }, - name: '{{ f }}', - text: {{ photometry[f][3] }}, - mode: 'markers', - type: 'scatter', -{% if f[:2] == 's_' %} - marker: { - size: 10, - symbol: 'circle', - }, - opacity: .8, -{% else %} - marker: { - size: 10, - symbol: 'square', - }, - opacity: .5, - visible: 'legendonly', -{% endif %} - }, -{% endfor %} -] diff --git a/web/static/README.md b/web/static/README.md deleted file mode 100644 index fa00a14..0000000 --- a/web/static/README.md +++ /dev/null @@ -1 +0,0 @@ -Put js9 in a folder called js9 diff --git a/web/target.html b/web/target.html deleted file mode 100644 index 2993f6e..0000000 --- a/web/target.html +++ /dev/null @@ -1,92 +0,0 @@ -{% extends 'index.html' %} - -{% block head %} - - - - - -{% endblock %} - -{% block title %} -flows.localweb / -{{ target['target_name'] }} -{% endblock %} - -{% block body %} -
- -
- -
-
File name
-
File ID
-
Site
-
Obstime
-
Filter
-
Exp. time
-
Inserted
-
- -{% for datafile in datafiles %} -
-
{{ datafile['filename'] }}
-
{{ datafile['fileid'] }}
-
{{ datafile['sitename'] }}
-
{{ datafile['obstime'] }}
-
{{ datafile['photfilter'] }}
-
{{ datafile['exptime'] }}
-
{{ datafile['inserted'] }}
-
-{% endfor %} - -{% endblock %} From 249f44e72b404593e70164a24d016099bf09eaff Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 21 Apr 2021 15:11:51 +0200 Subject: [PATCH 39/43] Rasmus Comments, flake8 things. --- flows/coordinatematch/__init__.py | 4 +-- flows/epsfbuilder/__init__.py | 2 +- flows/epsfbuilder/epsfbuilder.py | 5 ---- flows/load_image.py | 3 +-- flows/photometry.py | 42 +++++++++++++++---------------- run_photometry.py | 4 +-- 6 files changed, 27 insertions(+), 33 deletions(-) diff --git a/flows/coordinatematch/__init__.py b/flows/coordinatematch/__init__.py index 93e91ab..34a7466 100644 --- a/flows/coordinatematch/__init__.py +++ b/flows/coordinatematch/__init__.py @@ -1,2 +1,2 @@ -from .coordinatematch import CoordinateMatch # noqa -from .wcs import WCS2 # noqa +from .coordinatematch import CoordinateMatch # noqa: F401 +from .wcs import WCS2 # noqa: F401 diff --git a/flows/epsfbuilder/__init__.py b/flows/epsfbuilder/__init__.py index a7b2ccf..dfa50fe 100644 --- a/flows/epsfbuilder/__init__.py +++ b/flows/epsfbuilder/__init__.py @@ -1 +1 @@ -from .epsfbuilder import EPSFBuilder # noqa +from .epsfbuilder import EPSFBuilder # noqa: F401 diff --git a/flows/epsfbuilder/epsfbuilder.py b/flows/epsfbuilder/epsfbuilder.py index a9fd2af..1b7bf29 100644 --- a/flows/epsfbuilder/epsfbuilder.py +++ b/flows/epsfbuilder/epsfbuilder.py @@ -5,15 +5,10 @@ .. codeauthor:: Simon Holmbo """ import time - import numpy as np - -#from scipy.spatial import cKDTree from scipy.interpolate import griddata - import photutils.psf - class EPSFBuilder(photutils.psf.EPSFBuilder): def _create_initial_epsf(self, stars): diff --git a/flows/load_image.py b/flows/load_image.py index 579e875..cd80436 100644 --- a/flows/load_image.py +++ b/flows/load_image.py @@ -85,8 +85,7 @@ def load_image(FILENAME): image.image = np.asarray(hdul[0].data, dtype='float64') image.shape = image.image.shape - image.head = hdr - image.exthdu = [hdu.copy() for hdu in hdul[1:]] + image.header = hdr if origin == 'LCOGT': image.mask = np.asarray(hdul['BPM'].data, dtype='bool') diff --git a/flows/photometry.py b/flows/photometry.py index 66569eb..93a203e 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -6,7 +6,6 @@ .. codeauthor:: Rasmus Handberg .. codeauthor:: Emir Karamehmetoglu """ - import os import numpy as np from bottleneck import nansum, allnan, replace @@ -14,7 +13,6 @@ from timeit import default_timer import logging import warnings - from astropy.utils.exceptions import AstropyDeprecationWarning, AstropyUserWarning, ErfaWarning import astropy.units as u import astropy.coordinates as coords @@ -26,23 +24,23 @@ from astropy.time import Time warnings.simplefilter('ignore', category=AstropyDeprecationWarning) -from photutils import CircularAperture, CircularAnnulus, aperture_photometry # noqa -from photutils.psf import EPSFFitter, BasicPSFPhotometry, DAOGroup, extract_stars # noqa -from photutils import Background2D, SExtractorBackground, MedianBackground # noqa -from photutils.utils import calc_total_error # noqa - -from scipy.interpolate import UnivariateSpline # noqa - -from . import api # noqa -from .config import load_config # noqa -from .plots import plt, plot_image # noqa -from .version import get_version # noqa -from .load_image import load_image # noqa -from .run_imagematch import run_imagematch # noqa -from .zeropoint import bootstrap_outlier, sigma_from_Chauvenet # noqa -from .wcs import force_reject_g2d, clean_with_rsq_and_get_fwhm, get_clean_references # noqa -from .coordinatematch import CoordinateMatch, WCS2 # noqa -from .epsfbuilder import EPSFBuilder # noqa +from photutils import CircularAperture, CircularAnnulus, aperture_photometry # noqa: E402 +from photutils.psf import EPSFFitter, BasicPSFPhotometry, DAOGroup, extract_stars # noqa: E402 +from photutils import Background2D, SExtractorBackground, MedianBackground # noqa: E402 +from photutils.utils import calc_total_error # noqa: E402 + +from scipy.interpolate import UnivariateSpline # noqa: E402 + +from . import api # noqa: E402 +from .config import load_config # noqa: E402 +from .plots import plt, plot_image # noqa: E402 +from .version import get_version # noqa: E402 +from .load_image import load_image # noqa: E402 +from .run_imagematch import run_imagematch # noqa: E402 +from .zeropoint import bootstrap_outlier, sigma_from_Chauvenet # noqa: E402 +from .wcs import force_reject_g2d, clean_with_rsq_and_get_fwhm, get_clean_references # noqa: E402 +from .coordinatematch import CoordinateMatch, WCS2 # noqa: E402 +from .epsfbuilder import EPSFBuilder # noqa: E402 __version__ = get_version(pep440=False) @@ -54,7 +52,7 @@ def photometry(fileid, output_folder=None, attempt_imagematch=True, keep_diff_fixed=False, - timeoutpar=10): + timeoutpar='None'): """ Run photometry. @@ -271,8 +269,10 @@ def photometry(fileid, maximum_angle_distance=0.002, ) + # Set timeout par to infinity unless specified. + if timeoutpar=='None': timeoutpar=float('inf') try: - i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=float('inf')))) + i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=timeoutpar)) except TimeoutError: logger.warning('TimeoutError: No new WCS solution found') except StopIteration: diff --git a/run_photometry.py b/run_photometry.py index 85a8667..3beed68 100644 --- a/run_photometry.py +++ b/run_photometry.py @@ -16,7 +16,7 @@ # -------------------------------------------------------------------------------------------------- def process_fileid(fid, output_folder_root=None, attempt_imagematch=True, autoupload=False, keep_diff_fixed=False, - timeoutpar=10): + timeoutpar='None'): logger = logging.getLogger('flows') logging.captureWarnings(True) logger_warn = logging.getLogger('py.warnings') @@ -98,7 +98,7 @@ def process_fileid(fid, output_folder_root=None, attempt_imagematch=True, autoup help="Fix SN position during PSF photometry of difference image. \ Useful when difference image is noisy.", action='store_true') - group.add_argument('--timeoutpar', type=int, default=10, help='Timeout in Seconds for WCS') + group.add_argument('--timeoutpar', type=int, default='None', help="Timeout in Seconds for WCS") args = parser.parse_args() # Ensure that all input has been given: From 045e6f5414b79dfe87aa7cb93edd70666bea1618 Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 21 Apr 2021 15:34:07 +0200 Subject: [PATCH 40/43] Fix missing ) --- flows/photometry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flows/photometry.py b/flows/photometry.py index 93a203e..4a5b172 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -5,6 +5,7 @@ .. codeauthor:: Rasmus Handberg .. codeauthor:: Emir Karamehmetoglu +.. codeauthor:: Simon Holmbo """ import os import numpy as np @@ -272,7 +273,7 @@ def photometry(fileid, # Set timeout par to infinity unless specified. if timeoutpar=='None': timeoutpar=float('inf') try: - i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=timeoutpar)) + i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=timeoutpar))) except TimeoutError: logger.warning('TimeoutError: No new WCS solution found') except StopIteration: From 5beb8dca79f0f2f28c891778fa8b45b9c57d85f4 Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 21 Apr 2021 15:35:46 +0200 Subject: [PATCH 41/43] flake8 changes --- flows/photometry.py | 2 +- tests/test_photometry.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flows/photometry.py b/flows/photometry.py index 4a5b172..798175d 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -271,7 +271,7 @@ def photometry(fileid, ) # Set timeout par to infinity unless specified. - if timeoutpar=='None': timeoutpar=float('inf') + if timeoutpar == 'None': timeoutpar=float('inf') try: i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=timeoutpar))) except TimeoutError: diff --git a/tests/test_photometry.py b/tests/test_photometry.py index 38bc9a1..15b379f 100644 --- a/tests/test_photometry.py +++ b/tests/test_photometry.py @@ -8,7 +8,7 @@ import pytest import conftest # noqa: F401 -from flows import photometry +from flows import photometry # noqa: F401 #-------------------------------------------------------------------------------------------------- def test_import_photometry(): From 11d2f8bd2b57f8490b5681c594ab5360bc216c74 Mon Sep 17 00:00:00 2001 From: Rasmus Handberg Date: Wed, 21 Apr 2021 15:38:09 +0200 Subject: [PATCH 42/43] Changed astropy requirement to 4.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 08f90dd..f9f51d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ mplcursors == 0.3 seaborn pandas requests -astropy >= 4.1 +astropy == 4.1 photutils >= 1.0.2 PyYAML psycopg2-binary From 95462388c5317220046f31b41ede6b196db48796 Mon Sep 17 00:00:00 2001 From: Emir Date: Wed, 21 Apr 2021 15:46:22 +0200 Subject: [PATCH 43/43] missing operator flake8 --- flows/photometry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flows/photometry.py b/flows/photometry.py index 798175d..719f27f 100644 --- a/flows/photometry.py +++ b/flows/photometry.py @@ -271,7 +271,7 @@ def photometry(fileid, ) # Set timeout par to infinity unless specified. - if timeoutpar == 'None': timeoutpar=float('inf') + if timeoutpar == 'None': timeoutpar = float('inf') try: i_xy, i_rd = map(np.array, zip(*cm(5, 1.5, timeout=timeoutpar))) except TimeoutError: