From 28bcdf800c61b1bf95b7f2770d6e453778731aa2 Mon Sep 17 00:00:00 2001 From: Emir Date: Tue, 8 Sep 2020 11:21:49 +0200 Subject: [PATCH 01/12] Add interactive LC plotter plotlc.py --- plotlc.py | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 plotlc.py diff --git a/plotlc.py b/plotlc.py new file mode 100644 index 0000000..9d476e1 --- /dev/null +++ b/plotlc.py @@ -0,0 +1,122 @@ +#! /usr/bin/env python3 +import os +import glob +import numpy as np +import pandas as pd +import matplotlib +matplotlib.use('Qt5Agg') +import matplotlib.pyplot as plt +from astropy.table import Table,QTable +import astropy.units as u +from flows import api, load_config +import warnings +import argparse +import mplcursors +matplotlib.use('Qt5Agg') +import seaborn as sns + + +if __name__ == '__main__': + #Parser: + parser = argparse.ArgumentParser(description='Plot photometry for target') + parser.add_argument('--targetid','-t', type=int, required=True, help='Target id: -t ') + parser.add_argument('--fileid','-i', nargs='*', type=int, default=None, + help='[Optional] Specific file ids within target separed by spaces: -i ') + parser.add_argument('--filters','-f',type=str, default=None, + help='[Optional] List of comma delimited filters: -f u,b,g,r \n if not provided will use all') + parser.add_argument('--offset','-jd', type=float, default=2458800.0) + args = parser.parse_args() + + config=load_config() + + #To use when only plotting some filters + if args.filters is not None: + usefilts=args.filters.split(',') + + #To use when only plotting some fileids + #Parse input fileids: + if args.fileid is not None: + # Plot the specified fileid: + fileids = args.fileid + else: + fileids = [] + if len(fileids) > 1: + #not implemented yet + pass + + + TargetID= args.targetid + datafiles=api.get_datafiles(TargetID,filt='all') + + #Change to directory, raise if it does not exist + workdir_root=config.get('photometry', 'output', fallback='.') + snname=api.get_datafile(datafiles[0])['target_name'] + sndir=os.path.join(workdir_root,snname) + try: + os.chdir(sndir) + except: + print('No such directory as',sndir) + raise + + #Get list of photometry files + phot_files=glob.glob('*/*.ecsv') + + #Load all data into astropytable + Phot=Table(names=['jd','mag','mag_err','filter','tel','sub','fileid'],dtype=[None,None,None,'S64','S64',bool,'S64']) + + for file in phot_files: + fileid=file.split('/')[0] + df=api.get_datafile(fileid) #get fileid info + jd,filt,tel=df['obstime']+2400000.5,df['photfilter'],df['site'] + + #get phot of diff image + AT=Table.read(file) + AT.add_index('starid') + + try: + if -1 in AT['starid']: + mag,mag_err=AT.loc[-1]['mag'],AT.loc[-1]['mag_error'] + sub=True + elif 0 in AT['starid']: + print('No subtraction found for:',file,'in filter',filt) + mag,mag_err=AT.loc[0]['mag'],AT.loc[0]['mag_error'] + sub=False + else: + print('No object phot found, skipping: \n',file) + continue + Phot.add_row((jd,mag,mag_err,filt,tel,sub,fileid)) + except: + print('file:',file) + raise + + #TODO: Use filters argument here + filters=['gp','rp','B','V','ip'] + Phots=dict(zip(filters,[Phot[Phot['filter']==f] for f in filters])) + + #Plot + sns.set(style='ticks') + fig,ax=plt.subplots(figsize=(6.4,4),dpi=130) + fig.subplots_adjust(top=0.95,left=0.1,bottom=0.1,right=0.97) + + + shifts=dict(zip(filters,0.0*np.arange(len(filters)))) + cps=sns.color_palette() + colors=dict(zip(filters,(cps[2],cps[3],cps[0],cps[-1],cps[1]))) + + offset=args.offset + for filt in filters: + ax.errorbar(Phots[filt]['jd']-offset,Phots[filt]['mag']+shifts[filt],Phots[filt]['mag_err'], + marker='s',linestyle='None',label=filt,color=colors[filt]) + + ax.invert_yaxis() + ax.legend() + ax.set_xlabel('JD-'+str(offset),fontsize=16) + ax.set_ylabel('App. Mag',fontsize=16) + ax.set_title(snname) + + mplcursors.cursor(ax).connect( + "add", lambda sel: sel.annotation.set_text(Phots[str(sel.artist.get_label())]['fileid'][sel.target.index])) + + plt.show(block=True) + + \ No newline at end of file From d08bd68817abab7641bdef8371d879ef2a3e5ba7 Mon Sep 17 00:00:00 2001 From: Emir Date: Tue, 8 Sep 2020 11:44:41 +0200 Subject: [PATCH 02/12] add helper script for combining template_sub LCOGT header with original that has bad pixel mask --- tools/fixlcohead.py | 49 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tools/fixlcohead.py diff --git a/tools/fixlcohead.py b/tools/fixlcohead.py new file mode 100644 index 0000000..64f0625 --- /dev/null +++ b/tools/fixlcohead.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +import astropy.io.fits as fits +import numpy as np +from astropy.wcs import WCS +import argparse +parser = argparse.ArgumentParser(prog='FixTemplateSubHeaderforLCOGT', + description='Attach header and extra dimensions from second \ + file to the first, while checking to make sure WCS is identical.\ + If not, uses WCS offirst file.') + +parser.add_argument("diff_file", help="Specify the diff file here") +parser.add_argument("orig_file", help="Specify the original file here") +parser.add_argument("--overwrite", '-o', type=bool, default=False, + help='Overwrite difffile. Default=False outputs new file [diff_file_basename]_hfix.fits') + +args=parser.parse_args() + +hdul_diff= fits.open(args.diff_file) +wcs_diff=WCS(hdul_diff[0].header) + +hdul_orig= fits.open(args.orig_file) +wcs_orig=WCS(hdul_orig[0].header) + +if wcs_diff.low_level_wcs==wcs_diff.low_level_wcs: + new_hdul = fits.HDUList() + + new_hdul.append(hdul_diff[0]) + new_hdul.append(hdul_orig[1]) + new_hdul.append(hdul_orig[2]) + if args.overwrite: + print('Warning: Overwriting diff file!') + new_hdul.writeto(args.diff_file,overwrite=True) + else: + print('WCS is matched writing to [dill_file]_hfix.fits') + new_hdul.writeto(args.diff_file.split('.')[:-1][0]+'_hfix.fits',overwrite=True) +else: + print('WCS does not match, there may be problems in reducing the file \ + if extension 2 & 3 needs to have correct WCS. \ + Writing to file anyway..') + new_hdul = fits.HDUList() + + new_hdul.append(hdul_diff[0]) + new_hdul.append(hdul_orig[1]) + new_hdul.append(hdul_orig[2]) + if args.overwrite: + print('Warning: Overwriting diff file!') + new_hdul.writeto(args.diff_file,overwrite=True) + else: + new_hdul.writeto(args.diff_file.split('.')[:-1][0]+'_hfix.fits',overwrite=True) From 1399004fd5beb2071853650b958a3b42882b16d1 Mon Sep 17 00:00:00 2001 From: Emir Date: Tue, 8 Sep 2020 13:37:24 +0200 Subject: [PATCH 03/12] resole some Flake8 stuff --- plotlc.py | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/plotlc.py b/plotlc.py index 9d476e1..728bd11 100644 --- a/plotlc.py +++ b/plotlc.py @@ -2,37 +2,36 @@ import os import glob import numpy as np -import pandas as pd +import warnings +#Filter matplotlib depreciation warnings +warnings.filterwarnings("ignore", module='matplotlib') import matplotlib -matplotlib.use('Qt5Agg') import matplotlib.pyplot as plt -from astropy.table import Table,QTable -import astropy.units as u +from astropy.table import Table from flows import api, load_config -import warnings import argparse import mplcursors -matplotlib.use('Qt5Agg') import seaborn as sns +matplotlib.use('Qt5Agg') if __name__ == '__main__': #Parser: parser = argparse.ArgumentParser(description='Plot photometry for target') parser.add_argument('--targetid','-t', type=int, required=True, help='Target id: -t ') - parser.add_argument('--fileid','-i', nargs='*', type=int, default=None, + parser.add_argument('--fileid','-i', nargs='*', type=int, default=None, help='[Optional] Specific file ids within target separed by spaces: -i ') parser.add_argument('--filters','-f',type=str, default=None, help='[Optional] List of comma delimited filters: -f u,b,g,r \n if not provided will use all') parser.add_argument('--offset','-jd', type=float, default=2458800.0) args = parser.parse_args() - + config=load_config() - + #To use when only plotting some filters if args.filters is not None: usefilts=args.filters.split(',') - + #To use when only plotting some fileids #Parse input fileids: if args.fileid is not None: @@ -43,11 +42,10 @@ if len(fileids) > 1: #not implemented yet pass - - + TargetID= args.targetid datafiles=api.get_datafiles(TargetID,filt='all') - + #Change to directory, raise if it does not exist workdir_root=config.get('photometry', 'output', fallback='.') snname=api.get_datafile(datafiles[0])['target_name'] @@ -57,17 +55,17 @@ except: print('No such directory as',sndir) raise - + #Get list of photometry files phot_files=glob.glob('*/*.ecsv') - + #Load all data into astropytable Phot=Table(names=['jd','mag','mag_err','filter','tel','sub','fileid'],dtype=[None,None,None,'S64','S64',bool,'S64']) for file in phot_files: fileid=file.split('/')[0] df=api.get_datafile(fileid) #get fileid info - jd,filt,tel=df['obstime']+2400000.5,df['photfilter'],df['site'] + jd,filt,tel=df['obstime']+2400000.5,df['photfilter'],df['site'] #get phot of diff image AT=Table.read(file) @@ -88,17 +86,16 @@ except: print('file:',file) raise - + #TODO: Use filters argument here filters=['gp','rp','B','V','ip'] Phots=dict(zip(filters,[Phot[Phot['filter']==f] for f in filters])) - + #Plot sns.set(style='ticks') fig,ax=plt.subplots(figsize=(6.4,4),dpi=130) fig.subplots_adjust(top=0.95,left=0.1,bottom=0.1,right=0.97) - shifts=dict(zip(filters,0.0*np.arange(len(filters)))) cps=sns.color_palette() colors=dict(zip(filters,(cps[2],cps[3],cps[0],cps[-1],cps[1]))) @@ -118,5 +115,3 @@ "add", lambda sel: sel.annotation.set_text(Phots[str(sel.artist.get_label())]['fileid'][sel.target.index])) plt.show(block=True) - - \ No newline at end of file From b64d02cabed8fc1df127c24601a75d65118911f4 Mon Sep 17 00:00:00 2001 From: Emir Date: Tue, 8 Sep 2020 13:44:46 +0200 Subject: [PATCH 04/12] fix more style issues --- plotlc.py | 61 +++++++++++++++++++++++++++---------------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/plotlc.py b/plotlc.py index 728bd11..01e734a 100644 --- a/plotlc.py +++ b/plotlc.py @@ -26,11 +26,11 @@ parser.add_argument('--offset','-jd', type=float, default=2458800.0) args = parser.parse_args() - config=load_config() + config = load_config() #To use when only plotting some filters if args.filters is not None: - usefilts=args.filters.split(',') + usefilts = args.filters.split(',') #To use when only plotting some fileids #Parse input fileids: @@ -43,13 +43,13 @@ #not implemented yet pass - TargetID= args.targetid - datafiles=api.get_datafiles(TargetID,filt='all') + TargetID = args.targetid + datafiles = api.get_datafiles(TargetID,filt='all') #Change to directory, raise if it does not exist - workdir_root=config.get('photometry', 'output', fallback='.') - snname=api.get_datafile(datafiles[0])['target_name'] - sndir=os.path.join(workdir_root,snname) + workdir_root = config.get('photometry', 'output', fallback='.') + snname = api.get_datafile(datafiles[0])['target_name'] + sndir = os.path.join(workdir_root,snname) try: os.chdir(sndir) except: @@ -57,28 +57,28 @@ raise #Get list of photometry files - phot_files=glob.glob('*/*.ecsv') + phot_files = glob.glob('*/*.ecsv') #Load all data into astropytable - Phot=Table(names=['jd','mag','mag_err','filter','tel','sub','fileid'],dtype=[None,None,None,'S64','S64',bool,'S64']) + Phot = Table(names = ['jd','mag','mag_err','filter','tel','sub','fileid'],dtype=[None,None,None,'S64','S64',bool,'S64']) for file in phot_files: - fileid=file.split('/')[0] - df=api.get_datafile(fileid) #get fileid info - jd,filt,tel=df['obstime']+2400000.5,df['photfilter'],df['site'] + fileid = file.split('/')[0] + df = api.get_datafile(fileid) # get fileid info + jd, filt, tel = df['obstime'] + 2400000.5,df['photfilter'],df['site'] #get phot of diff image - AT=Table.read(file) + AT = Table.read(file) AT.add_index('starid') try: if -1 in AT['starid']: - mag,mag_err=AT.loc[-1]['mag'],AT.loc[-1]['mag_error'] + mag,mag_err = AT.loc[-1]['mag'],AT.loc[-1]['mag_error'] sub=True elif 0 in AT['starid']: print('No subtraction found for:',file,'in filter',filt) - mag,mag_err=AT.loc[0]['mag'],AT.loc[0]['mag_error'] - sub=False + mag,mag_err = AT.loc[0]['mag'],AT.loc[0]['mag_error'] + sub = False else: print('No object phot found, skipping: \n',file) continue @@ -88,30 +88,29 @@ raise #TODO: Use filters argument here - filters=['gp','rp','B','V','ip'] - Phots=dict(zip(filters,[Phot[Phot['filter']==f] for f in filters])) + filters = ['gp','rp','B','V','ip'] + Phots = dict(zip(filters,[Phot[Phot['filter'] == f] for f in filters])) #Plot - sns.set(style='ticks') - fig,ax=plt.subplots(figsize=(6.4,4),dpi=130) - fig.subplots_adjust(top=0.95,left=0.1,bottom=0.1,right=0.97) + sns.set(style = 'ticks') + fig,ax = plt.subplots(figsize = (6.4,4),dpi = 130) + fig.subplots_adjust(top = 0.95,left = 0.1,bottom = 0.1,right = 0.97) - shifts=dict(zip(filters,0.0*np.arange(len(filters)))) - cps=sns.color_palette() - colors=dict(zip(filters,(cps[2],cps[3],cps[0],cps[-1],cps[1]))) + shifts = dict(zip(filters,0.0 * np.arange(len(filters)))) + cps = sns.color_palette() + colors = dict(zip(filters,(cps[2],cps[3],cps[0],cps[-1],cps[1]))) - offset=args.offset + offset = args.offset for filt in filters: - ax.errorbar(Phots[filt]['jd']-offset,Phots[filt]['mag']+shifts[filt],Phots[filt]['mag_err'], - marker='s',linestyle='None',label=filt,color=colors[filt]) + ax.errorbar(Phots[filt]['jd'] - offset,Phots[filt]['mag'] + shifts[filt],Phots[filt]['mag_err'], + marker = 's',linestyle = 'None',label = filt,color = colors[filt]) ax.invert_yaxis() ax.legend() - ax.set_xlabel('JD-'+str(offset),fontsize=16) - ax.set_ylabel('App. Mag',fontsize=16) + ax.set_xlabel('JD-' + str(offset),fontsize = 16) + ax.set_ylabel('App. Mag',fontsize = 16) ax.set_title(snname) mplcursors.cursor(ax).connect( "add", lambda sel: sel.annotation.set_text(Phots[str(sel.artist.get_label())]['fileid'][sel.target.index])) - - plt.show(block=True) + plt.show(block = True) From 7163460e48b6d12df7ddd94e9b77cb978cffb9df Mon Sep 17 00:00:00 2001 From: Emir Date: Tue, 8 Sep 2020 13:46:07 +0200 Subject: [PATCH 05/12] name change --- run_interactive_LC_plot.py | 116 +++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 run_interactive_LC_plot.py diff --git a/run_interactive_LC_plot.py b/run_interactive_LC_plot.py new file mode 100644 index 0000000..01e734a --- /dev/null +++ b/run_interactive_LC_plot.py @@ -0,0 +1,116 @@ +#! /usr/bin/env python3 +import os +import glob +import numpy as np +import warnings +#Filter matplotlib depreciation warnings +warnings.filterwarnings("ignore", module='matplotlib') +import matplotlib +import matplotlib.pyplot as plt +from astropy.table import Table +from flows import api, load_config +import argparse +import mplcursors +import seaborn as sns +matplotlib.use('Qt5Agg') + + +if __name__ == '__main__': + #Parser: + parser = argparse.ArgumentParser(description='Plot photometry for target') + parser.add_argument('--targetid','-t', type=int, required=True, help='Target id: -t ') + parser.add_argument('--fileid','-i', nargs='*', type=int, default=None, + help='[Optional] Specific file ids within target separed by spaces: -i ') + parser.add_argument('--filters','-f',type=str, default=None, + help='[Optional] List of comma delimited filters: -f u,b,g,r \n if not provided will use all') + parser.add_argument('--offset','-jd', type=float, default=2458800.0) + args = parser.parse_args() + + config = load_config() + + #To use when only plotting some filters + if args.filters is not None: + usefilts = args.filters.split(',') + + #To use when only plotting some fileids + #Parse input fileids: + if args.fileid is not None: + # Plot the specified fileid: + fileids = args.fileid + else: + fileids = [] + if len(fileids) > 1: + #not implemented yet + pass + + TargetID = args.targetid + datafiles = api.get_datafiles(TargetID,filt='all') + + #Change to directory, raise if it does not exist + workdir_root = config.get('photometry', 'output', fallback='.') + snname = api.get_datafile(datafiles[0])['target_name'] + sndir = os.path.join(workdir_root,snname) + try: + os.chdir(sndir) + except: + print('No such directory as',sndir) + raise + + #Get list of photometry files + phot_files = glob.glob('*/*.ecsv') + + #Load all data into astropytable + Phot = Table(names = ['jd','mag','mag_err','filter','tel','sub','fileid'],dtype=[None,None,None,'S64','S64',bool,'S64']) + + for file in phot_files: + fileid = file.split('/')[0] + df = api.get_datafile(fileid) # get fileid info + jd, filt, tel = df['obstime'] + 2400000.5,df['photfilter'],df['site'] + + #get phot of diff image + AT = Table.read(file) + AT.add_index('starid') + + try: + if -1 in AT['starid']: + mag,mag_err = AT.loc[-1]['mag'],AT.loc[-1]['mag_error'] + sub=True + elif 0 in AT['starid']: + print('No subtraction found for:',file,'in filter',filt) + mag,mag_err = AT.loc[0]['mag'],AT.loc[0]['mag_error'] + sub = False + else: + print('No object phot found, skipping: \n',file) + continue + Phot.add_row((jd,mag,mag_err,filt,tel,sub,fileid)) + except: + print('file:',file) + raise + + #TODO: Use filters argument here + filters = ['gp','rp','B','V','ip'] + Phots = dict(zip(filters,[Phot[Phot['filter'] == f] for f in filters])) + + #Plot + sns.set(style = 'ticks') + fig,ax = plt.subplots(figsize = (6.4,4),dpi = 130) + fig.subplots_adjust(top = 0.95,left = 0.1,bottom = 0.1,right = 0.97) + + shifts = dict(zip(filters,0.0 * np.arange(len(filters)))) + cps = sns.color_palette() + colors = dict(zip(filters,(cps[2],cps[3],cps[0],cps[-1],cps[1]))) + + offset = args.offset + for filt in filters: + ax.errorbar(Phots[filt]['jd'] - offset,Phots[filt]['mag'] + shifts[filt],Phots[filt]['mag_err'], + marker = 's',linestyle = 'None',label = filt,color = colors[filt]) + + ax.invert_yaxis() + ax.legend() + ax.set_xlabel('JD-' + str(offset),fontsize = 16) + ax.set_ylabel('App. Mag',fontsize = 16) + ax.set_title(snname) + + mplcursors.cursor(ax).connect( + "add", lambda sel: sel.annotation.set_text(Phots[str(sel.artist.get_label())]['fileid'][sel.target.index])) + plt.show(block = True) From 23d79af7b7b4bd166f85720ee43f6889207aca4d Mon Sep 17 00:00:00 2001 From: Emir Date: Tue, 8 Sep 2020 13:51:06 +0200 Subject: [PATCH 06/12] fix most style issues, change name --- plotlc.py | 116 ------------------------------------- run_interactive_LC_plot.py | 16 ++--- 2 files changed, 8 insertions(+), 124 deletions(-) delete mode 100644 plotlc.py diff --git a/plotlc.py b/plotlc.py deleted file mode 100644 index 01e734a..0000000 --- a/plotlc.py +++ /dev/null @@ -1,116 +0,0 @@ -#! /usr/bin/env python3 -import os -import glob -import numpy as np -import warnings -#Filter matplotlib depreciation warnings -warnings.filterwarnings("ignore", module='matplotlib') -import matplotlib -import matplotlib.pyplot as plt -from astropy.table import Table -from flows import api, load_config -import argparse -import mplcursors -import seaborn as sns -matplotlib.use('Qt5Agg') - - -if __name__ == '__main__': - #Parser: - parser = argparse.ArgumentParser(description='Plot photometry for target') - parser.add_argument('--targetid','-t', type=int, required=True, help='Target id: -t ') - parser.add_argument('--fileid','-i', nargs='*', type=int, default=None, - help='[Optional] Specific file ids within target separed by spaces: -i ') - parser.add_argument('--filters','-f',type=str, default=None, - help='[Optional] List of comma delimited filters: -f u,b,g,r \n if not provided will use all') - parser.add_argument('--offset','-jd', type=float, default=2458800.0) - args = parser.parse_args() - - config = load_config() - - #To use when only plotting some filters - if args.filters is not None: - usefilts = args.filters.split(',') - - #To use when only plotting some fileids - #Parse input fileids: - if args.fileid is not None: - # Plot the specified fileid: - fileids = args.fileid - else: - fileids = [] - if len(fileids) > 1: - #not implemented yet - pass - - TargetID = args.targetid - datafiles = api.get_datafiles(TargetID,filt='all') - - #Change to directory, raise if it does not exist - workdir_root = config.get('photometry', 'output', fallback='.') - snname = api.get_datafile(datafiles[0])['target_name'] - sndir = os.path.join(workdir_root,snname) - try: - os.chdir(sndir) - except: - print('No such directory as',sndir) - raise - - #Get list of photometry files - phot_files = glob.glob('*/*.ecsv') - - #Load all data into astropytable - Phot = Table(names = ['jd','mag','mag_err','filter','tel','sub','fileid'],dtype=[None,None,None,'S64','S64',bool,'S64']) - - for file in phot_files: - fileid = file.split('/')[0] - df = api.get_datafile(fileid) # get fileid info - jd, filt, tel = df['obstime'] + 2400000.5,df['photfilter'],df['site'] - - #get phot of diff image - AT = Table.read(file) - AT.add_index('starid') - - try: - if -1 in AT['starid']: - mag,mag_err = AT.loc[-1]['mag'],AT.loc[-1]['mag_error'] - sub=True - elif 0 in AT['starid']: - print('No subtraction found for:',file,'in filter',filt) - mag,mag_err = AT.loc[0]['mag'],AT.loc[0]['mag_error'] - sub = False - else: - print('No object phot found, skipping: \n',file) - continue - Phot.add_row((jd,mag,mag_err,filt,tel,sub,fileid)) - except: - print('file:',file) - raise - - #TODO: Use filters argument here - filters = ['gp','rp','B','V','ip'] - Phots = dict(zip(filters,[Phot[Phot['filter'] == f] for f in filters])) - - #Plot - sns.set(style = 'ticks') - fig,ax = plt.subplots(figsize = (6.4,4),dpi = 130) - fig.subplots_adjust(top = 0.95,left = 0.1,bottom = 0.1,right = 0.97) - - shifts = dict(zip(filters,0.0 * np.arange(len(filters)))) - cps = sns.color_palette() - colors = dict(zip(filters,(cps[2],cps[3],cps[0],cps[-1],cps[1]))) - - offset = args.offset - for filt in filters: - ax.errorbar(Phots[filt]['jd'] - offset,Phots[filt]['mag'] + shifts[filt],Phots[filt]['mag_err'], - marker = 's',linestyle = 'None',label = filt,color = colors[filt]) - - ax.invert_yaxis() - ax.legend() - ax.set_xlabel('JD-' + str(offset),fontsize = 16) - ax.set_ylabel('App. Mag',fontsize = 16) - ax.set_title(snname) - - mplcursors.cursor(ax).connect( - "add", lambda sel: sel.annotation.set_text(Phots[str(sel.artist.get_label())]['fileid'][sel.target.index])) - plt.show(block = True) diff --git a/run_interactive_LC_plot.py b/run_interactive_LC_plot.py index 01e734a..558951e 100644 --- a/run_interactive_LC_plot.py +++ b/run_interactive_LC_plot.py @@ -60,7 +60,7 @@ phot_files = glob.glob('*/*.ecsv') #Load all data into astropytable - Phot = Table(names = ['jd','mag','mag_err','filter','tel','sub','fileid'],dtype=[None,None,None,'S64','S64',bool,'S64']) + Phot = Table(names=['jd','mag','mag_err','filter','tel','sub','fileid'],dtype=[None,None,None,'S64','S64',bool,'S64']) for file in phot_files: fileid = file.split('/')[0] @@ -74,7 +74,7 @@ try: if -1 in AT['starid']: mag,mag_err = AT.loc[-1]['mag'],AT.loc[-1]['mag_error'] - sub=True + sub = True elif 0 in AT['starid']: print('No subtraction found for:',file,'in filter',filt) mag,mag_err = AT.loc[0]['mag'],AT.loc[0]['mag_error'] @@ -92,9 +92,9 @@ Phots = dict(zip(filters,[Phot[Phot['filter'] == f] for f in filters])) #Plot - sns.set(style = 'ticks') - fig,ax = plt.subplots(figsize = (6.4,4),dpi = 130) - fig.subplots_adjust(top = 0.95,left = 0.1,bottom = 0.1,right = 0.97) + sns.set(style='ticks') + fig,ax = plt.subplots(figsize=(6.4,4),dpi=130) + fig.subplots_adjust(top=0.95,left=0.1,bottom=0.1,right=0.97) shifts = dict(zip(filters,0.0 * np.arange(len(filters)))) cps = sns.color_palette() @@ -103,12 +103,12 @@ offset = args.offset for filt in filters: ax.errorbar(Phots[filt]['jd'] - offset,Phots[filt]['mag'] + shifts[filt],Phots[filt]['mag_err'], - marker = 's',linestyle = 'None',label = filt,color = colors[filt]) + marker='s',linestyle='None',label=filt,color=colors[filt]) ax.invert_yaxis() ax.legend() - ax.set_xlabel('JD-' + str(offset),fontsize = 16) - ax.set_ylabel('App. Mag',fontsize = 16) + ax.set_xlabel('JD-' + str(offset),fontsize=16) + ax.set_ylabel('App. Mag',fontsize=16) ax.set_title(snname) mplcursors.cursor(ax).connect( From 6d039cf6806ed09c6808af5b1de69b488cddce3f Mon Sep 17 00:00:00 2001 From: Emir Date: Tue, 8 Sep 2020 13:55:50 +0200 Subject: [PATCH 07/12] final style fixes --- run_interactive_LC_plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_interactive_LC_plot.py b/run_interactive_LC_plot.py index 558951e..bd0b77d 100644 --- a/run_interactive_LC_plot.py +++ b/run_interactive_LC_plot.py @@ -113,4 +113,4 @@ mplcursors.cursor(ax).connect( "add", lambda sel: sel.annotation.set_text(Phots[str(sel.artist.get_label())]['fileid'][sel.target.index])) - plt.show(block = True) + plt.show(block=True) From 869aa61006621717e95e86de6ffbfb72c739fcc5 Mon Sep 17 00:00:00 2001 From: Rasmus Handberg Date: Tue, 8 Sep 2020 15:41:01 +0200 Subject: [PATCH 08/12] Delete fixlcohead Should be included later in a different PR --- tools/fixlcohead.py | 49 --------------------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 tools/fixlcohead.py diff --git a/tools/fixlcohead.py b/tools/fixlcohead.py deleted file mode 100644 index 64f0625..0000000 --- a/tools/fixlcohead.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 -import astropy.io.fits as fits -import numpy as np -from astropy.wcs import WCS -import argparse -parser = argparse.ArgumentParser(prog='FixTemplateSubHeaderforLCOGT', - description='Attach header and extra dimensions from second \ - file to the first, while checking to make sure WCS is identical.\ - If not, uses WCS offirst file.') - -parser.add_argument("diff_file", help="Specify the diff file here") -parser.add_argument("orig_file", help="Specify the original file here") -parser.add_argument("--overwrite", '-o', type=bool, default=False, - help='Overwrite difffile. Default=False outputs new file [diff_file_basename]_hfix.fits') - -args=parser.parse_args() - -hdul_diff= fits.open(args.diff_file) -wcs_diff=WCS(hdul_diff[0].header) - -hdul_orig= fits.open(args.orig_file) -wcs_orig=WCS(hdul_orig[0].header) - -if wcs_diff.low_level_wcs==wcs_diff.low_level_wcs: - new_hdul = fits.HDUList() - - new_hdul.append(hdul_diff[0]) - new_hdul.append(hdul_orig[1]) - new_hdul.append(hdul_orig[2]) - if args.overwrite: - print('Warning: Overwriting diff file!') - new_hdul.writeto(args.diff_file,overwrite=True) - else: - print('WCS is matched writing to [dill_file]_hfix.fits') - new_hdul.writeto(args.diff_file.split('.')[:-1][0]+'_hfix.fits',overwrite=True) -else: - print('WCS does not match, there may be problems in reducing the file \ - if extension 2 & 3 needs to have correct WCS. \ - Writing to file anyway..') - new_hdul = fits.HDUList() - - new_hdul.append(hdul_diff[0]) - new_hdul.append(hdul_orig[1]) - new_hdul.append(hdul_orig[2]) - if args.overwrite: - print('Warning: Overwriting diff file!') - new_hdul.writeto(args.diff_file,overwrite=True) - else: - new_hdul.writeto(args.diff_file.split('.')[:-1][0]+'_hfix.fits',overwrite=True) From 81d69a6aa0a93f7127bac7057ecf9ac6f0ae0865 Mon Sep 17 00:00:00 2001 From: Rasmus Handberg Date: Tue, 8 Sep 2020 17:20:35 +0200 Subject: [PATCH 09/12] Various changes to make this more efficient --- requirements.txt | 4 +- run_interactive_LC_plot.py | 239 ++++++++++++++++++++----------------- 2 files changed, 135 insertions(+), 108 deletions(-) diff --git a/requirements.txt b/requirements.txt index eeabd7b..2f57a11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ numpy >= 1.16 scipy Bottleneck == 1.3.2 -matplotlib == 3.0.3 +matplotlib == 3.3.1 +mplcursors == 0.3 +seaborn requests astropy < 4.0 photutils < 0.7 diff --git a/run_interactive_LC_plot.py b/run_interactive_LC_plot.py index bd0b77d..4fee68a 100644 --- a/run_interactive_LC_plot.py +++ b/run_interactive_LC_plot.py @@ -1,116 +1,141 @@ -#! /usr/bin/env python3 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" + +.. codeauthor:: Emir K +.. codeauthor:: Rasmus Handberg +""" + +import argparse import os import glob import numpy as np -import warnings -#Filter matplotlib depreciation warnings -warnings.filterwarnings("ignore", module='matplotlib') -import matplotlib -import matplotlib.pyplot as plt from astropy.table import Table +from flows.plots import plt, plots_interactive from flows import api, load_config -import argparse import mplcursors import seaborn as sns -matplotlib.use('Qt5Agg') - +#-------------------------------------------------------------------------------------------------- +def main(): + # All available filters: + # TODO: Get this from the Flows API + all_filters = ['B', 'V', 'rp', 'gp', 'rp', 'ip', 'zp', 'H', 'J', 'K'] + + # Parser: + parser = argparse.ArgumentParser(description='Plot photometry for target') + parser.add_argument('--target', '-t', type=str, required=True, help="""Target identifier: + Can be either the SN name (e.g. 2019yvr) or the Flows target ID.""") + parser.add_argument('--fileid', '-i', type=int, nargs='*', default=None, + help='Specific file ids within target separated by spaces: -i ') + parser.add_argument('--filters', '-f', type=str, nargs='*', default=None, choices=all_filters, + help='List of space delimited filters. If not provided will use all') + parser.add_argument('--offset', '-jd', type=float, default=2458800.0) + args = parser.parse_args() + + # To use when only plotting some filters + usefilts = args.filters + if usefilts is not None: + usefilts = set(args.filters) + + # To use when only plotting some fileids + # Parse input fileids: + if args.fileid is not None: + # Plot the specified fileid: + fileids = args.fileid + else: + fileids = [] + if len(fileids) > 1: + raise NotImplementedError("This has not been implemented yet") + + offset = args.offset + + # Get the name of the target: + snname = args.target + if snname.isdigit(): + datafiles = api.get_datafiles(int(snname), filt='all') + snname = api.get_datafile(datafiles[0])['target_name'] + + # Change to directory, raise if it does not exist + config = load_config() + workdir_root = config.get('photometry', 'output', fallback='.') + sndir = os.path.join(workdir_root, snname) + if not os.path.isdir(sndir): + print('No such directory as',sndir) + return + + # Get list of photometry files + phot_files = glob.iglob(os.path.join(sndir, '*', 'photometry.ecsv')) + + # Load all data into astropy table + tablerows = [] + for file in phot_files: + # Load photometry file into Table: + AT = Table.read(file, format='ascii.ecsv') + + # Pull out meta-data: + fileid = AT.meta['fileid'] + filt = AT.meta['photfilter'] + jd = AT.meta['obstime-bmjd'] + 2400000.5 + + # get phot of diff image + AT.add_index('starid') + if -1 in AT['starid']: + mag, mag_err = AT.loc[-1]['mag'], AT.loc[-1]['mag_error'] + sub = True + elif 0 in AT['starid']: + print('No subtraction found for:',file,'in filter',filt) + mag,mag_err = AT.loc[0]['mag'],AT.loc[0]['mag_error'] + sub = False + else: + print('No object phot found, skipping: \n',file) + continue + + tablerows.append((jd, mag, mag_err, filt, sub, fileid)) + + phot = Table( + rows=tablerows, + names=['jd','mag','mag_err','filter','sub','fileid'], + dtype=['float64','float64','float64','S64','bool','int64']) + + # Create list of filters to plot: + filters = list(np.unique(phot['filter'])) + if usefilts: + filters = set(filters).intersection(usefilts) + + # Split photometry table + shifts = dict(zip(filters, np.zeros(len(filters)))) + + # Create the plot: + plots_interactive() + sns.set(style='ticks') + fig, ax = plt.subplots(figsize=(6.4,4), dpi=130) + fig.subplots_adjust(top=0.95, left=0.1, bottom=0.1, right=0.97) + + cps = sns.color_palette() + colors = dict(zip(filters,(cps[2],cps[3],cps[0],cps[-1],cps[1]))) + + for filt in filters: + lc = phot[phot['filter'] == filt] + ax.errorbar(lc['jd'] - offset, lc['mag'] + shifts[filt], lc['mag_err'], + marker='s', linestyle='None', label=filt, color=colors[filt]) + + ax.invert_yaxis() + ax.legend() + ax.set_xlabel('JD - ' + str(offset), fontsize=16) + ax.set_ylabel('App. Mag', fontsize=16) + ax.set_title(snname) + + # Make the points interactive: + def annotate(sel): + lc = phot[phot['filter'] == str(sel.artist.get_label())] + point = lc[sel.target.index] + point = dict(zip(point.colnames, point)) # Convert table row to dict + return sel.annotation.set_text("Fileid: {fileid:d}\nJD: {jd:.3f}\nMag: {mag:.2f}$\pm${mag_err:.2f}".format(**point)) + + mplcursors.cursor(ax).connect("add", annotate) + plt.show(block=True) + +#-------------------------------------------------------------------------------------------------- if __name__ == '__main__': - #Parser: - parser = argparse.ArgumentParser(description='Plot photometry for target') - parser.add_argument('--targetid','-t', type=int, required=True, help='Target id: -t ') - parser.add_argument('--fileid','-i', nargs='*', type=int, default=None, - help='[Optional] Specific file ids within target separed by spaces: -i ') - parser.add_argument('--filters','-f',type=str, default=None, - help='[Optional] List of comma delimited filters: -f u,b,g,r \n if not provided will use all') - parser.add_argument('--offset','-jd', type=float, default=2458800.0) - args = parser.parse_args() - - config = load_config() - - #To use when only plotting some filters - if args.filters is not None: - usefilts = args.filters.split(',') - - #To use when only plotting some fileids - #Parse input fileids: - if args.fileid is not None: - # Plot the specified fileid: - fileids = args.fileid - else: - fileids = [] - if len(fileids) > 1: - #not implemented yet - pass - - TargetID = args.targetid - datafiles = api.get_datafiles(TargetID,filt='all') - - #Change to directory, raise if it does not exist - workdir_root = config.get('photometry', 'output', fallback='.') - snname = api.get_datafile(datafiles[0])['target_name'] - sndir = os.path.join(workdir_root,snname) - try: - os.chdir(sndir) - except: - print('No such directory as',sndir) - raise - - #Get list of photometry files - phot_files = glob.glob('*/*.ecsv') - - #Load all data into astropytable - Phot = Table(names=['jd','mag','mag_err','filter','tel','sub','fileid'],dtype=[None,None,None,'S64','S64',bool,'S64']) - - for file in phot_files: - fileid = file.split('/')[0] - df = api.get_datafile(fileid) # get fileid info - jd, filt, tel = df['obstime'] + 2400000.5,df['photfilter'],df['site'] - - #get phot of diff image - AT = Table.read(file) - AT.add_index('starid') - - try: - if -1 in AT['starid']: - mag,mag_err = AT.loc[-1]['mag'],AT.loc[-1]['mag_error'] - sub = True - elif 0 in AT['starid']: - print('No subtraction found for:',file,'in filter',filt) - mag,mag_err = AT.loc[0]['mag'],AT.loc[0]['mag_error'] - sub = False - else: - print('No object phot found, skipping: \n',file) - continue - Phot.add_row((jd,mag,mag_err,filt,tel,sub,fileid)) - except: - print('file:',file) - raise - - #TODO: Use filters argument here - filters = ['gp','rp','B','V','ip'] - Phots = dict(zip(filters,[Phot[Phot['filter'] == f] for f in filters])) - - #Plot - sns.set(style='ticks') - fig,ax = plt.subplots(figsize=(6.4,4),dpi=130) - fig.subplots_adjust(top=0.95,left=0.1,bottom=0.1,right=0.97) - - shifts = dict(zip(filters,0.0 * np.arange(len(filters)))) - cps = sns.color_palette() - colors = dict(zip(filters,(cps[2],cps[3],cps[0],cps[-1],cps[1]))) - - offset = args.offset - for filt in filters: - ax.errorbar(Phots[filt]['jd'] - offset,Phots[filt]['mag'] + shifts[filt],Phots[filt]['mag_err'], - marker='s',linestyle='None',label=filt,color=colors[filt]) - - ax.invert_yaxis() - ax.legend() - ax.set_xlabel('JD-' + str(offset),fontsize=16) - ax.set_ylabel('App. Mag',fontsize=16) - ax.set_title(snname) - - mplcursors.cursor(ax).connect( - "add", lambda sel: sel.annotation.set_text(Phots[str(sel.artist.get_label())]['fileid'][sel.target.index])) - plt.show(block=True) + main() From b1526a4a27f4a2ba6a4aa24f66bc805ac3b6caed Mon Sep 17 00:00:00 2001 From: Rasmus Handberg Date: Thu, 10 Sep 2020 14:15:35 +0200 Subject: [PATCH 10/12] Use list of photometric filters from API --- run_interactive_LC_plot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_interactive_LC_plot.py b/run_interactive_LC_plot.py index 4fee68a..06252f8 100644 --- a/run_interactive_LC_plot.py +++ b/run_interactive_LC_plot.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ +Plot photometry for target, loaded from local photometry working directory. .. codeauthor:: Emir K .. codeauthor:: Rasmus Handberg @@ -19,8 +20,7 @@ #-------------------------------------------------------------------------------------------------- def main(): # All available filters: - # TODO: Get this from the Flows API - all_filters = ['B', 'V', 'rp', 'gp', 'rp', 'ip', 'zp', 'H', 'J', 'K'] + all_filters = list(api.get_filters().keys()) # Parser: parser = argparse.ArgumentParser(description='Plot photometry for target') From 7dbdb074ebc391aaf7eac9c6097e65a8ce1c2c7e Mon Sep 17 00:00:00 2001 From: Rasmus Handberg Date: Thu, 10 Sep 2020 14:16:35 +0200 Subject: [PATCH 11/12] Gave command an easier name --- run_interactive_LC_plot.py => run_plotlc.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename run_interactive_LC_plot.py => run_plotlc.py (100%) diff --git a/run_interactive_LC_plot.py b/run_plotlc.py similarity index 100% rename from run_interactive_LC_plot.py rename to run_plotlc.py From e074dfc5155a56cb8bd1b61c77ccc31f1fbcff92 Mon Sep 17 00:00:00 2001 From: Rasmus Handberg Date: Thu, 10 Sep 2020 14:20:24 +0200 Subject: [PATCH 12/12] Fix invalid character --- run_plotlc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_plotlc.py b/run_plotlc.py index 06252f8..7e5d010 100644 --- a/run_plotlc.py +++ b/run_plotlc.py @@ -131,7 +131,7 @@ def annotate(sel): lc = phot[phot['filter'] == str(sel.artist.get_label())] point = lc[sel.target.index] point = dict(zip(point.colnames, point)) # Convert table row to dict - return sel.annotation.set_text("Fileid: {fileid:d}\nJD: {jd:.3f}\nMag: {mag:.2f}$\pm${mag_err:.2f}".format(**point)) + return sel.annotation.set_text("Fileid: {fileid:d}\nJD: {jd:.3f}\nMag: {mag:.2f}$\\pm${mag_err:.2f}".format(**point)) mplcursors.cursor(ax).connect("add", annotate) plt.show(block=True)