From 165d19d4159d5c5ac0d5c71c3a8701acbf7b9e42 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 12 Aug 2021 17:47:20 +0100 Subject: [PATCH 01/12] Upload fits files for future use --- uploadToArchive.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/uploadToArchive.py b/uploadToArchive.py index 6a9e8da..312f1f7 100644 --- a/uploadToArchive.py +++ b/uploadToArchive.py @@ -11,6 +11,7 @@ import boto3 import os import sys +import glob def uploadOneFile(arch_dir, dir_file, s3, targf, file_ext, log=None): @@ -24,6 +25,8 @@ def uploadOneFile(arch_dir, dir_file, s3, targf, file_ext, log=None): ctyp='text/plain' if file_ext=='.jpg': ctyp = 'image/jpeg' + if file_ext=='.fits': + ctyp = 'image/fits' elif file_ext=='.png': ctyp = 'image/png' elif file_ext=='.bmp': @@ -81,6 +84,19 @@ def uploadToArchive(arch_dir, log=None): uploadOneFile(arch_dir, dir_file, s3, targf, file_ext, log) elif dir_file == 'mask.bmp' or dir_file == 'flat.bmp' or dir_file == '.config': uploadOneFile(arch_dir, dir_file, s3, targf, file_ext, log) + + # upload two FITs files from around midnight if possible + # to be used for platepar creation if needed + fitslist = glob.glob1(arch_dir, 'FF*.fits') + if len(fitslist) > 0: + tss = [] + for ffname in fitslist: + tim=ffname[19:25] + tss.append([tim, ffname]) + tss.sort() + uploadOneFile(arch_dir, tss[0][1], s3, targf, '.fits', log) + uploadOneFile(arch_dir, tss[-1][1], s3, targf, '.fits', log) + return From 8fc45cc052caa9fe8b590f3d4427a38746718832 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 12 Aug 2021 22:20:39 +0100 Subject: [PATCH 02/12] Add fireball uploader support --- uploadToArchive.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/uploadToArchive.py b/uploadToArchive.py index 312f1f7..81dcb8b 100644 --- a/uploadToArchive.py +++ b/uploadToArchive.py @@ -12,6 +12,8 @@ import os import sys import glob +import datetime +import RMS.ConfigReader as cr def uploadOneFile(arch_dir, dir_file, s3, targf, file_ext, log=None): @@ -100,6 +102,42 @@ def uploadToArchive(arch_dir, log=None): return +def fireballUpload(ffname, log=None): + cfgname = '.config' + config = cr.parse(cfgname) + rmsdatadir = config.data_dir + + myloc = os.path.split(os.path.abspath(__file__))[0] + filename = os.path.join(myloc, 'archive.key') + with open(filename, 'r') as fin: + key = fin.readline().split('=')[1].strip() + secr = fin.readline().split('=')[1].strip() + reg = fin.readline().split('=')[1].strip() + targf = fin.readline().split('=')[1].strip() + if targf[0] == '"': + targf = targf[1:len(targf)-1] + conn = boto3.Session(aws_access_key_id=key, aws_secret_access_key=secr) + s3 = conn.resource('s3', region_name=reg) + + dtstamp = ffname[10:25] + ts = datetime.datetime.strptime(dtstamp,'%Y%m%d_%H%M%S') + if ts.hour < 12: + ts = ts + datetime.timedelta(days=-1) + dirpat = ts.strftime('%Y%m%d') + basarc = os.path.join(rmsdatadir, 'CapturedFiles') + cap_dirs = [name for name in os.listdir(basarc) if (os.path.isdir(os.path.join(basarc, name)) and dirpat in name)] + if len(cap_dirs > 0): + fldr = cap_dirs[0] +# arch_dir = os.path.join(basarc, fldr) + cap_dir = os.path.join(rmsdatadir, 'CapturedFiles', fldr) + fbname = 'FR' + ffname[3:-5] + '.bin' + uploadOneFile(cap_dir, fbname, s3, targf, '.fits', log) + uploadOneFile(cap_dir, ffname, s3, targf, '.fits', log) + else: + print('unable to find source folder') + return + + def manualUpload(targ_dir): """ Manually send the target folder to ukmon archive. To invoke this function open a Terminal window and type From b30b9cc75c81e694b46f96c23610673e1fef3c5c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 12 Aug 2021 22:23:44 +0100 Subject: [PATCH 03/12] bugfix --- uploadToArchive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uploadToArchive.py b/uploadToArchive.py index 81dcb8b..a9e2a97 100644 --- a/uploadToArchive.py +++ b/uploadToArchive.py @@ -126,7 +126,7 @@ def fireballUpload(ffname, log=None): dirpat = ts.strftime('%Y%m%d') basarc = os.path.join(rmsdatadir, 'CapturedFiles') cap_dirs = [name for name in os.listdir(basarc) if (os.path.isdir(os.path.join(basarc, name)) and dirpat in name)] - if len(cap_dirs > 0): + if len(cap_dirs) > 0: fldr = cap_dirs[0] # arch_dir = os.path.join(basarc, fldr) cap_dir = os.path.join(rmsdatadir, 'CapturedFiles', fldr) From 2e9a3c27c5d0f2559089883864fa33e7094ef909 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 12 Aug 2021 22:25:35 +0100 Subject: [PATCH 04/12] bugfix --- uploadToArchive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uploadToArchive.py b/uploadToArchive.py index a9e2a97..309b082 100644 --- a/uploadToArchive.py +++ b/uploadToArchive.py @@ -130,7 +130,7 @@ def fireballUpload(ffname, log=None): fldr = cap_dirs[0] # arch_dir = os.path.join(basarc, fldr) cap_dir = os.path.join(rmsdatadir, 'CapturedFiles', fldr) - fbname = 'FR' + ffname[3:-5] + '.bin' + fbname = 'FR' + ffname[2:-5] + '.bin' uploadOneFile(cap_dir, fbname, s3, targf, '.fits', log) uploadOneFile(cap_dir, ffname, s3, targf, '.fits', log) else: From 7ebb5a9da71c95b02ca23b02618a55ad0e58d5b6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 12 Aug 2021 22:34:46 +0100 Subject: [PATCH 05/12] fireball uploader script --- uploadFireball.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 uploadFireball.sh diff --git a/uploadFireball.sh b/uploadFireball.sh new file mode 100644 index 0000000..3984b7d --- /dev/null +++ b/uploadFireball.sh @@ -0,0 +1,15 @@ +#!/bin/bash +source ~/vRMS/bin/activate +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +source $here/ukmon.ini + +cd ~/source/RMS +export PYTHONPATH=/home/pi/source/ukmon-pitools +python << EOD +import uploadToArchive as ua +import sys +if len(sys.argv) < 2: + print('usage: ./uploadFireball.sh FF_filename.fits) +else: + ua.fireballUpload(sys.argv[1]) +EOD \ No newline at end of file From 4d80850e9046438c7a5fd64599e29b36ab310578 Mon Sep 17 00:00:00 2001 From: RMS Date: Thu, 12 Aug 2021 21:35:40 +0000 Subject: [PATCH 06/12] making executable --- uploadFireball.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 uploadFireball.sh diff --git a/uploadFireball.sh b/uploadFireball.sh old mode 100644 new mode 100755 From 5359ca5ce9e35879b842dd47a586622ae6e369d6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 12 Aug 2021 22:36:40 +0100 Subject: [PATCH 07/12] bugfix --- uploadFireball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uploadFireball.sh b/uploadFireball.sh index 3984b7d..60a9741 100755 --- a/uploadFireball.sh +++ b/uploadFireball.sh @@ -9,7 +9,7 @@ python << EOD import uploadToArchive as ua import sys if len(sys.argv) < 2: - print('usage: ./uploadFireball.sh FF_filename.fits) + print('usage: ./uploadFireball.sh FF_filename.fits') else: ua.fireballUpload(sys.argv[1]) EOD \ No newline at end of file From 0219b7c4474b01c694465ecb7beb431f30c15cc4 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 12 Aug 2021 22:37:43 +0100 Subject: [PATCH 08/12] bugfix --- uploadFireball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uploadFireball.sh b/uploadFireball.sh index 60a9741..2db0e9e 100755 --- a/uploadFireball.sh +++ b/uploadFireball.sh @@ -8,7 +8,7 @@ export PYTHONPATH=/home/pi/source/ukmon-pitools python << EOD import uploadToArchive as ua import sys -if len(sys.argv) < 2: +if len(sys.argv) < 1: print('usage: ./uploadFireball.sh FF_filename.fits') else: ua.fireballUpload(sys.argv[1]) From 4bebe3e7a9ff2a327c30da41b9a24fbb63a1f206 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 12 Aug 2021 22:39:27 +0100 Subject: [PATCH 09/12] bugfix --- uploadFireball.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uploadFireball.sh b/uploadFireball.sh index 2db0e9e..6b91b70 100755 --- a/uploadFireball.sh +++ b/uploadFireball.sh @@ -5,10 +5,10 @@ source $here/ukmon.ini cd ~/source/RMS export PYTHONPATH=/home/pi/source/ukmon-pitools -python << EOD +python $1 << EOD import uploadToArchive as ua import sys -if len(sys.argv) < 1: +if len(sys.argv) < 2: print('usage: ./uploadFireball.sh FF_filename.fits') else: ua.fireballUpload(sys.argv[1]) From e0ec16d25e2d6e4838e0d77f1510beecd0fac144 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 12 Aug 2021 22:42:19 +0100 Subject: [PATCH 10/12] bugfix --- uploadFireball.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/uploadFireball.sh b/uploadFireball.sh index 6b91b70..a0f6418 100755 --- a/uploadFireball.sh +++ b/uploadFireball.sh @@ -5,11 +5,11 @@ source $here/ukmon.ini cd ~/source/RMS export PYTHONPATH=/home/pi/source/ukmon-pitools -python $1 << EOD +if [ "$1" -eq "" ] ; then + echo usage ./uploadFireball.sh FF_name.fits + exit 0 +fi +python << EOD import uploadToArchive as ua -import sys -if len(sys.argv) < 2: - print('usage: ./uploadFireball.sh FF_filename.fits') -else: - ua.fireballUpload(sys.argv[1]) +ua.fireballUpload( ${1} ) EOD \ No newline at end of file From dc4501967f7313d8aa561eb91be256bd5375622c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 12 Aug 2021 22:43:35 +0100 Subject: [PATCH 11/12] bugfix --- uploadFireball.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uploadFireball.sh b/uploadFireball.sh index a0f6418..8da40a5 100755 --- a/uploadFireball.sh +++ b/uploadFireball.sh @@ -5,11 +5,11 @@ source $here/ukmon.ini cd ~/source/RMS export PYTHONPATH=/home/pi/source/ukmon-pitools -if [ "$1" -eq "" ] ; then +if [ "$1" == "" ] ; then echo usage ./uploadFireball.sh FF_name.fits exit 0 fi python << EOD import uploadToArchive as ua -ua.fireballUpload( ${1} ) +ua.fireballUpload("${1}") EOD \ No newline at end of file From 19ef7c310372d16bcf5df8a0544df3d2ce08bd9b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 13 Aug 2021 19:16:30 +0100 Subject: [PATCH 12/12] include pngs --- uploadToArchive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uploadToArchive.py b/uploadToArchive.py index 309b082..b4bd298 100644 --- a/uploadToArchive.py +++ b/uploadToArchive.py @@ -82,7 +82,7 @@ def uploadToArchive(arch_dir, log=None): uploadOneFile(arch_dir, dir_file, s3, targf, file_ext, log) if (file_ext == '.mp4') and ('FF_' in file_name): uploadOneFile(arch_dir, dir_file, s3, targf, file_ext, log) - elif file_ext in ('.jpg', '.kml', '.cal', '.json', '.csv') and ('DETECTED' not in file_name) and ('CAPTURED' not in file_name): + elif file_ext in ('.png', '.jpg', '.kml', '.cal', '.json', '.csv') and ('DETECTED' not in file_name) and ('CAPTURED' not in file_name): uploadOneFile(arch_dir, dir_file, s3, targf, file_ext, log) elif dir_file == 'mask.bmp' or dir_file == 'flat.bmp' or dir_file == '.config': uploadOneFile(arch_dir, dir_file, s3, targf, file_ext, log)