Small bugfixes

.github/workflows/python_test.yml

@@ -1,8 +1,23 @@
 name: Test ukmon-pitools
 on: [push]
 jobs:
+  container_test:
+    name: run containerised test
+    runs-on: ubuntu-latest
+    container:
+      image: docker.io/markmac99/rmstestcont:latest
+      env: 
+        MYENV: foo
+      ports:
+        - 22
+        - 443
+    steps:
+      - uses: actions/checkout@v3
+      - run: chmod +x ./tests/run_ub_tests.sh
+      - run: ./tests/run_ub_tests.sh
+
   test:
-    name: run python tests
+    name: run pi4 tests
     runs-on: [testpi4]
     strategy:
       matrix:
@@ -24,20 +39,19 @@ jobs:
         run: |
           # stop the build if there are Python syntax errors or undefined names
           source ~/vRMS/bin/activate
-          ruff --output-format=github --select=E9,F63,F7,F82 --ignore=E501 --target-version=py37 .
+          export PATH=/home/pi/.local/bin:$PATH
+          ruff check --output-format=github --select=E9,F63,F7,F82 --ignore=E501 --target-version=py37 .
           # default set of ruff rules with GitHub Annotations
-          ruff --output-format=github --ignore=E501 --target-version=py37 .
+          ruff check --output-format=github --ignore=E501 --target-version=py37 .
       - name: Test with pytest
         run: |
           source ~/vRMS/bin/activate
+          export PATH=/home/pi/.local/bin:$PATH
           pip install -r ~/source/RMS/requirements.txt
           export PYTHONPATH=$PYTHONPATH:$(pwd)
-          cp ~/source/testing/ukmon.ini .
-          cp ~/source/testing/live.key .
-          touch ./domp4s
+          cp /home/pi/RMS_data/test/* .
           source ukmon.ini
           pytest -v . --cov=. --cov-report=term-missing --cov-config=.coveragerc_lnx
-          rm ./live.key ./ukmon.ini ./domp4s
 #  Run-refreshtools-on-Ubuntu:
 #    name: Run refreshTools on Ubuntu
 #    runs-on: ubuntu-20.04

tests/run_ub_tests.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Copyright (c) Mark McIntyre
+
+# tests designed to run on a Pi 
+here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+if [ ! -f /keys/ukmon.ini ] ; then
+    echo no test config, aborting
+    exit
+fi 
+cp /keys/ukmon.ini . 
+cp /keys/live.key .
+pushd /root/source/RMS 
+git stash && git pull && git stash apply 
+cp /keys/.config /root/source/RMS/
+popd
+touch ./domp4s
+source ukmon.ini
+pip install -r ./requirements.txt
+pip install --upgrade ruff pytest xmltodict pytest-cov 
+export PYTHONPATH=$PYTHONPATH:/root/source/RMS:${here}/..
+cd /root/source/RMS
+ls -ltra .config
+pwd
+pytest -v $here/ --cov=$here/../ --cov-report=term-missing --cov-config=$here/../.coveragerc_lnx 

tests/test_uploadToArchive.py

@@ -2,7 +2,7 @@
 
 import boto3
 import os
-from uploadToArchive import readKeyFile, uploadOneFile, manualUpload, readIniFile
+from uploadToArchive import readKeyFile, uploadOneFile, manualUpload, readIniFile, checkMags
 from ukmonInstaller import createDefaultIni
 
 basedir = os.path.realpath(os.path.dirname(__file__))
@@ -11,6 +11,18 @@
     os.makedirs(tmpdir)
 
 
+def test_checkMags():
+    inifvals = readIniFile(os.path.join(basedir,'..','ukmon.ini'))
+    maglim = 6
+    if 'MAGLIM' in inifvals:
+        maglim = float(inifvals['MAGLIM'])
+    arch_dir = os.path.join(basedir, 'ukmarch','sampledata', 'UK0006_20220914_185543_087124')
+    daydir = 'UK0006_20220914_185543_087124'
+    validffs = checkMags(arch_dir, 'FTPdetectinfo_{}.txt'.format(daydir), maglim)
+    print(validffs)
+    assert 'FF_UK0006_20220914_200343_841_0101120.fits' in validffs
+
+
 def test_readIniFile():
     inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'))
     assert inifs['LOCATION']=='testpi4'

ukmonInstaller.py

@@ -6,7 +6,12 @@
 from subprocess import call
 
 import time
-import paramiko
+import warnings
+from cryptography.utils import CryptographyDeprecationWarning
+with warnings.catch_warnings():
+    warnings.filterwarnings('ignore', category=CryptographyDeprecationWarning)
+    import paramiko
+
 import json
 import tempfile
 import RMS.ConfigReader as cr
@@ -45,6 +50,8 @@ def createDefaultIni(homedir, helperip='3.11.55.160', location='NOTCONFIGURED',
         outf.write("export UKMONHELPER={}\n".format(helperip))
         outf.write("export UKMONKEY={}\n".format(keyfile))
         outf.write("export RMSCFG={}\n".format(rmscfg))
+        outf.write('export DOMP4s=1\n')
+        outf.write('export MINMAG=1\n')
     return True
 
 
@@ -76,9 +83,27 @@ def validateIni(homedir, newhelperip=None):
         createDefaultIni(homedir, newhelperip, location, keyfile, rmscfg)
     if helperip == oldip:
         updateHelperIp(homedir, newhelperip)
+    updateMp4andMag(inifname, homedir)
     return True
 
 
+def updateMp4andMag(inif, homedir):
+    """
+    Move the mp4 flag into the ini file and add the minmag flag if missing
+    """
+    domp4s = 0
+    if open(inif, 'r').read()[-1] != '\n':
+        open(inif, 'a').write('\n')
+    if os.path.isfile(os.path.join(homedir, 'domp4s')):
+        domp4s = 1
+        os.remove(os.path.join(homedir, 'domp4s'))
+    if 'DOMP4S' not in open(inif).read():
+        open(inif,'a').write('export DOMP4S={}\n'.format(domp4s))
+    if 'MAGLIM' not in open(inif).read():
+        open(inif,'a').write('export MAGLIM=1\n')
+    return
+
+
 def updateHelperIp(homedir, helperip):
     """
     Update the ukmon.ini file with a new IP address if neeeded. 
@@ -91,6 +116,7 @@ def updateHelperIp(homedir, helperip):
                 outf.write("export UKMONHELPER={}\n".format(helperip))
             else:
                 outf.write('{}'.format(li))
+    return
 
 
 def updateLocation(homedir, newloc):

ukmonPostProc.py

@@ -58,8 +58,14 @@ def rmsExternal(cap_dir, arch_dir, config):
         bff2i.batchFFtoImage(arch_dir, 'jpg')
 
     myloc = os.path.split(os.path.abspath(__file__))[0]
+    inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'))
     log.info('app home is {}'.format(myloc))
-    if os.path.isfile(os.path.join(myloc, 'domp4s')):
+    domp4s = 0
+    if 'DOMP4S' in inifvals:
+        domp4s = int(inifvals['DOMP4S'])
+    elif os.path.isfile(os.path.join(myloc, 'domp4s')):
+        domp4s = 1
+    if domp4s == 1: 
         # generate MP4s of detections
         log.info('generating MP4s')
         ftpdate=''
@@ -68,7 +74,13 @@ def rmsExternal(cap_dir, arch_dir, config):
         else:
             ftpdate=os.path.split(arch_dir)[1]
         ftpfile_name="FTPdetectinfo_"+ftpdate+'.txt'
-        gmp4.generateMP4s(arch_dir, ftpfile_name)
+        try:
+            maglim = 1
+            if 'MAGLIM' in inifvals:
+                maglim = float(inifvals['MAGLIM'])
+            gmp4.generateMP4s(arch_dir, ftpfile_name, min_mag=maglim)
+        except Exception:
+            gmp4.generateMP4s(arch_dir, ftpfile_name)
     else:
         log.info('mp4 creation not enabled')
 

uploadToArchive.py

@@ -17,8 +17,13 @@
 import glob
 import logging
 from time import sleep
-import paramiko
+import warnings
+from cryptography.utils import CryptographyDeprecationWarning
+with warnings.catch_warnings():
+    warnings.filterwarnings('ignore', category=CryptographyDeprecationWarning)
+    import paramiko
 import tempfile
+from RMS.Formats.FTPdetectinfo import readFTPdetectinfo
 
 log = logging.getLogger("logger")
 
@@ -254,6 +259,30 @@ def uploadOneFileUKMon(arch_dir, dir_file, s3, targf, file_ext, keys):
     return ret
 
 
+def checkMags(dir_path, ftpfile_name, min_mag):
+    log.info('checking for events brighter than mag {}'.format(min_mag))
+    ff_names = []
+    try:
+        meteor_list = readFTPdetectinfo(dir_path, ftpfile_name)  
+    except Exception:
+        log.info('FTPdetect file not present so unable to filter by magnitude')
+        return ff_names
+    for meteor in meteor_list:
+        ff_name, _, meteor_no, n_segments, _, _, _, _, _, _, _, \
+            meteor_meas = meteor
+        # checks on mag and shower        
+        best_mag = 999
+        if min_mag is not None:
+            for meas in meteor_meas:
+                best_mag = min(best_mag, meas[9])
+            if best_mag > min_mag:
+                log.info('rejecting {} as {} too dim'.format(ff_name, best_mag))
+                continue
+            else:
+                ff_names.append(ff_name.replace('.fits', '.jpg'))
+    return ff_names
+
+
 def uploadToArchive(arch_dir, sciencefiles=False, keys=False):
     # Upload all relevant files from *arch_dir* to ukmon's S3 Archive
 
@@ -269,12 +298,17 @@ def uploadToArchive(arch_dir, sciencefiles=False, keys=False):
     conn = boto3.Session(aws_access_key_id=keys['AWS_ACCESS_KEY_ID'], aws_secret_access_key=keys['AWS_SECRET_ACCESS_KEY']) 
     s3 = conn.resource('s3', region_name=reg)
     targf = keys['S3FOLDER']
+    maglim = 10.0
+    if 'MAGLIM' in inifvals:
+        maglim = float(inifvals['MAGLIM'])
 
     # upload the files but make sure we do the platepars file before the FTP file
     # otherwise there's a risk the matching engine will miss it
     dir_contents = os.listdir(arch_dir)
     daydir = os.path.split(arch_dir)[1]
 
+    validffs = checkMags(arch_dir, 'FTPdetectinfo_{}.txt'.format(daydir), maglim)
+
     uploadlist = []
     if sciencefiles:
         # upload just the critical files
@@ -295,10 +329,11 @@ def uploadToArchive(arch_dir, sciencefiles=False, keys=False):
                 continue
             # mp4 must be uploaded before corresponding jpg
             elif (file_ext == '.jpg') and ('FF_' in file_name):
-                mp4f = dir_file.replace('.jpg', '.mp4')
-                if os.path.isfile(os.path.join(arch_dir, mp4f)):
-                    uploadlist.append({'dir_file':mp4f, 'file_ext': '.mp4', 'src_dir': arch_dir})
-                uploadlist.append({'dir_file':dir_file, 'file_ext': file_ext, 'src_dir': arch_dir})
+                if dir_file in validffs or validffs == []:
+                    mp4f = dir_file.replace('.jpg', '.mp4')
+                    if os.path.isfile(os.path.join(arch_dir, mp4f)):
+                        uploadlist.append({'dir_file':mp4f, 'file_ext': '.mp4', 'src_dir': arch_dir})
+                    uploadlist.append({'dir_file':dir_file, 'file_ext': file_ext, 'src_dir': arch_dir})
             elif (file_ext == '.jpg') and ('stack_' in file_name) and ('track' not in file_name):
                 uploadlist.append({'dir_file':dir_file, 'file_ext': file_ext, 'src_dir': arch_dir})
             elif (file_ext == '.jpg') and ('calib' in file_name):
@@ -379,12 +414,17 @@ def manualUpload(targ_dir, sciencefiles=False):
         return True
     else:
         arch_dir = os.path.expanduser(targ_dir)
+        if not os.path.isdir(arch_dir):
+            print('folder {} not found'.format(arch_dir))
+            return False
         return uploadToArchive(arch_dir, sciencefiles, keys=None)
 
 
 if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print('usage: python uploadToArchive.py ~/RMS_data/ArchivedFiles/dated_dir')
+        exit(0)
     targdir = sys.argv[1]
-    if len(sys.argv) > 2:
-        manualUpload(sys.argv[1], True)
-    else:
-        manualUpload(sys.argv[1])
+    manualUpload(targdir, True)
+    if targdir != 'test':
+        manualUpload(targdir)