From c9dbf1c178770de4d6263db4b582e2d4b3a7b4e2 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sun, 4 Feb 2024 16:46:08 +0000
Subject: [PATCH 01/29] test containerised testing

---
 .github/workflows/python_test.yml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml
index 67aeaeb..a0236a2 100644
--- a/.github/workflows/python_test.yml
+++ b/.github/workflows/python_test.yml
@@ -1,6 +1,24 @@
 name: Test ukmon-pitools
 on: [push]
 jobs:
+  container_test:
+    name: run containerised test
+    runs-on: ubuntu-latest
+    container:
+      image: docker.io/markmac99/ubi8:latest
+      env: 
+        MYENV: foo
+      ports:
+        - 22
+        - 443
+    steps:
+      - uses: actions/checkout@v3
+#      - uses: actions/setup-python@v4
+#        with:
+#          python-version: '3.8'
+      - run: chmod +x ./tests/run_tests.sh
+      - run: ./tests/run_tests.sh
+  
   test:
     name: run python tests
     runs-on: [testpi4]

From 2f5c1e4434111f7a3dffe85a0d75e954c4c9a972 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sun, 4 Feb 2024 17:43:30 +0000
Subject: [PATCH 02/29] update to container testing

---
 .github/workflows/python_test.yml |  6 +++---
 tests/run_ub_tests.sh             | 16 ++++++++++++++++
 2 files changed, 19 insertions(+), 3 deletions(-)
 create mode 100644 tests/run_ub_tests.sh

diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml
index a0236a2..ec93db8 100644
--- a/.github/workflows/python_test.yml
+++ b/.github/workflows/python_test.yml
@@ -5,7 +5,7 @@ jobs:
     name: run containerised test
     runs-on: ubuntu-latest
     container:
-      image: docker.io/markmac99/ubi8:latest
+      image: docker.io/markmac99/rmstestcont:latest
       env: 
         MYENV: foo
       ports:
@@ -16,8 +16,8 @@ jobs:
 #      - uses: actions/setup-python@v4
 #        with:
 #          python-version: '3.8'
-      - run: chmod +x ./tests/run_tests.sh
-      - run: ./tests/run_tests.sh
+      - run: chmod +x ./tests/run_ub_tests.sh
+      - run: ./tests/run_ub_tests.sh
   
   test:
     name: run python tests
diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh
new file mode 100644
index 0000000..903c144
--- /dev/null
+++ b/tests/run_ub_tests.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+# Copyright (c) Mark McIntyre
+
+# tests designed to run on a Pi 
+here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+export PYTHONPATH=$PYTHONPATH:${here}
+if [ ! -f /keys/ukmon.ini ] ; then
+    echo no test config, aborting
+    exit
+fi 
+cp /keys/.config /source/RMS/
+cp /keys/ukmon.ini . 
+cp /keys/live.key .
+touch ./domp4s
+source ukmon.ini
+pytest -v ./tests/ --cov=./ --cov-report=term-missing --cov-config=.coveragerc_lnx 

From e47ecc138f64bcc3b219533bf25a1c246b316b62 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sun, 4 Feb 2024 17:52:14 +0000
Subject: [PATCH 03/29] tweak tests

---
 tests/run_ub_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh
index 903c144..bafb596 100644
--- a/tests/run_ub_tests.sh
+++ b/tests/run_ub_tests.sh
@@ -8,7 +8,7 @@ if [ ! -f /keys/ukmon.ini ] ; then
     echo no test config, aborting
     exit
 fi 
-cp /keys/.config /source/RMS/
+cp /keys/.config /root/source/RMS/
 cp /keys/ukmon.ini . 
 cp /keys/live.key .
 touch ./domp4s

From 735e9cf0008930f718ccfc512126c7f48db10806 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sun, 4 Feb 2024 17:56:23 +0000
Subject: [PATCH 04/29] more small tweaks to testing

---
 tests/run_ub_tests.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh
index bafb596..65f51e9 100644
--- a/tests/run_ub_tests.sh
+++ b/tests/run_ub_tests.sh
@@ -13,4 +13,6 @@ cp /keys/ukmon.ini .
 cp /keys/live.key .
 touch ./domp4s
 source ukmon.ini
+pip install -r ../requirements.txt
+pip install --upgrade ruff pytest xmltodict pytest-cov 
 pytest -v ./tests/ --cov=./ --cov-report=term-missing --cov-config=.coveragerc_lnx 

From 060b1066dc1d2eb314c65e7c07bede2f85655a67 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sun, 4 Feb 2024 17:59:13 +0000
Subject: [PATCH 05/29] add pythonpath

---
 tests/run_ub_tests.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh
index 65f51e9..d2b6272 100644
--- a/tests/run_ub_tests.sh
+++ b/tests/run_ub_tests.sh
@@ -13,6 +13,7 @@ cp /keys/ukmon.ini .
 cp /keys/live.key .
 touch ./domp4s
 source ukmon.ini
-pip install -r ../requirements.txt
+pip install -r ./requirements.txt
 pip install --upgrade ruff pytest xmltodict pytest-cov 
+export PYTHONPATH=$PYTHONPATH:/root/source/RMS
 pytest -v ./tests/ --cov=./ --cov-report=term-missing --cov-config=.coveragerc_lnx 

From d65b3c022003359d4586548679076f646162ede7 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sun, 4 Feb 2024 18:01:33 +0000
Subject: [PATCH 06/29] yet more tweaking

---
 ukmonInstaller.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ukmonInstaller.py b/ukmonInstaller.py
index a5ece8f..575b109 100644
--- a/ukmonInstaller.py
+++ b/ukmonInstaller.py
@@ -6,7 +6,12 @@
 from subprocess import call
 
 import time
-import paramiko
+import warnings
+from cryptography.utils import CryptographyDeprecationWarning
+with warnings.catch_warnings():
+    warnings.filterwarnings('ignore', category=CryptographyDeprecationWarning)
+    import paramiko
+
 import json
 import tempfile
 import RMS.ConfigReader as cr

From 721fc0ce1bc06ea165d150f745ea99168361be6c Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sun, 4 Feb 2024 18:05:37 +0000
Subject: [PATCH 07/29] more work on testing

---
 tests/run_ub_tests.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh
index d2b6272..fa55bcf 100644
--- a/tests/run_ub_tests.sh
+++ b/tests/run_ub_tests.sh
@@ -3,7 +3,6 @@
 
 # tests designed to run on a Pi 
 here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
-export PYTHONPATH=$PYTHONPATH:${here}
 if [ ! -f /keys/ukmon.ini ] ; then
     echo no test config, aborting
     exit
@@ -15,5 +14,8 @@ touch ./domp4s
 source ukmon.ini
 pip install -r ./requirements.txt
 pip install --upgrade ruff pytest xmltodict pytest-cov 
-export PYTHONPATH=$PYTHONPATH:/root/source/RMS
-pytest -v ./tests/ --cov=./ --cov-report=term-missing --cov-config=.coveragerc_lnx 
+export PYTHONPATH=$PYTHONPATH:/root/source/RMS:${here}
+chdir /root/source/RMS
+ls -ltra .config
+pwd
+pytest -v $here/tests/ --cov=$here/ --cov-report=term-missing --cov-config=$here/.coveragerc_lnx 

From d508759fa6cde6dcff7304ce99cba84542635132 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sun, 4 Feb 2024 18:08:03 +0000
Subject: [PATCH 08/29] dumbass type

---
 tests/run_ub_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh
index fa55bcf..01f3f8f 100644
--- a/tests/run_ub_tests.sh
+++ b/tests/run_ub_tests.sh
@@ -15,7 +15,7 @@ source ukmon.ini
 pip install -r ./requirements.txt
 pip install --upgrade ruff pytest xmltodict pytest-cov 
 export PYTHONPATH=$PYTHONPATH:/root/source/RMS:${here}
-chdir /root/source/RMS
+cd /root/source/RMS
 ls -ltra .config
 pwd
 pytest -v $here/tests/ --cov=$here/ --cov-report=term-missing --cov-config=$here/.coveragerc_lnx 

From ce2e5b9c77dada4cc8148373d8c5ff84e3a574a0 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sun, 4 Feb 2024 18:11:42 +0000
Subject: [PATCH 09/29] more tweakery

---
 tests/run_ub_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh
index 01f3f8f..b71e8dc 100644
--- a/tests/run_ub_tests.sh
+++ b/tests/run_ub_tests.sh
@@ -18,4 +18,4 @@ export PYTHONPATH=$PYTHONPATH:/root/source/RMS:${here}
 cd /root/source/RMS
 ls -ltra .config
 pwd
-pytest -v $here/tests/ --cov=$here/ --cov-report=term-missing --cov-config=$here/.coveragerc_lnx 
+pytest -v $here/ --cov=$here/../ --cov-report=term-missing --cov-config=$here/../.coveragerc_lnx 

From f1e40ac769f88c19770a4bffdd4f94c6902e728e Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sun, 4 Feb 2024 18:13:43 +0000
Subject: [PATCH 10/29] might be there now

---
 tests/run_ub_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh
index b71e8dc..b7a0e28 100644
--- a/tests/run_ub_tests.sh
+++ b/tests/run_ub_tests.sh
@@ -14,7 +14,7 @@ touch ./domp4s
 source ukmon.ini
 pip install -r ./requirements.txt
 pip install --upgrade ruff pytest xmltodict pytest-cov 
-export PYTHONPATH=$PYTHONPATH:/root/source/RMS:${here}
+export PYTHONPATH=$PYTHONPATH:/root/source/RMS:${here}/..
 cd /root/source/RMS
 ls -ltra .config
 pwd

From 57bf148904c727040fc69dc53f9de1330881f397 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Mon, 5 Feb 2024 23:29:44 +0000
Subject: [PATCH 11/29] Updating tests

---
 .github/workflows/python_test.yml | 2 +-
 tests/run_ub_tests.sh             | 3 +++
 uploadToArchive.py                | 6 +++++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml
index ec93db8..cd9d66f 100644
--- a/.github/workflows/python_test.yml
+++ b/.github/workflows/python_test.yml
@@ -20,7 +20,7 @@ jobs:
       - run: ./tests/run_ub_tests.sh
   
   test:
-    name: run python tests
+    name: run pi4 tests
     runs-on: [testpi4]
     strategy:
       matrix:
diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh
index b7a0e28..bb07db7 100644
--- a/tests/run_ub_tests.sh
+++ b/tests/run_ub_tests.sh
@@ -10,6 +10,9 @@ fi
 cp /keys/.config /root/source/RMS/
 cp /keys/ukmon.ini . 
 cp /keys/live.key .
+pushd /root/source/RMS 
+git stash && git pull && git stash apply 
+popd
 touch ./domp4s
 source ukmon.ini
 pip install -r ./requirements.txt
diff --git a/uploadToArchive.py b/uploadToArchive.py
index c617ed5..0985f01 100644
--- a/uploadToArchive.py
+++ b/uploadToArchive.py
@@ -17,7 +17,11 @@
 import glob
 import logging
 from time import sleep
-import paramiko
+import warnings
+from cryptography.utils import CryptographyDeprecationWarning
+with warnings.catch_warnings():
+    warnings.filterwarnings('ignore', category=CryptographyDeprecationWarning)
+    import paramiko
 import tempfile
 
 log = logging.getLogger("logger")

From 4b17778598cdc568158dffbe14cffbed527ff681 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Fri, 23 Feb 2024 17:44:09 +0000
Subject: [PATCH 12/29] add filter by mag to mp4 and jpg creation/upload

---
 .github/workflows/python_test.yml |  3 ---
 tests/test_uploadToArchive.py     | 14 ++++++++++++-
 ukmonPostProc.py                  |  8 +++++--
 uploadToArchive.py                | 35 +++++++++++++++++++++++++++----
 4 files changed, 50 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml
index cd9d66f..cab5e01 100644
--- a/.github/workflows/python_test.yml
+++ b/.github/workflows/python_test.yml
@@ -13,9 +13,6 @@ jobs:
         - 443
     steps:
       - uses: actions/checkout@v3
-#      - uses: actions/setup-python@v4
-#        with:
-#          python-version: '3.8'
       - run: chmod +x ./tests/run_ub_tests.sh
       - run: ./tests/run_ub_tests.sh
   
diff --git a/tests/test_uploadToArchive.py b/tests/test_uploadToArchive.py
index 2e60661..b041146 100644
--- a/tests/test_uploadToArchive.py
+++ b/tests/test_uploadToArchive.py
@@ -2,7 +2,7 @@
 
 import boto3
 import os
-from uploadToArchive import readKeyFile, uploadOneFile, manualUpload, readIniFile
+from uploadToArchive import readKeyFile, uploadOneFile, manualUpload, readIniFile, checkMags
 from ukmonInstaller import createDefaultIni
 
 basedir = os.path.realpath(os.path.dirname(__file__))
@@ -11,6 +11,18 @@
     os.makedirs(tmpdir)
 
 
+def test_checkMags():
+    inifvals = readIniFile(os.path.join(basedir,'..','ukmon.ini'))
+    maglim = 6
+    if 'MAGLIM' in inifvals:
+        maglim = float(inifvals['MAGLIM'])
+    arch_dir = os.path.join(basedir, 'ukmarch','sampledata', 'UK0006_20220914_185543_087124')
+    daydir = 'UK0006_20220914_185543_087124'
+    validffs = checkMags(arch_dir, 'FTPdetectinfo_{}.txt'.format(daydir), maglim)
+    print(validffs)
+    assert validffs[0] == 'FF_UK0006_20220914_200343_841_0101120.fits'
+
+
 def test_readIniFile():
     inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'))
     assert inifs['LOCATION']=='testpi4'
diff --git a/ukmonPostProc.py b/ukmonPostProc.py
index 2939384..be78f4a 100644
--- a/ukmonPostProc.py
+++ b/ukmonPostProc.py
@@ -59,7 +59,7 @@ def rmsExternal(cap_dir, arch_dir, config):
 
     myloc = os.path.split(os.path.abspath(__file__))[0]
     log.info('app home is {}'.format(myloc))
-    if os.path.isfile(os.path.join(myloc, 'domp4s')):
+    if os.path.isfile(os.path.join(myloc, 'domp4s')) or os.getenv('DOMP4S', default='1')=='1':
         # generate MP4s of detections
         log.info('generating MP4s')
         ftpdate=''
@@ -68,7 +68,11 @@ def rmsExternal(cap_dir, arch_dir, config):
         else:
             ftpdate=os.path.split(arch_dir)[1]
         ftpfile_name="FTPdetectinfo_"+ftpdate+'.txt'
-        gmp4.generateMP4s(arch_dir, ftpfile_name)
+        try:
+            maglim = float(os.getenv('MAGLIM', default='0'))
+            gmp4.generateMP4s(arch_dir, ftpfile_name, min_mag=maglim)
+        except:
+            gmp4.generateMP4s(arch_dir, ftpfile_name)
     else:
         log.info('mp4 creation not enabled')
     
diff --git a/uploadToArchive.py b/uploadToArchive.py
index 0985f01..49843b0 100644
--- a/uploadToArchive.py
+++ b/uploadToArchive.py
@@ -23,6 +23,7 @@
     warnings.filterwarnings('ignore', category=CryptographyDeprecationWarning)
     import paramiko
 import tempfile
+from RMS.Formats.FTPdetectinfo import readFTPdetectinfo
 
 log = logging.getLogger("logger")
 
@@ -258,6 +259,26 @@ def uploadOneFileUKMon(arch_dir, dir_file, s3, targf, file_ext, keys):
     return ret
 
 
+def checkMags(dir_path, ftpfile_name, min_mag):
+    print('checking for events brighter than mag ', min_mag)
+    ff_names = []
+    meteor_list = readFTPdetectinfo(dir_path, ftpfile_name)  
+    for meteor in meteor_list:
+        ff_name, _, meteor_no, n_segments, _, _, _, _, _, _, _, \
+            meteor_meas = meteor
+        # checks on mag and shower        
+        best_mag = 999
+        if min_mag is not None:
+            for meas in meteor_meas:
+                best_mag = min(best_mag, meas[9])
+            if best_mag > min_mag:
+                print('rejecting {} as too dim'.format(ff_name))
+                continue
+            else:
+                ff_names.append(ff_name)
+    return ff_names
+
+
 def uploadToArchive(arch_dir, sciencefiles=False, keys=False):
     # Upload all relevant files from *arch_dir* to ukmon's S3 Archive
 
@@ -273,12 +294,17 @@ def uploadToArchive(arch_dir, sciencefiles=False, keys=False):
     conn = boto3.Session(aws_access_key_id=keys['AWS_ACCESS_KEY_ID'], aws_secret_access_key=keys['AWS_SECRET_ACCESS_KEY']) 
     s3 = conn.resource('s3', region_name=reg)
     targf = keys['S3FOLDER']
+    maglim = 0
+    if 'MAGLIM' in inifvals:
+        maglim = float(inifvals['MAGLIM'])
 
     # upload the files but make sure we do the platepars file before the FTP file
     # otherwise there's a risk the matching engine will miss it
     dir_contents = os.listdir(arch_dir)
     daydir = os.path.split(arch_dir)[1]
 
+    validffs = checkMags(arch_dir, 'FTPdetectinfo_{}.txt'.format(daydir), maglim)
+
     uploadlist = []
     if sciencefiles:
         # upload just the critical files
@@ -299,10 +325,11 @@ def uploadToArchive(arch_dir, sciencefiles=False, keys=False):
                 continue
             # mp4 must be uploaded before corresponding jpg
             elif (file_ext == '.jpg') and ('FF_' in file_name):
-                mp4f = dir_file.replace('.jpg', '.mp4')
-                if os.path.isfile(os.path.join(arch_dir, mp4f)):
-                    uploadlist.append({'dir_file':mp4f, 'file_ext': '.mp4', 'src_dir': arch_dir})
-                uploadlist.append({'dir_file':dir_file, 'file_ext': file_ext, 'src_dir': arch_dir})
+                if dir_file in validffs:
+                    mp4f = dir_file.replace('.jpg', '.mp4')
+                    if os.path.isfile(os.path.join(arch_dir, mp4f)):
+                        uploadlist.append({'dir_file':mp4f, 'file_ext': '.mp4', 'src_dir': arch_dir})
+                    uploadlist.append({'dir_file':dir_file, 'file_ext': file_ext, 'src_dir': arch_dir})
             elif (file_ext == '.jpg') and ('stack_' in file_name) and ('track' not in file_name):
                 uploadlist.append({'dir_file':dir_file, 'file_ext': file_ext, 'src_dir': arch_dir})
             elif (file_ext == '.jpg') and ('calib' in file_name):

From c38d2c6ec880f88294d9f63c880a6f535b592c1e Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Fri, 23 Feb 2024 17:55:15 +0000
Subject: [PATCH 13/29] fix test and exception

---
 tests/test_uploadToArchive.py | 2 +-
 ukmonPostProc.py              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_uploadToArchive.py b/tests/test_uploadToArchive.py
index b041146..5e11a73 100644
--- a/tests/test_uploadToArchive.py
+++ b/tests/test_uploadToArchive.py
@@ -20,7 +20,7 @@ def test_checkMags():
     daydir = 'UK0006_20220914_185543_087124'
     validffs = checkMags(arch_dir, 'FTPdetectinfo_{}.txt'.format(daydir), maglim)
     print(validffs)
-    assert validffs[0] == 'FF_UK0006_20220914_200343_841_0101120.fits'
+    assert 'FF_UK0006_20220914_200343_841_0101120.fits' in validffs
 
 
 def test_readIniFile():
diff --git a/ukmonPostProc.py b/ukmonPostProc.py
index be78f4a..af3e88d 100644
--- a/ukmonPostProc.py
+++ b/ukmonPostProc.py
@@ -71,7 +71,7 @@ def rmsExternal(cap_dir, arch_dir, config):
         try:
             maglim = float(os.getenv('MAGLIM', default='0'))
             gmp4.generateMP4s(arch_dir, ftpfile_name, min_mag=maglim)
-        except:
+        except Exception:
             gmp4.generateMP4s(arch_dir, ftpfile_name)
     else:
         log.info('mp4 creation not enabled')

From c6a7faff96b3cb3742245989f8353b09e5e5d1d7 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Fri, 23 Feb 2024 18:01:13 +0000
Subject: [PATCH 14/29] handle case where ftpdetect file missing

---
 uploadToArchive.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/uploadToArchive.py b/uploadToArchive.py
index 49843b0..d446262 100644
--- a/uploadToArchive.py
+++ b/uploadToArchive.py
@@ -262,7 +262,11 @@ def uploadOneFileUKMon(arch_dir, dir_file, s3, targf, file_ext, keys):
 def checkMags(dir_path, ftpfile_name, min_mag):
     print('checking for events brighter than mag ', min_mag)
     ff_names = []
-    meteor_list = readFTPdetectinfo(dir_path, ftpfile_name)  
+    try:
+        meteor_list = readFTPdetectinfo(dir_path, ftpfile_name)  
+    except Exception:
+        print('FTPdetect file not present so unable to filter by magnitude')
+        return ff_names
     for meteor in meteor_list:
         ff_name, _, meteor_no, n_segments, _, _, _, _, _, _, _, \
             meteor_meas = meteor
@@ -325,7 +329,7 @@ def uploadToArchive(arch_dir, sciencefiles=False, keys=False):
                 continue
             # mp4 must be uploaded before corresponding jpg
             elif (file_ext == '.jpg') and ('FF_' in file_name):
-                if dir_file in validffs:
+                if dir_file in validffs or validffs == []:
                     mp4f = dir_file.replace('.jpg', '.mp4')
                     if os.path.isfile(os.path.join(arch_dir, mp4f)):
                         uploadlist.append({'dir_file':mp4f, 'file_ext': '.mp4', 'src_dir': arch_dir})

From 550e2c12dddc7c3fc66497a44e951d9b775b4a58 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sat, 24 Feb 2024 21:17:11 +0000
Subject: [PATCH 15/29] better logging of brightness checks

---
 uploadToArchive.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/uploadToArchive.py b/uploadToArchive.py
index d446262..1fa617c 100644
--- a/uploadToArchive.py
+++ b/uploadToArchive.py
@@ -260,12 +260,12 @@ def uploadOneFileUKMon(arch_dir, dir_file, s3, targf, file_ext, keys):
 
 
 def checkMags(dir_path, ftpfile_name, min_mag):
-    print('checking for events brighter than mag ', min_mag)
+    log.info('checking for events brighter than mag {}'.format(min_mag))
     ff_names = []
     try:
         meteor_list = readFTPdetectinfo(dir_path, ftpfile_name)  
     except Exception:
-        print('FTPdetect file not present so unable to filter by magnitude')
+        log.info('FTPdetect file not present so unable to filter by magnitude')
         return ff_names
     for meteor in meteor_list:
         ff_name, _, meteor_no, n_segments, _, _, _, _, _, _, _, \
@@ -276,7 +276,7 @@ def checkMags(dir_path, ftpfile_name, min_mag):
             for meas in meteor_meas:
                 best_mag = min(best_mag, meas[9])
             if best_mag > min_mag:
-                print('rejecting {} as too dim'.format(ff_name))
+                log.info('rejecting {} as {} too dim'.format(ff_name, best_mag))
                 continue
             else:
                 ff_names.append(ff_name)

From ecbaac05d8a04fa99e9bf713c5022bc11306a8ba Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sat, 2 Mar 2024 17:45:30 +0000
Subject: [PATCH 16/29] bugfix for error in manual upload

---
 uploadToArchive.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/uploadToArchive.py b/uploadToArchive.py
index 1fa617c..3059697 100644
--- a/uploadToArchive.py
+++ b/uploadToArchive.py
@@ -414,12 +414,16 @@ def manualUpload(targ_dir, sciencefiles=False):
         return True
     else:
         arch_dir = os.path.expanduser(targ_dir)
+        if not os.path.isdir(arch_dir):
+            print('folder {} not found'.format(arch_dir))
+            return False
         return uploadToArchive(arch_dir, sciencefiles, keys=None)
 
 
 if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print('usgae: python uploadToArchive.py ~/RMS_data/ArchivedFiles/dated_dir')
+        exit(0)
     targdir = sys.argv[1]
-    if len(sys.argv) > 2:
-        manualUpload(sys.argv[1], True)
-    else:
-        manualUpload(sys.argv[1])
+    manualUpload(targdir, True)
+    manualUpload(targdir)

From 90312e5319d7358d3064fe27dd9c808195292c35 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Tue, 2 Apr 2024 17:37:59 +0100
Subject: [PATCH 17/29] avoid calling manualUpload twice if testing

---
 uploadToArchive.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/uploadToArchive.py b/uploadToArchive.py
index 3059697..211c1e8 100644
--- a/uploadToArchive.py
+++ b/uploadToArchive.py
@@ -426,4 +426,5 @@ def manualUpload(targ_dir, sciencefiles=False):
         exit(0)
     targdir = sys.argv[1]
     manualUpload(targdir, True)
-    manualUpload(targdir)
+    if targdir != 'test':
+        manualUpload(targdir)

From 1ab0b21241c72c79367fd2657bfb472c69e1ef15 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sun, 7 Apr 2024 13:14:54 +0100
Subject: [PATCH 18/29] typo

---
 uploadToArchive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/uploadToArchive.py b/uploadToArchive.py
index 211c1e8..ccd2af3 100644
--- a/uploadToArchive.py
+++ b/uploadToArchive.py
@@ -422,7 +422,7 @@ def manualUpload(targ_dir, sciencefiles=False):
 
 if __name__ == '__main__':
     if len(sys.argv) < 2:
-        print('usgae: python uploadToArchive.py ~/RMS_data/ArchivedFiles/dated_dir')
+        print('usage: python uploadToArchive.py ~/RMS_data/ArchivedFiles/dated_dir')
         exit(0)
     targdir = sys.argv[1]
     manualUpload(targdir, True)

From 11a571f2ab4ed437a41a412a40f59c9391bd01c8 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sun, 7 Apr 2024 13:23:07 +0100
Subject: [PATCH 19/29] ensure maglim is set to sensible value

---
 uploadToArchive.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/uploadToArchive.py b/uploadToArchive.py
index ccd2af3..d112cff 100644
--- a/uploadToArchive.py
+++ b/uploadToArchive.py
@@ -127,6 +127,9 @@ def readIniFile(filename):
             data = valstr.split('=')
             val = data[1].strip().strip('"')
             vals[data[0]] = val
+    # default min brightness. To be adjusted later
+    if 'MAGLIM' not in vals:
+        vals['MAGLIM'] = 10
     return vals
 
 
@@ -298,7 +301,7 @@ def uploadToArchive(arch_dir, sciencefiles=False, keys=False):
     conn = boto3.Session(aws_access_key_id=keys['AWS_ACCESS_KEY_ID'], aws_secret_access_key=keys['AWS_SECRET_ACCESS_KEY']) 
     s3 = conn.resource('s3', region_name=reg)
     targf = keys['S3FOLDER']
-    maglim = 0
+    maglim = 10.0
     if 'MAGLIM' in inifvals:
         maglim = float(inifvals['MAGLIM'])
 

From 9c6cef1437bfcd97594b13db0910ab666e319161 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Fri, 26 Apr 2024 23:12:52 +0100
Subject: [PATCH 20/29] add mag and mp4 control to ini file

---
 ukmonInstaller.py  | 21 +++++++++++++++++++++
 ukmonPostProc.py   |  5 +++--
 uploadToArchive.py |  3 ---
 3 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/ukmonInstaller.py b/ukmonInstaller.py
index 575b109..6e8a6e2 100644
--- a/ukmonInstaller.py
+++ b/ukmonInstaller.py
@@ -50,6 +50,8 @@ def createDefaultIni(homedir, helperip='3.11.55.160', location='NOTCONFIGURED',
         outf.write("export UKMONHELPER={}\n".format(helperip))
         outf.write("export UKMONKEY={}\n".format(keyfile))
         outf.write("export RMSCFG={}\n".format(rmscfg))
+        outf.write('export DOMP4s=1\n')
+        outf.write('export MINMAG=1\n')
     return True
 
 
@@ -81,9 +83,27 @@ def validateIni(homedir, newhelperip=None):
         createDefaultIni(homedir, newhelperip, location, keyfile, rmscfg)
     if helperip == oldip:
         updateHelperIp(homedir, newhelperip)
+    updateMp4andMag(inifname)
     return True
 
 
+def updateMp4andMag(inif, homedir):
+    """
+    Move the mp4 flag into the ini file and add the minmag flag if missing
+    """
+    domp4s = 0
+    if open(inif, 'r').read()[-1] != '\n':
+        open(inif, 'a').write('\n')
+    if os.path.isfile(os.path.join(homedir, 'domp4s')):
+        domp4s = 1
+        os.remove(os.path.join(homedir, 'domp4s'))
+    if 'DOMP4S' not in open(inif).read():
+        open(inif,'a').write('export DOMP4S={}\n'.format(domp4s))
+    if 'MAGLIM' not in open(inif).read():
+        open(inif,'a').write('export MAGLIM=1\n')
+    return
+
+
 def updateHelperIp(homedir, helperip):
     """
     Update the ukmon.ini file with a new IP address if neeeded. 
@@ -96,6 +116,7 @@ def updateHelperIp(homedir, helperip):
                 outf.write("export UKMONHELPER={}\n".format(helperip))
             else:
                 outf.write('{}'.format(li))
+    return
 
 
 def updateLocation(homedir, newloc):
diff --git a/ukmonPostProc.py b/ukmonPostProc.py
index af3e88d..7dbe72b 100644
--- a/ukmonPostProc.py
+++ b/ukmonPostProc.py
@@ -58,8 +58,9 @@ def rmsExternal(cap_dir, arch_dir, config):
         bff2i.batchFFtoImage(arch_dir, 'jpg')
 
     myloc = os.path.split(os.path.abspath(__file__))[0]
+    inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'))
     log.info('app home is {}'.format(myloc))
-    if os.path.isfile(os.path.join(myloc, 'domp4s')) or os.getenv('DOMP4S', default='1')=='1':
+    if int(inifvals['DOMP4S']) == 1: 
         # generate MP4s of detections
         log.info('generating MP4s')
         ftpdate=''
@@ -69,7 +70,7 @@ def rmsExternal(cap_dir, arch_dir, config):
             ftpdate=os.path.split(arch_dir)[1]
         ftpfile_name="FTPdetectinfo_"+ftpdate+'.txt'
         try:
-            maglim = float(os.getenv('MAGLIM', default='0'))
+            maglim = float(inifvals['MAGLIM'])
             gmp4.generateMP4s(arch_dir, ftpfile_name, min_mag=maglim)
         except Exception:
             gmp4.generateMP4s(arch_dir, ftpfile_name)
diff --git a/uploadToArchive.py b/uploadToArchive.py
index d112cff..0fb7918 100644
--- a/uploadToArchive.py
+++ b/uploadToArchive.py
@@ -127,9 +127,6 @@ def readIniFile(filename):
             data = valstr.split('=')
             val = data[1].strip().strip('"')
             vals[data[0]] = val
-    # default min brightness. To be adjusted later
-    if 'MAGLIM' not in vals:
-        vals['MAGLIM'] = 10
     return vals
 
 

From 726e3009517059a5e79d0a17cd9181c86a9b82f7 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Fri, 26 Apr 2024 23:20:34 +0100
Subject: [PATCH 21/29] add missing param

---
 ukmonInstaller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ukmonInstaller.py b/ukmonInstaller.py
index 6e8a6e2..1ca0508 100644
--- a/ukmonInstaller.py
+++ b/ukmonInstaller.py
@@ -83,7 +83,7 @@ def validateIni(homedir, newhelperip=None):
         createDefaultIni(homedir, newhelperip, location, keyfile, rmscfg)
     if helperip == oldip:
         updateHelperIp(homedir, newhelperip)
-    updateMp4andMag(inifname)
+    updateMp4andMag(inifname, homedir)
     return True
 
 

From 5da02e92e00c8a2e7c5eabcfa7565efbbb88704a Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Fri, 26 Apr 2024 23:29:29 +0100
Subject: [PATCH 22/29] cater for inif not being updated yet

---
 ukmonPostProc.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/ukmonPostProc.py b/ukmonPostProc.py
index 7dbe72b..0291f93 100644
--- a/ukmonPostProc.py
+++ b/ukmonPostProc.py
@@ -60,7 +60,12 @@ def rmsExternal(cap_dir, arch_dir, config):
     myloc = os.path.split(os.path.abspath(__file__))[0]
     inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'))
     log.info('app home is {}'.format(myloc))
-    if int(inifvals['DOMP4S']) == 1: 
+    domp4s = 0
+    if 'DOMP4S' in inifvals:
+        domp4s = int(inifvals['DOMP4S'])
+    elif os.path.isfile(os.path.join(myloc, 'domp4s')):
+        domp4s = 1
+    if domp4s == 1: 
         # generate MP4s of detections
         log.info('generating MP4s')
         ftpdate=''
@@ -70,7 +75,9 @@ def rmsExternal(cap_dir, arch_dir, config):
             ftpdate=os.path.split(arch_dir)[1]
         ftpfile_name="FTPdetectinfo_"+ftpdate+'.txt'
         try:
-            maglim = float(inifvals['MAGLIM'])
+            maglim = 1
+            if 'MAGLIM' in inifvals:
+                maglim = float(inifvals['MAGLIM'])
             gmp4.generateMP4s(arch_dir, ftpfile_name, min_mag=maglim)
         except Exception:
             gmp4.generateMP4s(arch_dir, ftpfile_name)

From acfec4655d0e838203c84c027119112ee616d554 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sat, 27 Apr 2024 00:49:37 +0100
Subject: [PATCH 23/29] reorder the test build steps

---
 tests/run_ub_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh
index bb07db7..0fc8ffc 100644
--- a/tests/run_ub_tests.sh
+++ b/tests/run_ub_tests.sh
@@ -7,11 +7,11 @@ if [ ! -f /keys/ukmon.ini ] ; then
     echo no test config, aborting
     exit
 fi 
-cp /keys/.config /root/source/RMS/
 cp /keys/ukmon.ini . 
 cp /keys/live.key .
 pushd /root/source/RMS 
 git stash && git pull && git stash apply 
+cp /keys/.config /root/source/RMS/
 popd
 touch ./domp4s
 source ukmon.ini

From 4306ac645ae6b363e9183d8a2c8f735076653d2a Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Wed, 8 May 2024 21:38:20 +0100
Subject: [PATCH 24/29] fixing pi4 tests

---
 .github/workflows/python_test.yml | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml
index cab5e01..c03c7bc 100644
--- a/.github/workflows/python_test.yml
+++ b/.github/workflows/python_test.yml
@@ -39,20 +39,16 @@ jobs:
         run: |
           # stop the build if there are Python syntax errors or undefined names
           source ~/vRMS/bin/activate
-          ruff --output-format=github --select=E9,F63,F7,F82 --ignore=E501 --target-version=py37 .
+          ruff check --output-format=github --select=E9,F63,F7,F82 --ignore=E501 --target-version=py37 .
           # default set of ruff rules with GitHub Annotations
-          ruff --output-format=github --ignore=E501 --target-version=py37 .
+          ruff check --output-format=github --ignore=E501 --target-version=py37 .
       - name: Test with pytest
         run: |
           source ~/vRMS/bin/activate
           pip install -r ~/source/RMS/requirements.txt
           export PYTHONPATH=$PYTHONPATH:$(pwd)
-          cp ~/source/testing/ukmon.ini .
-          cp ~/source/testing/live.key .
-          touch ./domp4s
           source ukmon.ini
           pytest -v . --cov=. --cov-report=term-missing --cov-config=.coveragerc_lnx
-          rm ./live.key ./ukmon.ini ./domp4s
 #  Run-refreshtools-on-Ubuntu:
 #    name: Run refreshTools on Ubuntu
 #    runs-on: ubuntu-20.04

From 82a57f9ed42ed5cd652370a4bc8ba4e0cd884054 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Wed, 8 May 2024 21:47:51 +0100
Subject: [PATCH 25/29] add path back in

---
 .github/workflows/python_test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml
index c03c7bc..417c797 100644
--- a/.github/workflows/python_test.yml
+++ b/.github/workflows/python_test.yml
@@ -39,12 +39,14 @@ jobs:
         run: |
           # stop the build if there are Python syntax errors or undefined names
           source ~/vRMS/bin/activate
+          export PATH=/home/pi/.local/bin:$PATH
           ruff check --output-format=github --select=E9,F63,F7,F82 --ignore=E501 --target-version=py37 .
           # default set of ruff rules with GitHub Annotations
           ruff check --output-format=github --ignore=E501 --target-version=py37 .
       - name: Test with pytest
         run: |
           source ~/vRMS/bin/activate
+          export PATH=/home/pi/.local/bin:$PATH
           pip install -r ~/source/RMS/requirements.txt
           export PYTHONPATH=$PYTHONPATH:$(pwd)
           source ukmon.ini

From 79ec3502fb37a6920a65c34f066d1cb379001ed4 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Thu, 9 May 2024 22:57:55 +0100
Subject: [PATCH 26/29] bugfix to push jpgs properly

---
 uploadToArchive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/uploadToArchive.py b/uploadToArchive.py
index 0fb7918..e22a83d 100644
--- a/uploadToArchive.py
+++ b/uploadToArchive.py
@@ -329,7 +329,7 @@ def uploadToArchive(arch_dir, sciencefiles=False, keys=False):
                 continue
             # mp4 must be uploaded before corresponding jpg
             elif (file_ext == '.jpg') and ('FF_' in file_name):
-                if dir_file in validffs or validffs == []:
+                if file_name in validffs or validffs == []:
                     mp4f = dir_file.replace('.jpg', '.mp4')
                     if os.path.isfile(os.path.join(arch_dir, mp4f)):
                         uploadlist.append({'dir_file':mp4f, 'file_ext': '.mp4', 'src_dir': arch_dir})

From 951941a2d75ffa7e9dd5cddf7f3c87538df61063 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Thu, 9 May 2024 23:11:33 +0100
Subject: [PATCH 27/29] add some debug

---
 uploadToArchive.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/uploadToArchive.py b/uploadToArchive.py
index e22a83d..2367e52 100644
--- a/uploadToArchive.py
+++ b/uploadToArchive.py
@@ -285,7 +285,7 @@ def checkMags(dir_path, ftpfile_name, min_mag):
 
 def uploadToArchive(arch_dir, sciencefiles=False, keys=False):
     # Upload all relevant files from *arch_dir* to ukmon's S3 Archive
-
+    print(f'{arch_dir}')
     myloc = os.path.split(os.path.abspath(__file__))[0]
     inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'))
     if not inifvals:
@@ -322,6 +322,7 @@ def uploadToArchive(arch_dir, sciencefiles=False, keys=False):
                 break
     else:
         # upload everything
+        print('upload everythin')
         for dir_file in dir_contents:
             file_name, file_ext = os.path.splitext(dir_file)
             file_ext = file_ext.lower()

From 74cc30279cf693424b072d48fff1ee00e4883584 Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Thu, 9 May 2024 23:29:11 +0100
Subject: [PATCH 28/29] bugfix for jpg upload

---
 uploadToArchive.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/uploadToArchive.py b/uploadToArchive.py
index 2367e52..9e66f5b 100644
--- a/uploadToArchive.py
+++ b/uploadToArchive.py
@@ -279,13 +279,13 @@ def checkMags(dir_path, ftpfile_name, min_mag):
                 log.info('rejecting {} as {} too dim'.format(ff_name, best_mag))
                 continue
             else:
-                ff_names.append(ff_name)
+                ff_names.append(ff_name.replace('.fits', '.jpg'))
     return ff_names
 
 
 def uploadToArchive(arch_dir, sciencefiles=False, keys=False):
     # Upload all relevant files from *arch_dir* to ukmon's S3 Archive
-    print(f'{arch_dir}')
+
     myloc = os.path.split(os.path.abspath(__file__))[0]
     inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'))
     if not inifvals:
@@ -322,7 +322,6 @@ def uploadToArchive(arch_dir, sciencefiles=False, keys=False):
                 break
     else:
         # upload everything
-        print('upload everythin')
         for dir_file in dir_contents:
             file_name, file_ext = os.path.splitext(dir_file)
             file_ext = file_ext.lower()
@@ -330,7 +329,7 @@ def uploadToArchive(arch_dir, sciencefiles=False, keys=False):
                 continue
             # mp4 must be uploaded before corresponding jpg
             elif (file_ext == '.jpg') and ('FF_' in file_name):
-                if file_name in validffs or validffs == []:
+                if dir_file in validffs or validffs == []:
                     mp4f = dir_file.replace('.jpg', '.mp4')
                     if os.path.isfile(os.path.join(arch_dir, mp4f)):
                         uploadlist.append({'dir_file':mp4f, 'file_ext': '.mp4', 'src_dir': arch_dir})

From aa90f7c67632d220e8556337bc79889de3cbb81e Mon Sep 17 00:00:00 2001
From: Mark McIntyre <mark.jm.mcintyre@cesmail.net>
Date: Sat, 11 May 2024 18:22:18 +0100
Subject: [PATCH 29/29] fixing testing

---
 .github/workflows/python_test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml
index 417c797..5c0ccc4 100644
--- a/.github/workflows/python_test.yml
+++ b/.github/workflows/python_test.yml
@@ -49,6 +49,7 @@ jobs:
           export PATH=/home/pi/.local/bin:$PATH
           pip install -r ~/source/RMS/requirements.txt
           export PYTHONPATH=$PYTHONPATH:$(pwd)
+          cp /home/pi/RMS_data/test/* .
           source ukmon.ini
           pytest -v . --cov=. --cov-report=term-missing --cov-config=.coveragerc_lnx
 #  Run-refreshtools-on-Ubuntu: