Various fixes and improvements

.github/workflows/python_test.yml

@@ -18,7 +18,7 @@ jobs:
 
   test:
     name: run pi4 tests
-    runs-on: [testpi4]
+    runs-on: [arm64]
     strategy:
       matrix:
         python-version: ["3.7"] #, "3.8", "3.9", "3.10", "3.11"]
@@ -46,10 +46,12 @@ jobs:
       - name: Test with pytest
         run: |
           source ~/vRMS/bin/activate
-          export PATH=/home/pi/.local/bin:$PATH
+          export PATH=~/.local/bin:$PATH
           pip install -r ~/source/RMS/requirements.txt
-          export PYTHONPATH=$PYTHONPATH:$(pwd)
-          cp /home/pi/RMS_data/test/* .
+          export PYTHONPATH=$PYTHONPATH:$(pwd):~/source/RMS
+          cd ../ukmon-pitools/
+          cp ~/source/ukmon-pitools/ukmon.ini .
+          cp ~/source/ukmon-pitools/live.key .
           source ukmon.ini
           pytest -v . --cov=. --cov-report=term-missing --cov-config=.coveragerc_lnx
 #  Run-refreshtools-on-Ubuntu:

liveMonitor.py

@@ -134,8 +134,10 @@ def monitorLogFile(camloc, rmscfg):
             log.info('restarting to read {}'.format(logf))
             pass
         except Exception as e:
-            log.info('restarting due to crash:')
+            log.info('Problem reading RMS log: {} - will retry'.format(logf))
             log.info(e, exc_info=True)
+            # reload the RMS config file in case its been updated
+            cfg = cr.parse(os.path.expanduser(rmscfg))
             pass
 
 

tests/run_tests.sh

@@ -17,6 +17,7 @@ cp ~/source/testing/badini.ini .
 cp ~/source/testing/live.key .
 touch ./domp4s
 source ukmon.ini
+echo "ready to test"
 pytest -v ./tests/ --cov=./ --cov-report=term-missing --cov-config=.coveragerc_lnx 
 rm ./live.key ./ukmon.ini ./domp4s ./badini.ini
 [ -f /tmp/ukmon.ini ] && mv /tmp/ukmon.ini .

tests/run_ub_tests.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # Copyright (c) Mark McIntyre
 
-# tests designed to run on a Pi 
+# tests designed to run on ubuntu
 here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 if [ ! -f /keys/ukmon.ini ] ; then
     echo no test config, aborting
@@ -15,6 +15,7 @@ cp /keys/.config /root/source/RMS/
 popd
 touch ./domp4s
 source ukmon.ini
+echo testing for $LOCATION
 pip install -r ./requirements.txt
 pip install --upgrade ruff pytest xmltodict pytest-cov 
 export PYTHONPATH=$PYTHONPATH:/root/source/RMS:${here}/..

tests/test_uploadToArchive.py

@@ -20,7 +20,7 @@ def test_checkMags():
     daydir = 'UK0006_20220914_185543_087124'
     validffs = checkMags(arch_dir, 'FTPdetectinfo_{}.txt'.format(daydir), maglim)
     print(validffs)
-    assert 'FF_UK0006_20220914_200343_841_0101120.fits' in validffs
+    assert 'FF_UK0006_20220914_200343_841_0101120.jpg' in validffs
 
 
 def test_readIniFile():
@@ -31,7 +31,7 @@ def test_readIniFile():
 def test_readKeyFile():
     inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'))
     vals = readKeyFile(os.path.join(basedir,'..','live.key'), inifs)
-    assert vals['S3FOLDER'] == 'archive/Tackley'
+    assert vals['S3FOLDER'] in  ['tmp/testpi4','archive/Tackley']
 
 
 def test_readKeyfileIni():

ukmonInstaller.py

@@ -14,10 +14,15 @@
 
 import json
 import tempfile
+import logging
+
 import RMS.ConfigReader as cr
 from RMS.Misc import isRaspberryPi
 from uploadToArchive import readIniFile
 
+log = logging.getLogger()
+log.setLevel(logging.WARNING)
+
 oldip = '3.9.65.98'
 currip = '3.11.55.160'
 

ukmonPostProc.py

@@ -17,13 +17,37 @@
 import RMS.ConfigReader as cr
 from importlib import import_module as impmod
 import logging
-from RMS.Logger import initLogging
+import datetime
 
 from uploadToArchive import uploadToArchive, readIniFile
 
-log = logging.getLogger("logger")
+log = logging.getLogger()
+log.setLevel(logging.INFO)
 
-_rms_logger_initialized = False
+
+def setupLogging(logpath, prefix):
+    print('about to initialise logger')
+    logdir = os.path.expanduser(logpath)
+    os.makedirs(logdir, exist_ok=True)
+
+    logfilename = os.path.join(logdir, prefix + datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d_%H%M%S.%f') + '.log')
+    handler = logging.handlers.TimedRotatingFileHandler(logfilename, when='D', interval=1) 
+    handler.setLevel(logging.INFO)
+    formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s-%(module)s-line:%(lineno)d - %(message)s', 
+        datefmt='%Y/%m/%d %H:%M:%S')
+    handler.setFormatter(formatter)
+    log.addHandler(handler)
+
+    ch = logging.StreamHandler(sys.stdout)
+    ch.setLevel(logging.WARNING)
+    formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s-%(module)s-line:%(lineno)d - %(message)s', 
+        datefmt='%Y/%m/%d %H:%M:%S')
+    ch.setFormatter(formatter)
+    log.addHandler(ch)
+
+    log.setLevel(logging.INFO)
+    log.info('logging initialised')
+    return 
 
 
 def rmsExternal(cap_dir, arch_dir, config):
@@ -38,12 +62,7 @@ def rmsExternal(cap_dir, arch_dir, config):
     an RMS config object in Python. 
 
     """
-    # clear existing log handlers
-    log = logging.getLogger("logger")
-    while len(log.handlers) > 0:
-        log.removeHandler(log.handlers[0])
-
-    initLogging(config, log_file_prefix='ukmon_', level=logging.INFO)
+    setupLogging(os.path.join(config.data_dir, config.log_dir), f'ukmon_log_{config.stationID}_')
     log.info('ukmon external script started')
 
     rebootlockfile = os.path.join(config.data_dir, config.reboot_lock_file)
@@ -95,6 +114,8 @@ def rmsExternal(cap_dir, arch_dir, config):
     if os.path.isfile(extrascrfn):
         extrascript = open(extrascrfn,'r').readline().strip()
         log.info('running additional script {:s}'.format(extrascript))
+        while len(log.handlers) > 0:
+            log.removeHandler(log.handlers[0])  
         sloc, sname = os.path.split(extrascript)
         sys.path.append(sloc)
         scrname, _ = os.path.splitext(sname)
@@ -140,11 +161,12 @@ def main(args):
         print('eg python ukmonPostProc.py UK0006_20210312_183741_206154')
         print('\n nb: script must be run from RMS source folder')
         return False
+
     arch_dir = args[1]
     myloc = os.path.split(os.path.abspath(__file__))[0]
     inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'))
     if inifvals is None:
-        log.warning('unable to open ukmon ini file')
+        print('unable to open ukmon ini file')
         return 'unable to open ukmon ini file'
     try:
         rmscfg = inifvals['RMSCFG']
@@ -153,7 +175,7 @@ def main(args):
     try:
         if 'ConfirmedFiles' in arch_dir or 'ArchivedFiles' in arch_dir or 'CapturedFiles' in arch_dir:
             _, arch_dir = os.path.split(arch_dir)
-        log.info('RMS config read from {}'.format(rmscfg))
+        print('RMS config read from {}'.format(rmscfg))
         ret = manualRerun(arch_dir, rmscfg)
         return ret
     except Exception: