diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml index 5c0ccc4..be11608 100644 --- a/.github/workflows/python_test.yml +++ b/.github/workflows/python_test.yml @@ -18,7 +18,7 @@ jobs: test: name: run pi4 tests - runs-on: [testpi4] + runs-on: [arm64] strategy: matrix: python-version: ["3.7"] #, "3.8", "3.9", "3.10", "3.11"] @@ -46,10 +46,12 @@ jobs: - name: Test with pytest run: | source ~/vRMS/bin/activate - export PATH=/home/pi/.local/bin:$PATH + export PATH=~/.local/bin:$PATH pip install -r ~/source/RMS/requirements.txt - export PYTHONPATH=$PYTHONPATH:$(pwd) - cp /home/pi/RMS_data/test/* . + export PYTHONPATH=$PYTHONPATH:$(pwd):~/source/RMS + cd ../ukmon-pitools/ + cp ~/source/ukmon-pitools/ukmon.ini . + cp ~/source/ukmon-pitools/live.key . source ukmon.ini pytest -v . --cov=. --cov-report=term-missing --cov-config=.coveragerc_lnx # Run-refreshtools-on-Ubuntu: diff --git a/liveMonitor.py b/liveMonitor.py index 3336c63..076ed5c 100644 --- a/liveMonitor.py +++ b/liveMonitor.py @@ -134,8 +134,10 @@ def monitorLogFile(camloc, rmscfg): log.info('restarting to read {}'.format(logf)) pass except Exception as e: - log.info('restarting due to crash:') + log.info('Problem reading RMS log: {} - will retry'.format(logf)) log.info(e, exc_info=True) + # reload the RMS config file in case its been updated + cfg = cr.parse(os.path.expanduser(rmscfg)) pass diff --git a/tests/run_tests.sh b/tests/run_tests.sh index e4b4ec8..8a36820 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -17,6 +17,7 @@ cp ~/source/testing/badini.ini . cp ~/source/testing/live.key . touch ./domp4s source ukmon.ini +echo "ready to test" pytest -v ./tests/ --cov=./ --cov-report=term-missing --cov-config=.coveragerc_lnx rm ./live.key ./ukmon.ini ./domp4s ./badini.ini [ -f /tmp/ukmon.ini ] && mv /tmp/ukmon.ini . diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh index 0fc8ffc..06545fb 100644 --- a/tests/run_ub_tests.sh +++ b/tests/run_ub_tests.sh @@ -1,7 +1,7 @@ #!/bin/bash # Copyright (c) Mark McIntyre -# tests designed to run on a Pi +# tests designed to run on ubuntu here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" if [ ! -f /keys/ukmon.ini ] ; then echo no test config, aborting @@ -15,6 +15,7 @@ cp /keys/.config /root/source/RMS/ popd touch ./domp4s source ukmon.ini +echo testing for $LOCATION pip install -r ./requirements.txt pip install --upgrade ruff pytest xmltodict pytest-cov export PYTHONPATH=$PYTHONPATH:/root/source/RMS:${here}/.. diff --git a/tests/test_uploadToArchive.py b/tests/test_uploadToArchive.py index 5e11a73..dfb17b7 100644 --- a/tests/test_uploadToArchive.py +++ b/tests/test_uploadToArchive.py @@ -20,7 +20,7 @@ def test_checkMags(): daydir = 'UK0006_20220914_185543_087124' validffs = checkMags(arch_dir, 'FTPdetectinfo_{}.txt'.format(daydir), maglim) print(validffs) - assert 'FF_UK0006_20220914_200343_841_0101120.fits' in validffs + assert 'FF_UK0006_20220914_200343_841_0101120.jpg' in validffs def test_readIniFile(): @@ -31,7 +31,7 @@ def test_readIniFile(): def test_readKeyFile(): inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini')) vals = readKeyFile(os.path.join(basedir,'..','live.key'), inifs) - assert vals['S3FOLDER'] == 'archive/Tackley' + assert vals['S3FOLDER'] in ['tmp/testpi4','archive/Tackley'] def test_readKeyfileIni(): diff --git a/ukmonInstaller.py b/ukmonInstaller.py index a97c5e4..65a8126 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -14,10 +14,15 @@ import json import tempfile +import logging + import RMS.ConfigReader as cr from RMS.Misc import isRaspberryPi from uploadToArchive import readIniFile +log = logging.getLogger() +log.setLevel(logging.WARNING) + oldip = '3.9.65.98' currip = '3.11.55.160' diff --git a/ukmonPostProc.py b/ukmonPostProc.py index 19b04df..9fe69b4 100644 --- a/ukmonPostProc.py +++ b/ukmonPostProc.py @@ -17,13 +17,37 @@ import RMS.ConfigReader as cr from importlib import import_module as impmod import logging -from RMS.Logger import initLogging +import datetime from uploadToArchive import uploadToArchive, readIniFile -log = logging.getLogger("logger") +log = logging.getLogger() +log.setLevel(logging.INFO) -_rms_logger_initialized = False + +def setupLogging(logpath, prefix): + print('about to initialise logger') + logdir = os.path.expanduser(logpath) + os.makedirs(logdir, exist_ok=True) + + logfilename = os.path.join(logdir, prefix + datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d_%H%M%S.%f') + '.log') + handler = logging.handlers.TimedRotatingFileHandler(logfilename, when='D', interval=1) + handler.setLevel(logging.INFO) + formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s-%(module)s-line:%(lineno)d - %(message)s', + datefmt='%Y/%m/%d %H:%M:%S') + handler.setFormatter(formatter) + log.addHandler(handler) + + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.WARNING) + formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s-%(module)s-line:%(lineno)d - %(message)s', + datefmt='%Y/%m/%d %H:%M:%S') + ch.setFormatter(formatter) + log.addHandler(ch) + + log.setLevel(logging.INFO) + log.info('logging initialised') + return def rmsExternal(cap_dir, arch_dir, config): @@ -38,12 +62,7 @@ def rmsExternal(cap_dir, arch_dir, config): an RMS config object in Python. """ - # clear existing log handlers - log = logging.getLogger("logger") - while len(log.handlers) > 0: - log.removeHandler(log.handlers[0]) - - initLogging(config, log_file_prefix='ukmon_', level=logging.INFO) + setupLogging(os.path.join(config.data_dir, config.log_dir), f'ukmon_log_{config.stationID}_') log.info('ukmon external script started') rebootlockfile = os.path.join(config.data_dir, config.reboot_lock_file) @@ -95,6 +114,8 @@ def rmsExternal(cap_dir, arch_dir, config): if os.path.isfile(extrascrfn): extrascript = open(extrascrfn,'r').readline().strip() log.info('running additional script {:s}'.format(extrascript)) + while len(log.handlers) > 0: + log.removeHandler(log.handlers[0]) sloc, sname = os.path.split(extrascript) sys.path.append(sloc) scrname, _ = os.path.splitext(sname) @@ -140,11 +161,12 @@ def main(args): print('eg python ukmonPostProc.py UK0006_20210312_183741_206154') print('\n nb: script must be run from RMS source folder') return False + arch_dir = args[1] myloc = os.path.split(os.path.abspath(__file__))[0] inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) if inifvals is None: - log.warning('unable to open ukmon ini file') + print('unable to open ukmon ini file') return 'unable to open ukmon ini file' try: rmscfg = inifvals['RMSCFG'] @@ -153,7 +175,7 @@ def main(args): try: if 'ConfirmedFiles' in arch_dir or 'ArchivedFiles' in arch_dir or 'CapturedFiles' in arch_dir: _, arch_dir = os.path.split(arch_dir) - log.info('RMS config read from {}'.format(rmscfg)) + print('RMS config read from {}'.format(rmscfg)) ret = manualRerun(arch_dir, rmscfg) return ret except Exception: