From 598b1536548923516327dcc8a4fee88a2d98725d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 8 Feb 2025 20:06:22 +0000 Subject: [PATCH 01/12] improve error messages --- liveMonitor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/liveMonitor.py b/liveMonitor.py index 3336c63..076ed5c 100644 --- a/liveMonitor.py +++ b/liveMonitor.py @@ -134,8 +134,10 @@ def monitorLogFile(camloc, rmscfg): log.info('restarting to read {}'.format(logf)) pass except Exception as e: - log.info('restarting due to crash:') + log.info('Problem reading RMS log: {} - will retry'.format(logf)) log.info(e, exc_info=True) + # reload the RMS config file in case its been updated + cfg = cr.parse(os.path.expanduser(rmscfg)) pass From 5241a5569aa22804cf22f951f9f2051523712289 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 13 Jun 2025 13:32:09 +0100 Subject: [PATCH 02/12] Hopefully fix logging --- ukmonInstaller.py | 5 +++++ ukmonPostProc.py | 44 +++++++++++++++++++++++++++++++++----------- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/ukmonInstaller.py b/ukmonInstaller.py index a97c5e4..65a8126 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -14,10 +14,15 @@ import json import tempfile +import logging + import RMS.ConfigReader as cr from RMS.Misc import isRaspberryPi from uploadToArchive import readIniFile +log = logging.getLogger() +log.setLevel(logging.WARNING) + oldip = '3.9.65.98' currip = '3.11.55.160' diff --git a/ukmonPostProc.py b/ukmonPostProc.py index 19b04df..9fe69b4 100644 --- a/ukmonPostProc.py +++ b/ukmonPostProc.py @@ -17,13 +17,37 @@ import RMS.ConfigReader as cr from importlib import import_module as impmod import logging -from RMS.Logger import initLogging +import datetime from uploadToArchive import uploadToArchive, readIniFile -log = logging.getLogger("logger") +log = logging.getLogger() +log.setLevel(logging.INFO) -_rms_logger_initialized = False + +def setupLogging(logpath, prefix): + print('about to initialise logger') + logdir = os.path.expanduser(logpath) + os.makedirs(logdir, exist_ok=True) + + logfilename = os.path.join(logdir, prefix + datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d_%H%M%S.%f') + '.log') + handler = logging.handlers.TimedRotatingFileHandler(logfilename, when='D', interval=1) + handler.setLevel(logging.INFO) + formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s-%(module)s-line:%(lineno)d - %(message)s', + datefmt='%Y/%m/%d %H:%M:%S') + handler.setFormatter(formatter) + log.addHandler(handler) + + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.WARNING) + formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s-%(module)s-line:%(lineno)d - %(message)s', + datefmt='%Y/%m/%d %H:%M:%S') + ch.setFormatter(formatter) + log.addHandler(ch) + + log.setLevel(logging.INFO) + log.info('logging initialised') + return def rmsExternal(cap_dir, arch_dir, config): @@ -38,12 +62,7 @@ def rmsExternal(cap_dir, arch_dir, config): an RMS config object in Python. """ - # clear existing log handlers - log = logging.getLogger("logger") - while len(log.handlers) > 0: - log.removeHandler(log.handlers[0]) - - initLogging(config, log_file_prefix='ukmon_', level=logging.INFO) + setupLogging(os.path.join(config.data_dir, config.log_dir), f'ukmon_log_{config.stationID}_') log.info('ukmon external script started') rebootlockfile = os.path.join(config.data_dir, config.reboot_lock_file) @@ -95,6 +114,8 @@ def rmsExternal(cap_dir, arch_dir, config): if os.path.isfile(extrascrfn): extrascript = open(extrascrfn,'r').readline().strip() log.info('running additional script {:s}'.format(extrascript)) + while len(log.handlers) > 0: + log.removeHandler(log.handlers[0]) sloc, sname = os.path.split(extrascript) sys.path.append(sloc) scrname, _ = os.path.splitext(sname) @@ -140,11 +161,12 @@ def main(args): print('eg python ukmonPostProc.py UK0006_20210312_183741_206154') print('\n nb: script must be run from RMS source folder') return False + arch_dir = args[1] myloc = os.path.split(os.path.abspath(__file__))[0] inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) if inifvals is None: - log.warning('unable to open ukmon ini file') + print('unable to open ukmon ini file') return 'unable to open ukmon ini file' try: rmscfg = inifvals['RMSCFG'] @@ -153,7 +175,7 @@ def main(args): try: if 'ConfirmedFiles' in arch_dir or 'ArchivedFiles' in arch_dir or 'CapturedFiles' in arch_dir: _, arch_dir = os.path.split(arch_dir) - log.info('RMS config read from {}'.format(rmscfg)) + print('RMS config read from {}'.format(rmscfg)) ret = manualRerun(arch_dir, rmscfg) return ret except Exception: From 95706867e3a547cbe2eb2fab5a8c4ff8b791e283 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 13 Jun 2025 13:32:47 +0100 Subject: [PATCH 03/12] error in tests --- tests/test_uploadToArchive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_uploadToArchive.py b/tests/test_uploadToArchive.py index 5e11a73..984d662 100644 --- a/tests/test_uploadToArchive.py +++ b/tests/test_uploadToArchive.py @@ -20,7 +20,7 @@ def test_checkMags(): daydir = 'UK0006_20220914_185543_087124' validffs = checkMags(arch_dir, 'FTPdetectinfo_{}.txt'.format(daydir), maglim) print(validffs) - assert 'FF_UK0006_20220914_200343_841_0101120.fits' in validffs + assert 'FF_UK0006_20220914_200343_841_0101120.jpg' in validffs def test_readIniFile(): From 77e77a423cf684f8f081ed8acdb8177e2dac41d3 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 13 Jun 2025 13:42:23 +0100 Subject: [PATCH 04/12] debugging testing --- tests/run_ub_tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh index 0fc8ffc..06545fb 100644 --- a/tests/run_ub_tests.sh +++ b/tests/run_ub_tests.sh @@ -1,7 +1,7 @@ #!/bin/bash # Copyright (c) Mark McIntyre -# tests designed to run on a Pi +# tests designed to run on ubuntu here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" if [ ! -f /keys/ukmon.ini ] ; then echo no test config, aborting @@ -15,6 +15,7 @@ cp /keys/.config /root/source/RMS/ popd touch ./domp4s source ukmon.ini +echo testing for $LOCATION pip install -r ./requirements.txt pip install --upgrade ruff pytest xmltodict pytest-cov export PYTHONPATH=$PYTHONPATH:/root/source/RMS:${here}/.. From 6879496a716d0ca8b8d3d2956123a20a5f811638 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 13 Jun 2025 13:56:33 +0100 Subject: [PATCH 05/12] update action script --- .github/workflows/python_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml index 5c0ccc4..8fb81b4 100644 --- a/.github/workflows/python_test.yml +++ b/.github/workflows/python_test.yml @@ -18,7 +18,7 @@ jobs: test: name: run pi4 tests - runs-on: [testpi4] + runs-on: [arm64] strategy: matrix: python-version: ["3.7"] #, "3.8", "3.9", "3.10", "3.11"] From edcc8c62e8c12d8b11c7ab884c726f9a9c735052 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 13 Jun 2025 14:11:35 +0100 Subject: [PATCH 06/12] small bugfix --- .github/workflows/python_test.yml | 2 +- tests/run_tests.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml index 8fb81b4..38f1bd2 100644 --- a/.github/workflows/python_test.yml +++ b/.github/workflows/python_test.yml @@ -49,7 +49,7 @@ jobs: export PATH=/home/pi/.local/bin:$PATH pip install -r ~/source/RMS/requirements.txt export PYTHONPATH=$PYTHONPATH:$(pwd) - cp /home/pi/RMS_data/test/* . + #cp /home/pi/RMS_data/test/* . source ukmon.ini pytest -v . --cov=. --cov-report=term-missing --cov-config=.coveragerc_lnx # Run-refreshtools-on-Ubuntu: diff --git a/tests/run_tests.sh b/tests/run_tests.sh index e4b4ec8..8a36820 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -17,6 +17,7 @@ cp ~/source/testing/badini.ini . cp ~/source/testing/live.key . touch ./domp4s source ukmon.ini +echo "ready to test" pytest -v ./tests/ --cov=./ --cov-report=term-missing --cov-config=.coveragerc_lnx rm ./live.key ./ukmon.ini ./domp4s ./badini.ini [ -f /tmp/ukmon.ini ] && mv /tmp/ukmon.ini . From 7629dcfae7bb588edaecc21601f90a6691f8b79f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 13 Jun 2025 14:19:35 +0100 Subject: [PATCH 07/12] fix path --- .github/workflows/python_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml index 38f1bd2..9ba7790 100644 --- a/.github/workflows/python_test.yml +++ b/.github/workflows/python_test.yml @@ -49,7 +49,7 @@ jobs: export PATH=/home/pi/.local/bin:$PATH pip install -r ~/source/RMS/requirements.txt export PYTHONPATH=$PYTHONPATH:$(pwd) - #cp /home/pi/RMS_data/test/* . + cd ../ukmon-pitools/ source ukmon.ini pytest -v . --cov=. --cov-report=term-missing --cov-config=.coveragerc_lnx # Run-refreshtools-on-Ubuntu: From 96a575f8a83f12f7703f571dd4b7db48c7bda3ba Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 13 Jun 2025 14:22:27 +0100 Subject: [PATCH 08/12] whazzup here --- .github/workflows/python_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml index 9ba7790..0a01c7e 100644 --- a/.github/workflows/python_test.yml +++ b/.github/workflows/python_test.yml @@ -50,6 +50,8 @@ jobs: pip install -r ~/source/RMS/requirements.txt export PYTHONPATH=$PYTHONPATH:$(pwd) cd ../ukmon-pitools/ + pwd + ls -ltr source ukmon.ini pytest -v . --cov=. --cov-report=term-missing --cov-config=.coveragerc_lnx # Run-refreshtools-on-Ubuntu: From 37feb39df1b0dfe173c9433e3a69fb80fdfa1cf8 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 13 Jun 2025 14:40:18 +0100 Subject: [PATCH 09/12] grab the ini files for testing --- .github/workflows/python_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml index 0a01c7e..046faba 100644 --- a/.github/workflows/python_test.yml +++ b/.github/workflows/python_test.yml @@ -50,7 +50,8 @@ jobs: pip install -r ~/source/RMS/requirements.txt export PYTHONPATH=$PYTHONPATH:$(pwd) cd ../ukmon-pitools/ - pwd + cp ~/source/ukmon-pitools/ukmon.ini . + cp ~/source/ukmon-pitools/live.key . ls -ltr source ukmon.ini pytest -v . --cov=. --cov-report=term-missing --cov-config=.coveragerc_lnx From 845c34130c200e5edf74075d9668d04ae9422bb0 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 13 Jun 2025 14:43:00 +0100 Subject: [PATCH 10/12] more fixing tests --- .github/workflows/python_test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml index 046faba..be11608 100644 --- a/.github/workflows/python_test.yml +++ b/.github/workflows/python_test.yml @@ -46,13 +46,12 @@ jobs: - name: Test with pytest run: | source ~/vRMS/bin/activate - export PATH=/home/pi/.local/bin:$PATH + export PATH=~/.local/bin:$PATH pip install -r ~/source/RMS/requirements.txt - export PYTHONPATH=$PYTHONPATH:$(pwd) + export PYTHONPATH=$PYTHONPATH:$(pwd):~/source/RMS cd ../ukmon-pitools/ cp ~/source/ukmon-pitools/ukmon.ini . cp ~/source/ukmon-pitools/live.key . - ls -ltr source ukmon.ini pytest -v . --cov=. --cov-report=term-missing --cov-config=.coveragerc_lnx # Run-refreshtools-on-Ubuntu: From 0cb2279e30aa420df58549f4a4101bd6101b80e4 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 13 Jun 2025 14:49:34 +0100 Subject: [PATCH 11/12] fixing test on pi4 --- tests/test_uploadToArchive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_uploadToArchive.py b/tests/test_uploadToArchive.py index 984d662..93c9bed 100644 --- a/tests/test_uploadToArchive.py +++ b/tests/test_uploadToArchive.py @@ -31,7 +31,7 @@ def test_readIniFile(): def test_readKeyFile(): inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini')) vals = readKeyFile(os.path.join(basedir,'..','live.key'), inifs) - assert vals['S3FOLDER'] == 'archive/Tackley' + assert vals['S3FOLDER'] == 'tmp/testpi4' def test_readKeyfileIni(): From 86f85682351bda48a0ef60c8ed12b55c5884a074 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 13 Jun 2025 15:53:51 +0100 Subject: [PATCH 12/12] fix test on both targets --- tests/test_uploadToArchive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_uploadToArchive.py b/tests/test_uploadToArchive.py index 93c9bed..dfb17b7 100644 --- a/tests/test_uploadToArchive.py +++ b/tests/test_uploadToArchive.py @@ -31,7 +31,7 @@ def test_readIniFile(): def test_readKeyFile(): inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini')) vals = readKeyFile(os.path.join(basedir,'..','live.key'), inifs) - assert vals['S3FOLDER'] == 'tmp/testpi4' + assert vals['S3FOLDER'] in ['tmp/testpi4','archive/Tackley'] def test_readKeyfileIni():