diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml index 67aeaeb..5c0ccc4 100644 --- a/.github/workflows/python_test.yml +++ b/.github/workflows/python_test.yml @@ -1,8 +1,23 @@ name: Test ukmon-pitools on: [push] jobs: + container_test: + name: run containerised test + runs-on: ubuntu-latest + container: + image: docker.io/markmac99/rmstestcont:latest + env: + MYENV: foo + ports: + - 22 + - 443 + steps: + - uses: actions/checkout@v3 + - run: chmod +x ./tests/run_ub_tests.sh + - run: ./tests/run_ub_tests.sh + test: - name: run python tests + name: run pi4 tests runs-on: [testpi4] strategy: matrix: @@ -24,20 +39,19 @@ jobs: run: | # stop the build if there are Python syntax errors or undefined names source ~/vRMS/bin/activate - ruff --output-format=github --select=E9,F63,F7,F82 --ignore=E501 --target-version=py37 . + export PATH=/home/pi/.local/bin:$PATH + ruff check --output-format=github --select=E9,F63,F7,F82 --ignore=E501 --target-version=py37 . # default set of ruff rules with GitHub Annotations - ruff --output-format=github --ignore=E501 --target-version=py37 . + ruff check --output-format=github --ignore=E501 --target-version=py37 . - name: Test with pytest run: | source ~/vRMS/bin/activate + export PATH=/home/pi/.local/bin:$PATH pip install -r ~/source/RMS/requirements.txt export PYTHONPATH=$PYTHONPATH:$(pwd) - cp ~/source/testing/ukmon.ini . - cp ~/source/testing/live.key . - touch ./domp4s + cp /home/pi/RMS_data/test/* . source ukmon.ini pytest -v . --cov=. --cov-report=term-missing --cov-config=.coveragerc_lnx - rm ./live.key ./ukmon.ini ./domp4s # Run-refreshtools-on-Ubuntu: # name: Run refreshTools on Ubuntu # runs-on: ubuntu-20.04 diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh new file mode 100644 index 0000000..0fc8ffc --- /dev/null +++ b/tests/run_ub_tests.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright (c) Mark McIntyre + +# tests designed to run on a Pi +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +if [ ! -f /keys/ukmon.ini ] ; then + echo no test config, aborting + exit +fi +cp /keys/ukmon.ini . +cp /keys/live.key . +pushd /root/source/RMS +git stash && git pull && git stash apply +cp /keys/.config /root/source/RMS/ +popd +touch ./domp4s +source ukmon.ini +pip install -r ./requirements.txt +pip install --upgrade ruff pytest xmltodict pytest-cov +export PYTHONPATH=$PYTHONPATH:/root/source/RMS:${here}/.. +cd /root/source/RMS +ls -ltra .config +pwd +pytest -v $here/ --cov=$here/../ --cov-report=term-missing --cov-config=$here/../.coveragerc_lnx diff --git a/tests/test_uploadToArchive.py b/tests/test_uploadToArchive.py index 2e60661..5e11a73 100644 --- a/tests/test_uploadToArchive.py +++ b/tests/test_uploadToArchive.py @@ -2,7 +2,7 @@ import boto3 import os -from uploadToArchive import readKeyFile, uploadOneFile, manualUpload, readIniFile +from uploadToArchive import readKeyFile, uploadOneFile, manualUpload, readIniFile, checkMags from ukmonInstaller import createDefaultIni basedir = os.path.realpath(os.path.dirname(__file__)) @@ -11,6 +11,18 @@ os.makedirs(tmpdir) +def test_checkMags(): + inifvals = readIniFile(os.path.join(basedir,'..','ukmon.ini')) + maglim = 6 + if 'MAGLIM' in inifvals: + maglim = float(inifvals['MAGLIM']) + arch_dir = os.path.join(basedir, 'ukmarch','sampledata', 'UK0006_20220914_185543_087124') + daydir = 'UK0006_20220914_185543_087124' + validffs = checkMags(arch_dir, 'FTPdetectinfo_{}.txt'.format(daydir), maglim) + print(validffs) + assert 'FF_UK0006_20220914_200343_841_0101120.fits' in validffs + + def test_readIniFile(): inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini')) assert inifs['LOCATION']=='testpi4' diff --git a/ukmonInstaller.py b/ukmonInstaller.py index a5ece8f..1ca0508 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -6,7 +6,12 @@ from subprocess import call import time -import paramiko +import warnings +from cryptography.utils import CryptographyDeprecationWarning +with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=CryptographyDeprecationWarning) + import paramiko + import json import tempfile import RMS.ConfigReader as cr @@ -45,6 +50,8 @@ def createDefaultIni(homedir, helperip='3.11.55.160', location='NOTCONFIGURED', outf.write("export UKMONHELPER={}\n".format(helperip)) outf.write("export UKMONKEY={}\n".format(keyfile)) outf.write("export RMSCFG={}\n".format(rmscfg)) + outf.write('export DOMP4s=1\n') + outf.write('export MINMAG=1\n') return True @@ -76,9 +83,27 @@ def validateIni(homedir, newhelperip=None): createDefaultIni(homedir, newhelperip, location, keyfile, rmscfg) if helperip == oldip: updateHelperIp(homedir, newhelperip) + updateMp4andMag(inifname, homedir) return True +def updateMp4andMag(inif, homedir): + """ + Move the mp4 flag into the ini file and add the minmag flag if missing + """ + domp4s = 0 + if open(inif, 'r').read()[-1] != '\n': + open(inif, 'a').write('\n') + if os.path.isfile(os.path.join(homedir, 'domp4s')): + domp4s = 1 + os.remove(os.path.join(homedir, 'domp4s')) + if 'DOMP4S' not in open(inif).read(): + open(inif,'a').write('export DOMP4S={}\n'.format(domp4s)) + if 'MAGLIM' not in open(inif).read(): + open(inif,'a').write('export MAGLIM=1\n') + return + + def updateHelperIp(homedir, helperip): """ Update the ukmon.ini file with a new IP address if neeeded. @@ -91,6 +116,7 @@ def updateHelperIp(homedir, helperip): outf.write("export UKMONHELPER={}\n".format(helperip)) else: outf.write('{}'.format(li)) + return def updateLocation(homedir, newloc): diff --git a/ukmonPostProc.py b/ukmonPostProc.py index 2939384..0291f93 100644 --- a/ukmonPostProc.py +++ b/ukmonPostProc.py @@ -58,8 +58,14 @@ def rmsExternal(cap_dir, arch_dir, config): bff2i.batchFFtoImage(arch_dir, 'jpg') myloc = os.path.split(os.path.abspath(__file__))[0] + inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) log.info('app home is {}'.format(myloc)) - if os.path.isfile(os.path.join(myloc, 'domp4s')): + domp4s = 0 + if 'DOMP4S' in inifvals: + domp4s = int(inifvals['DOMP4S']) + elif os.path.isfile(os.path.join(myloc, 'domp4s')): + domp4s = 1 + if domp4s == 1: # generate MP4s of detections log.info('generating MP4s') ftpdate='' @@ -68,7 +74,13 @@ def rmsExternal(cap_dir, arch_dir, config): else: ftpdate=os.path.split(arch_dir)[1] ftpfile_name="FTPdetectinfo_"+ftpdate+'.txt' - gmp4.generateMP4s(arch_dir, ftpfile_name) + try: + maglim = 1 + if 'MAGLIM' in inifvals: + maglim = float(inifvals['MAGLIM']) + gmp4.generateMP4s(arch_dir, ftpfile_name, min_mag=maglim) + except Exception: + gmp4.generateMP4s(arch_dir, ftpfile_name) else: log.info('mp4 creation not enabled') diff --git a/uploadToArchive.py b/uploadToArchive.py index c617ed5..9e66f5b 100644 --- a/uploadToArchive.py +++ b/uploadToArchive.py @@ -17,8 +17,13 @@ import glob import logging from time import sleep -import paramiko +import warnings +from cryptography.utils import CryptographyDeprecationWarning +with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=CryptographyDeprecationWarning) + import paramiko import tempfile +from RMS.Formats.FTPdetectinfo import readFTPdetectinfo log = logging.getLogger("logger") @@ -254,6 +259,30 @@ def uploadOneFileUKMon(arch_dir, dir_file, s3, targf, file_ext, keys): return ret +def checkMags(dir_path, ftpfile_name, min_mag): + log.info('checking for events brighter than mag {}'.format(min_mag)) + ff_names = [] + try: + meteor_list = readFTPdetectinfo(dir_path, ftpfile_name) + except Exception: + log.info('FTPdetect file not present so unable to filter by magnitude') + return ff_names + for meteor in meteor_list: + ff_name, _, meteor_no, n_segments, _, _, _, _, _, _, _, \ + meteor_meas = meteor + # checks on mag and shower + best_mag = 999 + if min_mag is not None: + for meas in meteor_meas: + best_mag = min(best_mag, meas[9]) + if best_mag > min_mag: + log.info('rejecting {} as {} too dim'.format(ff_name, best_mag)) + continue + else: + ff_names.append(ff_name.replace('.fits', '.jpg')) + return ff_names + + def uploadToArchive(arch_dir, sciencefiles=False, keys=False): # Upload all relevant files from *arch_dir* to ukmon's S3 Archive @@ -269,12 +298,17 @@ def uploadToArchive(arch_dir, sciencefiles=False, keys=False): conn = boto3.Session(aws_access_key_id=keys['AWS_ACCESS_KEY_ID'], aws_secret_access_key=keys['AWS_SECRET_ACCESS_KEY']) s3 = conn.resource('s3', region_name=reg) targf = keys['S3FOLDER'] + maglim = 10.0 + if 'MAGLIM' in inifvals: + maglim = float(inifvals['MAGLIM']) # upload the files but make sure we do the platepars file before the FTP file # otherwise there's a risk the matching engine will miss it dir_contents = os.listdir(arch_dir) daydir = os.path.split(arch_dir)[1] + validffs = checkMags(arch_dir, 'FTPdetectinfo_{}.txt'.format(daydir), maglim) + uploadlist = [] if sciencefiles: # upload just the critical files @@ -295,10 +329,11 @@ def uploadToArchive(arch_dir, sciencefiles=False, keys=False): continue # mp4 must be uploaded before corresponding jpg elif (file_ext == '.jpg') and ('FF_' in file_name): - mp4f = dir_file.replace('.jpg', '.mp4') - if os.path.isfile(os.path.join(arch_dir, mp4f)): - uploadlist.append({'dir_file':mp4f, 'file_ext': '.mp4', 'src_dir': arch_dir}) - uploadlist.append({'dir_file':dir_file, 'file_ext': file_ext, 'src_dir': arch_dir}) + if dir_file in validffs or validffs == []: + mp4f = dir_file.replace('.jpg', '.mp4') + if os.path.isfile(os.path.join(arch_dir, mp4f)): + uploadlist.append({'dir_file':mp4f, 'file_ext': '.mp4', 'src_dir': arch_dir}) + uploadlist.append({'dir_file':dir_file, 'file_ext': file_ext, 'src_dir': arch_dir}) elif (file_ext == '.jpg') and ('stack_' in file_name) and ('track' not in file_name): uploadlist.append({'dir_file':dir_file, 'file_ext': file_ext, 'src_dir': arch_dir}) elif (file_ext == '.jpg') and ('calib' in file_name): @@ -379,12 +414,17 @@ def manualUpload(targ_dir, sciencefiles=False): return True else: arch_dir = os.path.expanduser(targ_dir) + if not os.path.isdir(arch_dir): + print('folder {} not found'.format(arch_dir)) + return False return uploadToArchive(arch_dir, sciencefiles, keys=None) if __name__ == '__main__': + if len(sys.argv) < 2: + print('usage: python uploadToArchive.py ~/RMS_data/ArchivedFiles/dated_dir') + exit(0) targdir = sys.argv[1] - if len(sys.argv) > 2: - manualUpload(sys.argv[1], True) - else: - manualUpload(sys.argv[1]) + manualUpload(targdir, True) + if targdir != 'test': + manualUpload(targdir)