diff --git a/demo/FKt230303_S0492_eventTemplates.json b/demo/FKt230303_S0492_eventTemplates.json index ea2537d..f5e798e 100644 --- a/demo/FKt230303_S0492_eventTemplates.json +++ b/demo/FKt230303_S0492_eventTemplates.json @@ -385,7 +385,15 @@ "Deployed" ], "event_option_required": true, - "event_option_allow_freeform": false + "event_option_allow_freeform": false, + "event_option_visibility": { + "show_hide": "show if", + "event_option_name": "Type", + "event_option_values": [ + "Push Core", + "Water Sample" + ] + } }, { "event_option_name": "Sample Description", diff --git a/lib/utils.js b/lib/utils.js index 4c2a7af..d812183 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -376,6 +376,12 @@ const hashedPassword = (password) => { }; +const hashedApiKey = (apiKey) => { + + return hashSync( apiKey, saltRounds ); + +}; + const mvFilesToDir = (sourcePath, destPath, createIfMissing = false) => { try { @@ -484,6 +490,7 @@ module.exports = { filePreProcessor, flattenEventObjs, hashedPassword, + hashedApiKey, mvFilesToDir, randomAsciiString, randomString, diff --git a/lib/validations.js b/lib/validations.js index 285cbd0..746c420 100644 --- a/lib/validations.js +++ b/lib/validations.js @@ -1,6 +1,7 @@ const Joi = require('joi'); const { + roles, useAccessControl } = require('../config/server_settings'); @@ -360,6 +361,12 @@ const eventTemplateQuery = Joi.object({ sort: Joi.string().valid('event_name').optional() }).optional().label('eventTemplateQuery'); +const eventTemplateVisibility = Joi.object({ + show_hide: Joi.string().required(), + event_option_name: Joi.string().required(), + event_option_values: Joi.array().items(Joi.string()).required() +}); + const eventTemplateResponse = Joi.object({ id: Joi.object(), event_name: Joi.string(), @@ -376,7 +383,8 @@ const eventTemplateResponse = Joi.object({ event_option_default_value: Joi.string().allow(''), event_option_values: Joi.array().items(Joi.string()), event_option_allow_freeform: Joi.boolean(), - event_option_required: Joi.boolean() + event_option_required: Joi.boolean(), + event_option_visibility: eventTemplateVisibility.optional() })) }).label('eventTemplateResponse'); @@ -396,7 +404,8 @@ const eventTemplateCreatePayload = Joi.object({ event_option_default_value: Joi.string().allow('').optional(), event_option_values: Joi.array().items(Joi.string()).required(), event_option_allow_freeform: Joi.boolean().required(), - event_option_required: Joi.boolean().required() + event_option_required: Joi.boolean().required(), + event_option_visibility: eventTemplateVisibility.optional() })).optional() }).label('eventTemplateCreatePayload'); @@ -415,7 +424,8 @@ const eventTemplateUpdatePayload = Joi.object({ event_option_default_value: Joi.string().allow('').optional(), event_option_values: Joi.array().items(Joi.string()).required(), event_option_allow_freeform: Joi.boolean().required(), - event_option_required: Joi.boolean().required() + event_option_required: Joi.boolean().required(), + event_option_visibility: eventTemplateVisibility.optional() })).optional() }).required().min(1).label('eventTemplateUpdatePayload'); @@ -626,7 +636,7 @@ const userCreatePayload = Joi.object({ fullname: Joi.string().min(1).max(100).required(), email: Joi.string().email().required(), password: Joi.string().allow('').max(50).required(), - roles: Joi.array().items(Joi.string()).min(1).required(), + roles: Joi.array().items(Joi.string().valid(...roles)).min(1).required(), system_user: Joi.boolean().optional(), disabled: Joi.boolean().optional(), resetURL: Joi.string().uri().required() @@ -637,7 +647,7 @@ const userUpdatePayload = Joi.object({ fullname: Joi.string().min(1).max(100).optional(), // email: Joi.string().email().optional(), password: Joi.string().allow('').max(50).optional(), - roles: Joi.array().items(Joi.string()).min(1).optional(), + roles: Joi.array().items(Joi.string().valid(...roles)).min(1).optional(), system_user: Joi.boolean().optional(), disabled: Joi.boolean().optional() }).required().min(1).label('userUpdatePayload'); @@ -649,7 +659,7 @@ const userResponse = Joi.object({ last_login: Joi.date(), username: Joi.string(), fullname: Joi.string(), - roles: Joi.array().items(Joi.string()), + roles: Joi.array().items(Joi.string().valid(...roles)), disabled: Joi.boolean() }).label('userResponse'); @@ -658,6 +668,46 @@ const userSuccessResponse = Joi.alternatives().try( Joi.array().items(userResponse) ).label('userSuccessResponse'); + +// api keys +// ---------------------------------------------------------------------------- +const apiKeyParam = Joi.object({ + id: Joi.string().length(24).required() +}).label('apiKeyParam'); + +const apiKeyQuery = Joi.object({ + id: Joi.string().length(24).optional(), + label: Joi.string().max(100).optional() +}).label('apiKeyQuery'); + +const apiKeyCreatePayload = Joi.object({ + label: Joi.string().max(100).required(), + scope: Joi.array().items(Joi.string()).optional(), + expires: Joi.date().optional() +}).label('apiKeyCreatePayload'); + +const apiKeyResponse = Joi.object({ + id: Joi.object(), + user_id: Joi.object(), + label: Joi.string(), + scope: Joi.array().items(Joi.string()), + created: Joi.date(), + last_used: Joi.date().allow(null), + disabled: Joi.boolean(), + expires: Joi.date().allow(null) +}).label('apiKeyResponse'); + +const apiKeySuccessResponse = Joi.alternatives().try( + apiKeyResponse, + Joi.array().items(apiKeyResponse) +).label('apiKeySuccessResponse'); + +const apiKeyUpdatePayload = Joi.object({ + label: Joi.string().max(100).optional(), + scope: Joi.array().items(Joi.string()).optional(), + expires: Joi.date().optional() +}).label('apiKeyUpdatePayload'); + module.exports = { authorizationHeader, autoLoginPayload, @@ -733,5 +783,10 @@ module.exports = { userQuery, userSuccessResponse, userToken, - userUpdatePayload + userUpdatePayload, + apiKeyCreatePayload, + apiKeyParam, + apiKeyQuery, + apiKeySuccessResponse, + apiKeyUpdatePayload }; diff --git a/misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py b/misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py new file mode 100644 index 0000000..41b7113 --- /dev/null +++ b/misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +''' +FILE: base_aux_data_file_cleaner.py + +DESCRIPTION: Base class for cleaning up any changes made by aux_data_inserters outside + of the aux data mongo collection. + +BUGS: +NOTES: +AUTHOR: Lindsey Jones +COMPANY: OET +VERSION: 1.0 +CREATED: 2026-02-20 +REVISION: + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' +import logging + +from abc import ABC, abstractmethod + + +class AuxDataFileCleaner(ABC): + ''' + Abstract base class for building sealog aux_data records from various data sources. + Handles common functionality for querying external data sources and building + aux_data records with transformations. + ''' + + def __init__(self, aux_data_config): + ''' + Initialize the base builder with configuration. + ''' + self._data_source = aux_data_config['data_source'] + self.logger = logging.getLogger(__name__) + + def _get_aux_data_for_source(self, event): + ''' + Get the aux data record for the given event based on the data source. + + Args: + event (dict): Event dictionary containing 'id' and 'ts' keys + + Returns: + str or None: ID of cleaned aux data record, if there is one + ''' + all_aux_data = event.get('aux_data', {}) + if not all_aux_data: + self.logger.debug("No aux data found for event %s", event['id']) + return None + + # Find the first aux_data record that matches the data source + # There should only ever be one record per data source + aux_data_for_source = next( + (aux_data for aux_data in all_aux_data + if aux_data["data_source"] == self._data_source), + None # default if not found + ) + + if not aux_data_for_source: + self.logger.debug("No %s aux data found for event %s", self._data_source, event['id']) + + return aux_data_for_source + + @abstractmethod + def open_connections(self): + ''' + Open any necessary connections to external data sources. + Must be implemented by subclasses. + ''' + + @abstractmethod + def close_connections(self): + ''' + Close any open connections to external data sources. + Must be implemented by subclasses. + ''' + + @abstractmethod + def clean_aux_data_record(self, event, dry_run): + ''' + Do any clean up required for the given event + Must be implemented by subclasses. + + Args: + event (dict): Event dictionary containing 'id' and 'ts' keys + dry_run (bool): If True, do not actually delete any aux data records + + Returns: + dict or None: Aux data record or None if no data available + ''' + + @property + def data_source(self): + ''' + Getter method for the data_source property + ''' + return self._data_source diff --git a/misc/aux_data_file_cleaners/delete_files_aux_data_file_cleaner.py b/misc/aux_data_file_cleaners/delete_files_aux_data_file_cleaner.py new file mode 100644 index 0000000..e4a2279 --- /dev/null +++ b/misc/aux_data_file_cleaners/delete_files_aux_data_file_cleaner.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +''' +FILE: delete_files_aux_data_file_cleaner.py + +DESCRIPTION: Cleans up the files created by the aux data creation process. + +BUGS: +NOTES: +AUTHOR: Lindsey Jones +COMPANY: OET +VERSION: 1.0 +CREATED: 2026-02-20 +REVISION: + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' +import os +import sys + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(dirname(realpath(__file__))))) + +from misc.aux_data_file_cleaners.base_aux_data_file_cleaner import AuxDataFileCleaner + + +class DeleteFilesAuxDataFileCleaner(AuxDataFileCleaner): + ''' + Cleans up the files created by the stillcap_ffmpeg aux data creation process + ''' + + def open_connections(self): + ''' + No connections to open. + ''' + + def close_connections(self): + ''' + No connections to close. + ''' + + def clean_aux_data_record(self, event, dry_run): + ''' + Do any clean up required for the given event + + Args: + event (dict): Event dictionary containing 'id' and 'ts' keys + dry_run (bool): If True, do not actually delete any aux data records + + Returns: + str or None: ID of cleaned aux data record, if there is one + ''' + aux_data = self._get_aux_data_for_source(event) + + if not aux_data: + return None + + file_paths = [ + data["data_value"] + for data in aux_data["data_array"] + if data["data_name"] == "filename" + ] + for file_path in file_paths: + try: + self.logger.debug("Deleting file %s for event %s", file_path, event['id']) + if not dry_run: + os.remove(file_path) + except Exception as e: # pylint: disable=W0718 + self.logger.error( + "Error deleting file %s for event %s: %s", + file_path, + event['id'], + e + ) + + return aux_data["_id"] diff --git a/misc/aux_data_file_cleaners/do_nothing_aux_data_file_cleaner.py b/misc/aux_data_file_cleaners/do_nothing_aux_data_file_cleaner.py new file mode 100644 index 0000000..cc36aa5 --- /dev/null +++ b/misc/aux_data_file_cleaners/do_nothing_aux_data_file_cleaner.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +''' +FILE: do_nothing_aux_data_file_cleaner.py + +DESCRIPTION: Aux data file cleaner for aux data inserters that don't make any external changes. + +BUGS: +NOTES: +AUTHOR: Lindsey Jones +COMPANY: OET +VERSION: 1.0 +CREATED: 2026-02-20 +REVISION: + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' +import sys + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(dirname(realpath(__file__))))) + +from misc.aux_data_file_cleaners.base_aux_data_file_cleaner import AuxDataFileCleaner + + +class DoNothingAuxDataFileCleaner(AuxDataFileCleaner): + ''' + Aux data file cleaner for aux data inserters that don't make any external changes. + ''' + + def open_connections(self): + ''' + No connections to open. + ''' + + def close_connections(self): + ''' + No connections to close. + ''' + + def clean_aux_data_record(self, event, dry_run): # pylint: disable=W0613 + ''' + Do any clean up required for the given event + + Args: + event (dict): Event dictionary containing 'id' and 'ts' keys + dry_run (bool): If True, do not actually delete any aux data records + + Returns: + str or None: ID of cleaned aux data record, if there is one + ''' + aux_data = self._get_aux_data_for_source(event) + + if aux_data: + self.logger.debug( + "No additional clean up required for event %s %s aux data records", + event['id'], + self._data_source + ) + return aux_data["_id"] + + return None diff --git a/misc/aux_data_file_cleaners/stillcapffmpeg_aux_data_file_cleaner.py b/misc/aux_data_file_cleaners/stillcapffmpeg_aux_data_file_cleaner.py new file mode 100644 index 0000000..d545ba5 --- /dev/null +++ b/misc/aux_data_file_cleaners/stillcapffmpeg_aux_data_file_cleaner.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +''' +FILE: stillcapffmpeg_aux_data_file_cleaner.py + +DESCRIPTION: Cleans up the files created by the stillcap_ffmpeg aux data creation process. + +BUGS: +NOTES: +AUTHOR: Lindsey Jones +COMPANY: OET +VERSION: 1.0 +CREATED: 2026-02-20 +REVISION: + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' +import os +import sys + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(dirname(realpath(__file__))))) + +from misc.aux_data_file_cleaners.base_aux_data_file_cleaner import AuxDataFileCleaner + + +class StillcapFFMPEGAuxDataFileCleaner(AuxDataFileCleaner): + ''' + Cleans up the files created by the stillcap_ffmpeg aux data creation process + ''' + + def open_connections(self): + ''' + No connections to open. + ''' + + def close_connections(self): + ''' + No connections to close. + ''' + + def clean_aux_data_record(self, event, dry_run): + ''' + Do any clean up required for the given event + + Args: + event (dict): Event dictionary containing 'id' and 'ts' keys + dry_run (bool): If True, do not actually delete any aux data records + + Returns: + str or None: ID of cleaned aux data record, if there is one + ''' + aux_data = self._get_aux_data_for_source(event) + + if aux_data: + file_paths = [ + data["data_value"] + for data in aux_data["data_array"] + if data["data_name"] == "filename" + ] + for file_path in file_paths: + try: + self.logger.debug("Deleting file %s for event %s", file_path, event['id']) + if not dry_run: + os.remove(file_path) + except Exception as e: # pylint: disable=W0718 + self.logger.error( + "Error deleting file %s for event %s: %s", + file_path, + event['id'], + e + ) + + try: + sym_path = file_path.replace('png', 'symlink') + self.logger.debug("Deleting symlink %s for event %s", sym_path, event['id']) + if not dry_run: + os.remove(sym_path) + except Exception as e: # pylint: disable=W0718 + self.logger.error( + "Error deleting symlink %s for event %s: %s", + sym_path, + event['id'], + e + ) + return aux_data["_id"] + + return None diff --git a/misc/aux_data_manager_runner.py b/misc/aux_data_manager_runner.py new file mode 100644 index 0000000..191e814 --- /dev/null +++ b/misc/aux_data_manager_runner.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python3 +''' +FILE: sealog_aux_data_inserter_runner.py + +DESCRIPTION: Shared runner for aux-data inserter scripts. +''' +import argparse +import asyncio +import json +import logging +import os +import re +import sys +import time +import websockets +import yaml + +from typing import Callable, Dict, Any + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(realpath(__file__)))) + +from misc.aux_data_file_cleaners.base_aux_data_file_cleaner import AuxDataFileCleaner +from misc.base_aux_data_record_builder import AuxDataRecordBuilder +from misc.python_sealog.cruises import get_cruise_uid_by_id +from misc.python_sealog.events import get_event, get_events_by_cruise, get_events_by_lowering +from misc.python_sealog.event_aux_data import create_event_aux_data, delete_event_aux_data +from misc.python_sealog.lowerings import get_lowering_uid_by_id + +LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} +LOGGING_FORMAT = '%(asctime)-15s %(levelname)s %(lineno)s - %(message)s' + + +def parse_event_ids(event_id_file): + ''' + Builds list of event uid from csv-formatted file. + ''' + event_ids_from_file = [] + with open(event_id_file, 'r', encoding='utf-8') as event_id_fp: + for line in event_id_fp: + line = line.rstrip('\n') + logging.debug(line) + event_ids_from_file += line.split(',') + + event_ids_from_file = [event_id.strip() for event_id in event_ids_from_file] + + for event_id in event_ids_from_file: + if re.match(r"^[a-f\d]{24}$", event_id) is None: + logging.error("\"%s\" is an invalid event_id... quiting", event_id) + raise ValueError(f'"{event_id}" is an invalid event_id... quiting') + + return event_ids_from_file + + +def delete_aux_data(aux_data_cleaners, event, dry_run=False): + ''' + Delete aux_data records for the specified event and any impacts they may have had + ''' + for cleaner in aux_data_cleaners: + logging.debug("Cleaning aux data record") + # NOTE: I think this code would be more single-responsibility if we got the aux data + # and then passed it into the cleaner, but I wanted to match the pattern of the builders + aux_data_id = cleaner.clean_aux_data_record(event, dry_run) + if aux_data_id: + try: + logging.debug("Deleting aux data files from Sealog Server") + logging.debug(json.dumps(aux_data_id)) + if not dry_run: + delete_event_aux_data(aux_data_id) + + except Exception as exc: # pylint:disable=W0718 + logging.warning("Error deleting aux data record") + logging.debug(str(exc)) + else: + logging.debug("No aux data for data_source: %s", cleaner.data_source) + + +def insert_aux_data(aux_data_builders, event, dry_run=False): + ''' + Add aux_data records for only the specified event + ''' + for builder in aux_data_builders: + logging.debug("Building aux data record") + record = builder.build_aux_data_record(event) + if record: + try: + logging.debug("Submitting aux data record to Sealog Server") + logging.debug(json.dumps(record)) + if not dry_run: + create_event_aux_data(record) + + except Exception as exc: # pylint:disable=W0718 + logging.warning("Error submitting aux data record") + logging.debug(str(exc)) + else: + logging.debug("No aux data for data_source: %s", builder.data_source) + + +def insert_aux_data_from_list(aux_data_builders, event_ids_from_list, dry_run=False): + ''' + Add aux_data records for only the events in the specified list + ''' + for event_id in event_ids_from_list: + try: + logging.debug("Retrieving event record from Sealog Server") + event = get_event(event_id) + logging.debug("Event: %s", event) + + except Exception as exc: + logging.warning("Error submitting aux data record") + logging.debug(str(exc)) + raise exc + + insert_aux_data(aux_data_builders, event, dry_run) + + +def insert_aux_data_for_cruise(aux_data_builders, cruise_id, dry_run=False): + ''' + Add aux_data records for only the events in the specified cruise + ''' + cruise_uid = get_cruise_uid_by_id(cruise_id) + + # exit if no cruise found + if not cruise_uid: + logging.error("cruise not found") + return + + # retrieve events for cruise + cruise_events = get_events_by_cruise(cruise_uid) + + # exit if no cruise found + if not cruise_events: + logging.error("no events found for cruise") + return + + for event in cruise_events: + insert_aux_data(aux_data_builders, event, dry_run) + + +def insert_aux_data_for_lowering(aux_data_builders, lowering_id, dry_run=False): + ''' + Add aux_data records for only the events in the specified lowering + ''' + lowering_uid = get_lowering_uid_by_id(lowering_id) + + # exit if no lowering found + if not lowering_uid: + logging.error("lowering not found") + return + + # retrieve events for lowering + lowering_events = get_events_by_lowering(lowering_uid) + + # exit if no lowering found + if not lowering_events: + logging.error("no events found for lowering") + return + + for event in lowering_events: + insert_aux_data(aux_data_builders, event, dry_run) + + +async def manage_aux_data_from_ws(aux_data_builders, + aux_data_cleaners, + ws_server_url, + headers, + client_wsid, + exclude_set, + dry_run): # pylint: disable=R0914 + ''' + Use the aux_data_builder and to submit aux_data + records built from external data to the sealog-server API + ''' + async with websockets.connect(ws_server_url) as websocket: + + subscription_message = { + 'type': 'hello', + 'id': client_wsid, + 'auth': {'headers': headers}, + 'version': '2', + 'subs': ['/ws/status/newEvents', + '/ws/status/deleteEvents'] + } + # updateEvents is only published when the timestamp is unchanged + # no time change-->no need to get aux data for a different time-->no need to subscribe + # if you wanted to, say, add a new type of aux data, you wouldn't be using the ws pathway + await websocket.send(json.dumps(subscription_message)) + + # Precompute PING JSON to avoid reconstructing on every ping + ping_message = json.dumps({'type': 'ping', 'id': client_wsid}) + + while True: + event = await websocket.recv() + event_obj = json.loads(event) + event_type = event_obj.get('type') + + if event_type: + if event_type == 'ping': + await websocket.send(ping_message) + + elif event_type == 'pub': + message = event_obj.get('message') + if message.get('event_value') in exclude_set: + logging.debug("Skipping because event value is in the exclude set") + else: + logging.debug("Event: %s", message) + pub_type = event_obj.get('topic', '').split('/')[-1] + match pub_type: + case 'newEvents': + insert_aux_data(aux_data_builders, message, dry_run) + case 'deleteEvents': + delete_aux_data(aux_data_cleaners, message, dry_run) + case _: + logging.warning("Unhandled pub type: %s", pub_type) + else: + logging.warning("Malformed event received: %s", event_obj) + + +def parse_aux_data_args(): + ''' + Parse command line arguments for aux data manager scripts. + ''' + parser = argparse.ArgumentParser(description='Aux Data Manager Service (shared)') + parser.add_argument('-v', '--verbosity', dest='verbosity', default=0, action='count', + help='Increase output verbosity') + parser.add_argument('-f', '--config_file', help='use the specifed configuration file') + parser.add_argument('-n', '--dry_run', action='store_true', + help='compile the aux_data records but do not submit to server API') + parser.add_argument('-e', '--events', help='list of event_ids to apply aux data') + parser.add_argument('-c', '--cruise_id', help='cruise_id to fix aux_data for') + parser.add_argument('-l', '--lowering_id', help='lowering_id to fix aux_data for') + + parsed_args = parser.parse_args() + parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS)) + return parsed_args + + +def get_aux_data_configs(inline_config, config_file): + ''' + Get aux data configuration from file or inline config. + ''' + if config_file: + config_file, data_source = ( + config_file.split(":") + if ":" in config_file + else [config_file, None] + ) + + try: + with open(config_file, 'r', encoding='utf-8') as config_fp: + aux_data_configs = yaml.safe_load(config_fp) + if data_source: + aux_data_configs = [ + c + for c in aux_data_configs + if c['data_source'] == data_source] + except yaml.parser.ParserError: + logging.error("Invalid YAML syntax") + sys.exit(1) + else: + try: + aux_data_configs = yaml.safe_load(inline_config) + except yaml.parser.ParserError: + logging.error("Invalid YAML syntax") + sys.exit(1) + + logging.debug(json.dumps(aux_data_configs, indent=2)) + return aux_data_configs + + +def run_aux_data_manager( + builder_factory: Callable[[Dict[str, Any]], AuxDataRecordBuilder], + cleaner_factory: Callable[[Dict[str, Any]], AuxDataFileCleaner], + inline_config: str, + ws_server_url: str, + headers: Dict[str, Any], + client_wsid: str, + exclude_set: set = () +): + """ + Shared CLI and main loop. + + builder_factory: callable(config_dict) -> builder instance + inline_config: YAML string used if -f not provided + ws_server_url, headers, client_wsid: for websocket connection. + """ + parsed_args = parse_aux_data_args() + + # Logging setup + logging.basicConfig(format=LOGGING_FORMAT) + logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity]) + + aux_data_configs = get_aux_data_configs(inline_config, parsed_args.config_file) + + # Build builders and cleaners + aux_data_builder_list = [builder_factory(cfg) for cfg in aux_data_configs] + aux_data_cleaner_list = [cleaner_factory(cfg) for cfg in aux_data_configs] + + # Dispatch actions requested on CLI + if parsed_args.events: + event_ids = parse_event_ids(parsed_args.events) + logging.info("Event IDs:\n%s", json.dumps(event_ids, indent=2)) + insert_aux_data_from_list(aux_data_builder_list, event_ids, parsed_args.dry_run) + sys.exit(0) + + if parsed_args.cruise_id: + insert_aux_data_for_cruise( + aux_data_builder_list, + parsed_args.cruise_id, + parsed_args.dry_run) + sys.exit(0) + + if parsed_args.lowering_id: + insert_aux_data_for_lowering( + aux_data_builder_list, + parsed_args.lowering_id, + parsed_args.dry_run) + sys.exit(0) + + # Wait then start WS loop forever. Sleep and retry on any disconnect/error + while True: + time.sleep(5) + try: + for aux_data_builder in aux_data_builder_list: + aux_data_builder.open_connections() + for aux_data_cleaner in aux_data_cleaner_list: + aux_data_cleaner.open_connections() + logging.debug("Connecting to event websocket feed...") + asyncio.get_event_loop().run_until_complete( + manage_aux_data_from_ws( + aux_data_builder_list, + aux_data_cleaner_list, + ws_server_url, + headers, + client_wsid, + exclude_set, + parsed_args.dry_run) + ) + except KeyboardInterrupt: + logging.error('Keyboard Interrupted') + try: + sys.exit(0) + except SystemExit: + try: + for aux_data_builder in aux_data_builder_list: + aux_data_builder.close_connections() + for aux_data_cleaner in aux_data_cleaner_list: + aux_data_cleaner.close_connections() + except Exception as err: # pylint: disable=W0718 + logging.debug(str(err)) + os._exit(0) # pylint: disable=protected-access + except Exception as err: # pylint: disable=W0718 + logging.debug(str(err)) + logging.error("Lost connection to server, trying again in 5 seconds") + finally: + for aux_data_builder in aux_data_builder_list: + aux_data_builder.close_connections() + for aux_data_cleaner in aux_data_cleaner_list: + aux_data_cleaner.close_connections() diff --git a/misc/base_aux_data_record_builder.py b/misc/base_aux_data_record_builder.py new file mode 100644 index 0000000..29cef96 --- /dev/null +++ b/misc/base_aux_data_record_builder.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +''' +FILE: base_aux_data_record_builder.py + +DESCRIPTION: Base class for building sealog aux_data records from various data sources. + +BUGS: +NOTES: +AUTHOR: Webb Pinner +COMPANY: OceanDataTools.org +VERSION: 1.0 +CREATED: 2025-02-25 +REVISION: + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' +import json +import logging +from abc import ABC, abstractmethod + + +class AuxDataRecordBuilder(ABC): + ''' + Abstract base class for building sealog aux_data records from various data sources. + Handles common functionality for querying external data sources and building + aux_data records with transformations. + ''' + + def __init__(self, aux_data_config): + ''' + Initialize the base builder with configuration. + + Args: + aux_data_config (dict): Configuration dictionary containing: + - query_measurements: List of measurements to query + - aux_record_lookup: Mapping of fields to output configuration + - data_source: Name of the data source + ''' + self._data_source = aux_data_config['data_source'] + # don't use get for data source--want to throw an error if not specified + self._query_measurements = aux_data_config.get('query_measurements') + self._aux_record_lookup = aux_data_config.get('aux_record_lookup') + if self._aux_record_lookup is None: + self._query_fields = [] + else: + self._query_fields = list(self._aux_record_lookup.keys()) + self.logger = logging.getLogger(__name__) + + @abstractmethod + def open_connections(self): + ''' + Open any necessary connections to external data sources. + Must be implemented by subclasses. + ''' + + @abstractmethod + def close_connections(self): + ''' + Close any open connections to external data sources. + Must be implemented by subclasses. + ''' + + @abstractmethod + def build_aux_data_record(self, event): + ''' + Build the aux_data record for the given event. + Must be implemented by subclasses. + + Args: + event (dict): Event dictionary containing 'id' and 'ts' keys + + Returns: + dict or None: Aux data record or None if no data available + ''' + + def _build_aux_data_dict(self, event_id, query_data): # pylint:disable=R0915 + ''' + Internal method to build the sealog aux_data record using the event_id, + query_data and the class instance's datasource value. + + This method handles common transformations and modifications across all data sources. + + Args: + event_id (str): The ID of the event + query_data (dict): Dictionary of field names to values from the data source + + Returns: + dict or None: Aux data record or None if no data available + ''' + aux_data_record = { + 'event_id': event_id, + 'data_source': self._data_source, + 'data_array': [] + } + + logging.debug("raw values: %s", json.dumps(query_data, indent=2)) + + if not query_data: + return None + + for key, value in self._aux_record_lookup.items(): + try: + if "no_output" in value and value['no_output'] is True: + continue + + if key not in query_data: + continue + + output_value = query_data[key] + + if "modify" in value: + logging.debug("modify found in record") + for mod_op in value['modify']: + test_result = True + + if 'test' in mod_op: + logging.debug("test found in mod_op") + test_result = False + + for test in mod_op['test']: + logging.debug(json.dumps(test)) + + if 'field' in test: + + if test['field'] not in query_data: + logging.error("test field data not in query results") + return None + + if 'eq' in test and query_data[test['field']] == test['eq']: + test_result = True + break + + if 'gt' in test and query_data[test['field']] > test['gt']: + test_result = True + break + + if 'gte' in test and query_data[test['field']] >= test['gt']: + test_result = True + break + + if 'lt' in test and query_data[test['field']] < test['lt']: + test_result = True + break + + if 'lte' in test and query_data[test['field']] <= test['lt']: + test_result = True + break + + if 'ne' in test and query_data[test['field']] != test['ne']: + test_result = True + break + + if test_result and 'operation' in mod_op: + logging.debug("operation found in mod_op") + for operan in mod_op['operation']: + + if 'add' in operan: + output_value += operan['add'] + + if 'subtract' in operan: + output_value -= operan['subtract'] + + if 'multiply' in operan: + output_value *= operan['multiply'] + + if 'divide' in operan: + output_value /= operan['divide'] + + aux_data_record['data_array'].append({ + 'data_name': value['name'], + 'data_value': ( + str(round(output_value, value['round'])) + if 'round' in value + else str(output_value) + ), + 'data_uom': value['uom'] if 'uom' in value else '' + }) + except ValueError as exc: + logging.warning("Problem adding %s", key) + logging.debug(str(exc)) + continue + + if len(aux_data_record['data_array']) > 0: + return aux_data_record + + return None + + @property + def data_source(self): + ''' + Getter method for the data_source property + ''' + return self._data_source + + @property + def measurements(self): + ''' + Getter method for the _query_measurements property + ''' + return self._query_measurements + + @property + def fields(self): + ''' + Getter method for the _query_fields property + ''' + return self._query_fields + + @property + def record_lookup(self): + ''' + Getter method for the _aux_record_lookup property + ''' + return self._aux_record_lookup diff --git a/misc/coriolix_sealog/aux_data_record_builder.py b/misc/coriolix_sealog/aux_data_record_builder.py index f3bfce2..815f14c 100644 --- a/misc/coriolix_sealog/aux_data_record_builder.py +++ b/misc/coriolix_sealog/aux_data_record_builder.py @@ -2,8 +2,8 @@ ''' FILE: aux_data_record_builder.py -DESCRIPTION: This script builds a sealog aux_data record with data pulled from an - influx database. +DESCRIPTION: This script builds a sealog aux_data record with data pulled from a + CORIOLIX API. BUGS: NOTES: @@ -16,11 +16,11 @@ LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) Copyright (C) OceanDataTools.org 2025 ''' -import os -import sys import json -import logging +import os import requests +import sys + from datetime import datetime, timedelta from urllib.parse import quote, urlparse from urllib3.exceptions import NewConnectionError @@ -28,28 +28,30 @@ from os.path import dirname, realpath sys.path.append(dirname(dirname(dirname(realpath(__file__))))) +from misc.base_aux_data_record_builder import AuxDataRecordBuilder from misc.coriolix_sealog.settings import CORIOLIX_URL -class SealogCORIOLIXAuxDataRecordBuilder(): +class SealogCORIOLIXAuxDataRecordBuilder(AuxDataRecordBuilder): ''' - Class that handles the construction of an influxDB query and using the + Class that handles the construction of CORIOLIX API queries and using the resulting data to build a sealog aux_data record. ''' def __init__(self, aux_data_config, url=None): + super().__init__(aux_data_config) self.url = url or CORIOLIX_URL - self._query_measurements = aux_data_config['query_measurements'] - self._query_fields = list(aux_data_config['aux_record_lookup'].keys()) - self._aux_record_lookup = aux_data_config['aux_record_lookup'] - self._data_source = aux_data_config['data_source'] - self.logger = logging.getLogger(__name__) - - @staticmethod - def _build_query_range(ts): + + def _build_query_range(self, ts): ''' - Builds the temporal range for the influxDB query based on the provided + Builds the temporal range for the CORIOLIX query based on the provided timestamp (ts). + + Args: + ts (str): Timestamp in ISO 8601 format (YYYY-MM-DDTHH:MM:SS.fffZ) + + Returns: + str or None: Query range string ''' try: start_ts = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ") - timedelta(minutes=1) @@ -58,12 +60,12 @@ def _build_query_range(ts): f'&date_before={quote(ts)}') except ValueError as exc: - logging.debug(str(exc)) + self.logger.debug(str(exc)) return None def _build_query_urls(self, ts): ''' - Builds the complete influxDB query using the provided timestamp (ts) + Builds the CORIOLIX API URLs using the provided timestamp (ts) and the class instance's query_measurements and query_fields values. ''' @@ -73,132 +75,44 @@ def _build_query_urls(self, ts): for measurement in self._query_measurements: query_urls.append(f'{self.url}/api/{measurement}/?format=json&{query_range}') - logging.debug("Query: %s", query_urls[-1]) + self.logger.debug("Query: %s", query_urls[-1]) return query_urls - def _build_aux_data_dict(self, event_id, query_results): # pylint:disable=R0915 + def open_connections(self): ''' - Internal method to build the sealog aux_data record using the event_id, - query_results and the class instance's datasource value. + Open any necessary connections to external data sources. + For CORIOLIX, no persistent connection is needed. ''' - aux_data_record = { - 'event_id': event_id, - 'data_source': self._data_source, - 'data_array': [] - } - - coriolix_data = query_results - - logging.debug("raw values: %s", json.dumps(coriolix_data, indent=2)) - - if not coriolix_data: - return None - - for key, value in self._aux_record_lookup.items(): - try: - if "no_output" in value and value['no_output'] is True: - continue - - if key not in coriolix_data: - continue - - output_value = coriolix_data[key] - - if "modify" in value: - logging.debug("modify found in record") - for mod_op in value['modify']: - test_result = True - - if 'test' in mod_op: - logging.debug("test found in mod_op") - test_result = False - - for test in mod_op['test']: - logging.debug(json.dumps(test)) - - if 'field' in test: - - if test['field'] not in coriolix_data: - logging.warning("test field data not in CORIOLIX query") - return None - - if 'eq' in test and coriolix_data[test['field']] == test['eq']: - test_result = True - break - - if 'gt' in test and coriolix_data[test['field']] > test['gt']: - test_result = True - break - - if 'gte' in test and coriolix_data[test['field']] >= test['gt']: - test_result = True - break - - if 'lt' in test and coriolix_data[test['field']] < test['lt']: - test_result = True - break - - if 'lte' in test and coriolix_data[test['field']] <= test['lt']: - test_result = True - break - - if 'ne' in test and coriolix_data[test['field']] != test['ne']: - test_result = True - break - - if test_result and 'operation' in mod_op: - logging.debug("operation found in mod_op") - for operan in mod_op['operation']: - - if 'add' in operan: - output_value += operan['add'] - - if 'subtract' in operan: - output_value -= operan['subtract'] - - if 'multiply' in operan: - output_value *= operan['multiply'] - - if 'divide' in operan: - output_value /= operan['divide'] - - aux_data_record['data_array'].append({ - 'data_name': value['name'], - 'data_value': str(round(output_value, value['round'])) if 'round' in value - else str(output_value), - 'data_uom': value['uom'] if 'uom' in value else '' - }) - except ValueError as exc: - logging.warning("Problem adding %s", key) - logging.debug(str(exc)) - continue - - if len(aux_data_record['data_array']) > 0: - return aux_data_record - - return None + def close_connections(self): + ''' + Close any open connections to external data sources. + For CORIOLIX, no persistent connection is needed. + ''' def build_aux_data_record(self, event): ''' Build the aux_data record for the given event. ''' - logging.debug("building query") + self.logger.debug("building query") query_urls = self._build_query_urls(event['ts']) query_results = {} for url in query_urls: - logging.debug("Query URL: %s", url) + self.logger.debug("Query URL: %s", url) measurement = os.path.basename(urlparse(url).path.strip('/')) - # run the query against the influxDB + # run the query against the CORIOLIX API try: response = requests.get(url, timeout=2) if response.status_code != 200: - logging.error("Failed to retrieve data. Status code: %s", response.status_code) + self.logger.error( + "Failed to retrieve data. Status code: %s", + response.status_code + ) response_obj = json.loads(response.text) if isinstance(response_obj, dict): @@ -213,41 +127,13 @@ def build_aux_data_record(self, event): } except NewConnectionError: - logging.error("CORIOLIX connection error, verify URL: %s", self.url) + self.logger.error("CORIOLIX connection error, verify URL: %s", self.url) except json.decoder.JSONDecodeError: - logging.error("Unable to decode response from URL: %s", url) - logging.debug(response) + self.logger.error("Unable to decode response from URL: %s", url) + self.logger.debug(response) except KeyError: - logging.error("Something went wrong processing the API response") + self.logger.error("Something went wrong processing the API response") aux_data_record = self._build_aux_data_dict(event['id'], query_results) return aux_data_record - - @property - def data_source(self): - ''' - Getter method for the data_source property - ''' - return self._data_source - - @property - def measurements(self): - ''' - Getter method for the _query_measurements property - ''' - return self._query_measurements - - @property - def fields(self): - ''' - Getter method for the _query_fields property - ''' - return self._query_fields - - @property - def record_lookup(self): - ''' - Getter method for the _aux_record_lookup property - ''' - return self._aux_record_lookup diff --git a/misc/delete_aux_data_records.py b/misc/delete_aux_data_records.py index d856ec6..41528df 100644 --- a/misc/delete_aux_data_records.py +++ b/misc/delete_aux_data_records.py @@ -28,6 +28,7 @@ def main(uid_file, dry_run): ''' Main function of script, read the file containing aux_data record ids and delete them. + This does not use the aux data managers, so it does not do any additional clean up ''' logging.info("Starting main function.") logging.info(dry_run) diff --git a/misc/framegrab_aux/aux_data_record_builder_framegrab_http.py b/misc/framegrab_aux/aux_data_record_builder_framegrab_http.py new file mode 100644 index 0000000..ef7b075 --- /dev/null +++ b/misc/framegrab_aux/aux_data_record_builder_framegrab_http.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +''' +FILE: aux_data_record_builder_framegrab_http.py + +DESCRIPTION: This script builds a sealog aux_data record by fetching frame grab images from HTTP source. +''' +import os +import requests +import shutil +import sys + +from datetime import datetime, timedelta +from os.path import dirname, realpath +sys.path.append(dirname(dirname(realpath(__file__)))) + +from misc.base_aux_data_record_builder import AuxDataRecordBuilder +from misc.framegrab_aux.settings import DEST_DIR, SOURCES, THRESHOLD + + +class FramegrabHTTPAuxDataRecordBuilder(AuxDataRecordBuilder): + ''' + Class that handles generating test images and using the + resulting data to build a sealog aux_data record. + ''' + + def open_connections(self): + ''' + Open any necessary connections to external data sources. + Must be implemented by subclasses. + ''' + + def close_connections(self): + ''' + Close any open connections to external data sources. + Must be implemented by subclasses. + ''' + + def _build_destination_filepath(self, str_timestamp, filename_prefix, filename_suffix): + timestamp = datetime.strptime( + str_timestamp, + '%Y-%m-%dT%H:%M:%S.%fZ' + ) + filename_date = datetime.date(timestamp) + filename_time = datetime.time(timestamp) + filename_middle = datetime.combine( + filename_date, filename_time + ).strftime("%Y%m%d_%H%M%S%f")[:-3] + + destination_filepath = os.path.join( + DEST_DIR, + filename_prefix + filename_middle + filename_suffix + ) + + return destination_filepath + + def build_aux_data_record(self, event): + ''' + Build the aux_data record for the given event. + ''' + if datetime.strptime( + event['ts'], + '%Y-%m-%dT%H:%M:%S.%fZ' + ) < datetime.utcnow()-timedelta(seconds=THRESHOLD): + self.logger.debug("Skipping because event ts is older than thresold") + return None + + aux_data_record = { + 'event_id': event['id'], + 'data_source': self._data_source, + 'data_array': [] + } + + for source in SOURCES: + + dst = self._build_destination_filepath( + event['ts'], + source['filename_prefix'], + source['filename_suffix'] + ) + + self.logger.debug("dst: %s", dst) + + try: + res = requests.get( + source['source_url'] + source['source_filename'], + stream=True, + timeout=(2, None) + ) + + if res.status_code != 200: + self.logger.error( + "Unable to retrieve image from: %s", + source['source_url'] + source['source_filename'] + ) + continue + + except requests.exceptions.RequestException as exc: + self.logger.error("Unable to retrieve image from remote server") + self.logger.error(exc) + + try: + with open(dst, 'wb') as f: + shutil.copyfileobj(res.raw, f) + + except shutil.Error as exc: + self.logger.error("Unable to save image to server") + self.logger.error(exc) + + aux_data_record['data_array'].append( + {'data_name': "camera_name", 'data_value': source['source_name']} + ) + aux_data_record['data_array'].append( + {'data_name': "filename", 'data_value': dst} + ) + + if len(aux_data_record['data_array']) > 0: + return aux_data_record + return None diff --git a/misc/framegrab_aux/aux_data_record_builder_framegrab_local.py b/misc/framegrab_aux/aux_data_record_builder_framegrab_local.py new file mode 100644 index 0000000..f3a0bcc --- /dev/null +++ b/misc/framegrab_aux/aux_data_record_builder_framegrab_local.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +''' +FILE: aux_data_record_builder_framegrab_local.py + +DESCRIPTION: This script builds a sealog aux_data record by copying frame grab images from local directory. +''' +import os +import shutil +import sys + +from datetime import datetime, timedelta +from os.path import dirname, realpath +sys.path.append(dirname(dirname(realpath(__file__)))) + +from misc.base_aux_data_record_builder import AuxDataRecordBuilder +from misc.framegrab_aux.settings import SOURCE_DIR, DEST_DIR, SOURCES, THRESHOLD + + +class FramegrabLocalAuxDataRecordBuilder(AuxDataRecordBuilder): + ''' + Class that handles generating test images and using the + resulting data to build a sealog aux_data record. + ''' + + def open_connections(self): + ''' + Open any necessary connections to external data sources. + Must be implemented by subclasses. + ''' + + def close_connections(self): + ''' + Close any open connections to external data sources. + Must be implemented by subclasses. + ''' + + def _build_destination_filepath(self, str_timestamp, filename_prefix, filename_suffix): + timestamp = datetime.strptime( + str_timestamp, + '%Y-%m-%dT%H:%M:%S.%fZ' + ) + filename_date = datetime.date(timestamp) + filename_time = datetime.time(timestamp) + filename_middle = datetime.combine( + filename_date, filename_time + ).strftime("%Y%m%d_%H%M%S%f")[:-3] + + destination_filepath = os.path.join( + DEST_DIR, + filename_prefix + filename_middle + filename_suffix + ) + + return destination_filepath + + def build_aux_data_record(self, event): + ''' + Build the aux_data record for the given event. + ''' + if datetime.strptime( + event['ts'], + '%Y-%m-%dT%H:%M:%S.%fZ' + ) < datetime.utcnow()-timedelta(seconds=THRESHOLD): + self.logger.debug("Skipping because event ts is older than thresold") + return None + + aux_data_record = { + 'event_id': event['id'], + 'data_source': self._data_source, + 'data_array': [] + } + + for source in SOURCES: + + dst = self._build_destination_filepath( + event['ts'], + source['filename_prefix'], + source['filename_suffix'] + ) + + self.logger.debug("dst: %s", dst) + + latest_file = os.path.join(SOURCE_DIR, source['source_filename']) + src = os.path.join(SOURCE_DIR, latest_file) + try: + shutil.copyfile(src, dst) + + except shutil.Error as exc: + self.logger.error("Unable to save image to server") + self.logger.error(exc) + + aux_data_record['data_array'].append( + {'data_name': "camera_name", 'data_value': source['source_name']} + ) + aux_data_record['data_array'].append( + {'data_name': "filename", 'data_value': dst} + ) + + if len(aux_data_record['data_array']) > 0: + return aux_data_record + return None diff --git a/misc/framegrab_aux/aux_data_record_builder_framegrab_scp.py b/misc/framegrab_aux/aux_data_record_builder_framegrab_scp.py new file mode 100644 index 0000000..e357437 --- /dev/null +++ b/misc/framegrab_aux/aux_data_record_builder_framegrab_scp.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +''' +FILE: aux_data_record_builder_framegrab_scp.py + +DESCRIPTION: This script builds a sealog aux_data record by transferring frame grab images via SCP. +''' +import os +import sys + +from datetime import datetime, timedelta + +try: + from paramiko import RSAKey, SFTPClient, Transport # noqa:F401 pylint:disable=W0611 + from paramiko.sftp import SFTPError # noqa:F401 pylint:disable=W0611 + PARAMIKO_ENABLED = True +except ImportError: + PARAMIKO_ENABLED = False + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(realpath(__file__)))) + +from misc.base_aux_data_record_builder import AuxDataRecordBuilder +from misc.framegrab_aux.settings import SOURCE_DIR, DEST_DIR, SOURCES, THRESHOLD, \ + HOST, USER, KEY_FILE, PORT + + +class FramegrabSCPAuxDataRecordBuilder(AuxDataRecordBuilder): + ''' + Class that handles generating test images and using the + resulting data to build a sealog aux_data record. + ''' + + def __init__(self, + aux_data_config): + super().__init__(aux_data_config) + if not PARAMIKO_ENABLED: + raise ModuleNotFoundError( + 'paramiko module is not installed. Try "pip3 install paramiko" prior to use.' + ) + self._host = HOST + self._user = USER + self._key = RSAKey.from_private_key_file(KEY_FILE) + self._port = PORT + + self._scp_transport = None + self._sftp_client = None + + def open_connections(self): + ''' + Open any necessary connections to external data sources. + Must be implemented by subclasses. + ''' + self._scp_transport = Transport(self._host, self._port) + self._scp_transport.connect(username=self._user, pkey=self._key) + self._scp_transport.set_keepalive(30) + # since opening connection higher up, set keepalive to prevent going stale + self.logger.info("Opening SFTP connection") + self._sftp_client = SFTPClient.from_transport(self._scp_transport) + + def close_connections(self): + ''' + Close any open connections to external data sources. + Must be implemented by subclasses. + ''' + try: + self._scp_transport.close() + except Exception as err: # pylint: disable=W0718 + self.logger.error("Error closing SCP Transport: %s", str(err)) + + def _build_destination_filepath(self, str_timestamp, filename_prefix, filename_suffix): + timestamp = datetime.strptime( + str_timestamp, + '%Y-%m-%dT%H:%M:%S.%fZ' + ) + filename_date = datetime.date(timestamp) + filename_time = datetime.time(timestamp) + filename_middle = datetime.combine( + filename_date, filename_time + ).strftime("%Y%m%d_%H%M%S%f")[:-3] + + destination_filepath = os.path.join( + DEST_DIR, + filename_prefix + filename_middle + filename_suffix + ) + + return destination_filepath + + def build_aux_data_record(self, event): + ''' + Build the aux_data record for the given event. + ''' + if datetime.strptime( + event['ts'], + '%Y-%m-%dT%H:%M:%S.%fZ' + ) < datetime.utcnow()-timedelta(seconds=THRESHOLD): + self.logger.debug("Skipping because event ts is older than thresold") + return None + + aux_data_record = { + 'event_id': event['id'], + 'data_source': self._data_source, + 'data_array': [] + } + + for source in SOURCES: + + dst = self._build_destination_filepath( + event['ts'], + source['filename_prefix'], + source['filename_suffix'] + ) + + self.logger.debug("dst: %s", dst) + + try: + latest_file = os.path.join(SOURCE_DIR, source['source_filename']) + src = os.path.join(SOURCE_DIR, latest_file) + self._sftp_client.put(src, dst) + except SFTPError as exc: + self.logger.error("Unable to copy image to server") + self.logger.error(exc) + except OSError as exc: + self.logger.error("Unable to copy image to server") + self.logger.error(exc) + + aux_data_record['data_array'].append( + {'data_name': "camera_name", 'data_value': source['source_name']} + ) + aux_data_record['data_array'].append( + {'data_name': "filename", 'data_value': dst} + ) + + if len(aux_data_record['data_array']) > 0: + return aux_data_record + return None diff --git a/misc/framegrab_aux/settings.py.dist b/misc/framegrab_aux/settings.py.dist new file mode 100644 index 0000000..df165be --- /dev/null +++ b/misc/framegrab_aux/settings.py.dist @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +''' +FILE: settings.py + +DESCRIPTION: This file contains the settings used by the influx_sealog + functions to communicate with the influxDB API + +BUGS: +NOTES: +AUTHOR: Webb Pinner +COMPANY: OceanDataTools.org +VERSION: 2.0 +CREATED: 2025-02-08 +REVISION: + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' + +THRESHOLD = 20 # seconds + +# ------------ only needed for scp transfers -------------- +USER = 'survey' +HOST = '192.168.1.42' +PORT = 22 +KEY_FILE = '/home/sealog/.ssh/id_rsa' + +# ------------ only needed for local transfers ------------ +SOURCE_DIR = '/mnt/ramdisk' + +# --------------------------------------------------------- + +# This needs to match the FILEPATH_ROOT variable in ../config/server_setting.js +DEST_DIR = '/opt/sealog-server/sealog-files/images' + +SOURCES = [ + { + 'source_url': 'http://192.168.1.42/images/', + 'source_filename': 'camera1.jpg', + 'source_name': 'CAMERA_1', + 'filename_prefix': '', + 'filename_suffix': '.jpg' + } +] diff --git a/misc/influx_sealog/aux_data_record_builder.py b/misc/influx_sealog/aux_data_record_builder.py index 4bab6ab..b609d74 100644 --- a/misc/influx_sealog/aux_data_record_builder.py +++ b/misc/influx_sealog/aux_data_record_builder.py @@ -3,7 +3,7 @@ FILE: aux_data_record_builder.py DESCRIPTION: This script builds a sealog aux_data record with data pulled from an - influx database. + influx v2 database. BUGS: NOTES: @@ -17,15 +17,15 @@ Copyright (C) OceanDataTools.org 2025 ''' import sys -import json -import logging + from datetime import datetime, timedelta -from urllib3.exceptions import NewConnectionError from influxdb_client.rest import ApiException +from urllib3.exceptions import NewConnectionError from os.path import dirname, realpath sys.path.append(dirname(dirname(dirname(realpath(__file__))))) +from misc.base_aux_data_record_builder import AuxDataRecordBuilder from misc.influx_sealog.settings import ( INFLUXDB_URL, INFLUXDB_AUTH_TOKEN, @@ -34,35 +34,36 @@ ) -class SealogInfluxAuxDataRecordBuilder(): +class SealogInfluxAuxDataRecordBuilder(AuxDataRecordBuilder): ''' Class that handles the construction of an influxDB query and using the resulting data to build a sealog aux_data record. ''' def __init__(self, influxdb_client, aux_data_config, influxdb_bucket=INFLUXDB_BUCKET): + super().__init__(aux_data_config) self._influxdb_client = influxdb_client.query_api() self._influxdb_bucket = ( aux_data_config['query_bucket'] if 'query_bucket' in aux_data_config else influxdb_bucket ) - self._query_measurements = aux_data_config['query_measurements'] - self._query_fields = list(aux_data_config['aux_record_lookup'].keys()) - self._aux_record_lookup = aux_data_config['aux_record_lookup'] - self._data_source = aux_data_config['data_source'] - self.logger = logging.getLogger(__name__) - @staticmethod - def _build_query_range(ts): + def _build_query_range(self, ts): ''' Builds the temporal range for the influxDB query based on the provided timestamp (ts). + + Args: + ts (str): Timestamp in ISO 8601 format (YYYY-MM-DDTHH:MM:SS.fffZ) + + Returns: + str or None: Query range string ''' try: start_ts = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ") - timedelta(minutes=1) return f'start: {start_ts.strftime("%Y-%m-%dT%H:%M:%S.%fZ")}, stop: {ts}' except ValueError as exc: - logging.debug(str(exc)) + self.logger.debug(str(exc)) return None def _build_query(self, ts): @@ -91,184 +92,60 @@ def _build_query(self, ts): query += '|> sort(columns: ["_time"], desc: true)\n' query += '|> limit(n:1)' - logging.debug("Query: %s", query) + self.logger.debug("Query: %s", query) return query - def _build_aux_data_dict( - self, event_id, influx_query_result - ): # pylint:disable=R0915 + def open_connections(self): ''' - Internal method to build the sealog aux_data record using the event_id, - influx_query_result and the class instance's datasource value. + Open any necessary connections to external data sources. + For Influx, no persistent connection is needed. ''' - aux_data_record = { - 'event_id': event_id, - 'data_source': self._data_source, - 'data_array': [] - } - - influx_data = { - } - - for table in influx_query_result: - for record in table.records: - - influx_data[record.get_field()] = record.get_value() - - logging.debug("raw values: %s", json.dumps(influx_data, indent=2)) - - if not influx_data: - return None - - for key, value in self._aux_record_lookup.items(): - try: - if "no_output" in value and value['no_output'] is True: - continue - - if key not in influx_data: - continue - - output_value = influx_data[key] - - if "modify" in value: - logging.debug("modify found in record") - for mod_op in value['modify']: - test_result = True - - if 'test' in mod_op: - logging.debug("test found in mod_op") - test_result = False - - for test in mod_op['test']: - logging.debug(json.dumps(test)) - - if 'field' in test: - - if test['field'] not in influx_data: - logging.error("test field data not in influx query") - return None - - if 'eq' in test and influx_data[test['field']] == test['eq']: - test_result = True - break - - if 'gt' in test and influx_data[test['field']] > test['gt']: - test_result = True - break - - if 'gte' in test and influx_data[test['field']] >= test['gt']: - test_result = True - break - - if 'lt' in test and influx_data[test['field']] < test['lt']: - test_result = True - break - - if 'lte' in test and influx_data[test['field']] <= test['lt']: - test_result = True - break - - if 'ne' in test and influx_data[test['field']] != test['ne']: - test_result = True - break - - if test_result and 'operation' in mod_op: - logging.debug("operation found in mod_op") - for operan in mod_op['operation']: - - if 'add' in operan: - output_value += operan['add'] - - if 'subtract' in operan: - output_value -= operan['subtract'] - - if 'multiply' in operan: - output_value *= operan['multiply'] - - if 'divide' in operan: - output_value /= operan['divide'] - - aux_data_record['data_array'].append({ - 'data_name': value['name'], - 'data_value': ( - str(round(output_value, value['round'])) - if 'round' in value - else str(output_value) - ), - 'data_uom': value['uom'] if 'uom' in value else '' - }) - except ValueError as exc: - logging.warning("Problem adding %s", key) - logging.debug(str(exc)) - continue - - if len(aux_data_record['data_array']) > 0: - return aux_data_record - - return None + def close_connections(self): + ''' + Close any open connections to external data sources. + For Influx, no persistent connection is needed. + ''' def build_aux_data_record(self, event): ''' Build the aux_data record for the given event. ''' - logging.debug("building query") + self.logger.debug("building query") query = self._build_query(event['ts']) - logging.debug("Query: %s", query) + self.logger.debug("Query: %s", query) # run the query against the influxDB try: query_result = self._influxdb_client.query(query=query) except NewConnectionError: - logging.error("InfluxDB connection error, verify URL: %s", INFLUXDB_URL) + self.logger.error("InfluxDB connection error, verify URL: %s", INFLUXDB_URL) except ApiException as exc: _, value, _ = sys.exc_info() if str(value).startswith("(400)"): - logging.error("InfluxDB API error, verify org: %s", INFLUXDB_ORG) + self.logger.error("InfluxDB API error, verify org: %s", INFLUXDB_ORG) elif str(value).startswith("(401)"): - logging.error("InfluxDB API error, verify token: %s", INFLUXDB_AUTH_TOKEN) + self.logger.error("InfluxDB API error, verify token: %s", INFLUXDB_AUTH_TOKEN) elif str(value).startswith("(404)"): - logging.error("InfluxDB API error, verify bucket: %s", self._influxdb_bucket) + self.logger.error("InfluxDB API error, verify bucket: %s", self._influxdb_bucket) else: - logging.error("Error with query:") - logging.error(query.replace("|>", '\n')) - logging.error(str(exc)) + self.logger.error("Error with query:") + self.logger.error(query.replace("|>", '\n')) + self.logger.error(str(exc)) raise exc else: - aux_data_record = self._build_aux_data_dict(event['id'], query_result) + # Parse InfluxDB result into a dictionary format + influx_data = {} + for table in query_result: + for record in table.records: + influx_data[record.get_field()] = record.get_value() + + aux_data_record = self._build_aux_data_dict(event['id'], influx_data) return aux_data_record return None - - @property - def data_source(self): - ''' - Getter method for the data_source property - ''' - return self._data_source - - @property - def measurements(self): - ''' - Getter method for the _query_measurements property - ''' - return self._query_measurements - - @property - def fields(self): - ''' - Getter method for the _query_fields property - ''' - return self._query_fields - - @property - def record_lookup(self): - ''' - Getter method for the _aux_record_lookup property - ''' - return self._aux_record_lookup diff --git a/misc/influx_sealog/aux_data_record_builder_v1.py b/misc/influx_sealog/aux_data_record_builder_v1.py new file mode 100644 index 0000000..1202f9e --- /dev/null +++ b/misc/influx_sealog/aux_data_record_builder_v1.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +''' +FILE: aux_data_record_builder.py + +DESCRIPTION: This script builds a sealog aux_data record with data pulled from an + influx database. + +BUGS: +NOTES: +AUTHOR: Webb Pinner +COMPANY: OceanDataTools.org +VERSION: 1.0 +CREATED: 2021-01-01 +REVISION: 2022-02-13 + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' +import sys + +from datetime import datetime, timedelta +from influxdb.exceptions import InfluxDBClientError, InfluxDBServerError +from urllib3.exceptions import NewConnectionError + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(dirname(realpath(__file__))))) + +from misc.base_aux_data_record_builder import AuxDataRecordBuilder +from misc.influx_sealog.settings import ( + INFLUXDB_URL, + INFLUXDB_AUTH_TOKEN, + INFLUXDB_ORG, + INFLUXDB_BUCKET +) + + +class SealogInfluxV1AuxDataRecordBuilder(AuxDataRecordBuilder): + ''' + Class that handles the construction of an influxDB query and using the + resulting data to build a sealog aux_data record. + ''' + + def __init__(self, influxdb_client, aux_data_config, influxdb_bucket=INFLUXDB_BUCKET): + super().__init__(aux_data_config) + self._query_filters = aux_data_config.get('query_filters', []) + self._influxdb_client = influxdb_client + self._influxdb_bucket = ( + aux_data_config['query_bucket'] + if 'query_bucket' in aux_data_config else influxdb_bucket + ) + + def _build_query_range(self, ts): + ''' + Builds the temporal range for the influxDB query based on the provided + timestamp (ts). + + Args: + ts (str): Timestamp in ISO 8601 format (YYYY-MM-DDTHH:MM:SS.fffZ) + + Returns: + str or None: Query range string + ''' + str_start_ts = datetime.strftime( + datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ") - timedelta(seconds=20), + "%Y-%m-%dT%H:%M:%SZ" + ) + # Note: in the new code, you subtract a whole minute. Did we choose 20 s on purpose? + + ts_filter = f"time <= '{ts}' AND time > '{str_start_ts}'" + return ts_filter + + def _build_query(self, ts): + ''' + Builds the complete influxDB query using the provided timestamp (ts) + and the class instance's query_measurements and query_fields values. + ''' + + str_field_names = ", ".join([ + f'"{q_field}"' + for q_field in self._query_fields + ]) + + str_time_range = self._build_query_range(ts) + + str_filters = " AND ".join(self._query_filters + [str_time_range]) + + # not sure what to do when there's more than one query measurement. + # Is that even possible in influx v1? + query = f'''SELECT {str_field_names} + FROM "{self._influxdb_bucket}"."one_month"."{self._query_measurements[0]}" + WHERE {str_filters} + ORDER BY DESC LIMIT 1''' + + self.logger.debug("Query: %s", query) + + return query + + def open_connections(self): + ''' + Open any necessary connections to external data sources. + For Influx, no persistent connection is needed. + ''' + + def close_connections(self): + ''' + Close any open connections to external data sources. + For Influx, no persistent connection is needed. + ''' + + def build_aux_data_record(self, event): + ''' + Build the aux_data record for the given event. + ''' + + self.logger.debug("building query") + query = self._build_query(event['ts']) + + self.logger.debug("Query: %s", query) + # run the query against the influxDB + try: + query_result = self._influxdb_client.query(query=query) + + except NewConnectionError: + self.logger.error("InfluxDB connection error, verify URL: %s", INFLUXDB_URL) + + except (InfluxDBClientError, InfluxDBServerError) as exc: + _, value, _ = sys.exc_info() + + if str(value).startswith("(400)"): + self.logger.error("InfluxDB API error, verify org: %s", INFLUXDB_ORG) + elif str(value).startswith("(401)"): + self.logger.error("InfluxDB API error, verify token: %s", INFLUXDB_AUTH_TOKEN) + elif str(value).startswith("(404)"): + self.logger.error("InfluxDB API error, verify bucket: %s", self._influxdb_bucket) + else: + self.logger.error("Error with query:") + self.logger.error(query.replace("|>", '\n')) + self.logger.error(str(exc)) + raise exc + else: + # Parse InfluxDB result into a dictionary format + influx_data = {} + for table in query_result: + for record in table.records: + influx_data[record.get_field()] = record.get_value() + + aux_data_record = self._build_aux_data_dict(event['id'], influx_data) + + return aux_data_record + + return None diff --git a/misc/influx_sealog/settings.py.dist b/misc/influx_sealog/settings.py.dist index bc94b5c..054e9e2 100644 --- a/misc/influx_sealog/settings.py.dist +++ b/misc/influx_sealog/settings.py.dist @@ -23,3 +23,8 @@ INFLUXDB_ORG = 'openrvdas' INFLUXDB_BUCKET = 'openrvdas' INFLUXDB_AUTH_TOKEN = 'DEFAULT_INFLUXDB_AUTH_TOKEN' # noqa:E501 INFLUXDB_VERIFY_SSL = False + +INFLUX_HOST = '10.1.90.40' +INFLUX_PORT = 8086 +INFLUX_USER = '' +INFLUX_PASS = '' diff --git a/misc/sealog_aux_data_inserter_coriolix.py.dist b/misc/sealog_aux_data_inserter_coriolix.py.dist deleted file mode 100644 index ce28f06..0000000 --- a/misc/sealog_aux_data_inserter_coriolix.py.dist +++ /dev/null @@ -1,388 +0,0 @@ -#!/usr/bin/env python3 -''' -FILE: sealog_aux_data_inserter_coriolix.py - -DESCRIPTION: This service listens for new events submitted to Sealog, create - aux_data records containing the specified real-time data and - associates the aux data records with the newly created event. - - This script leverages the CORILOX database/API so that - ancillary data can be added to events from any point in time so - long as the data is availble from the CORIOLIX API. In the event - the data is not available the script will NOT add the - corresponding aux_data records. - - This script can also add CORIOLIX data to a list of event ids - (comma-separated) using the -e flag, or all the events for a given - lowering using the -l flag, or all the events for a given - cruise using the -c flag - -BUGS: -NOTES: -AUTHOR: Webb Pinner -COMPANY: OceanDataTools.org -VERSION: 2.0 -CREATED: 2025-02-09 -REVISION: - -LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) - Copyright (C) OceanDataTools.org 2025 -''' - -import re -import sys -import json -import time -import logging -import asyncio -import websockets -import yaml - -from os.path import dirname, realpath -sys.path.append(dirname(dirname(realpath(__file__)))) - -from misc.python_sealog.events import get_event, get_events_by_cruise, get_events_by_lowering -from misc.python_sealog.event_aux_data import create_event_aux_data -from misc.python_sealog.lowerings import get_lowering_uid_by_id -from misc.python_sealog.cruises import get_cruise_uid_by_id - -from misc.python_sealog.settings import WS_SERVER_URL, HEADERS -from misc.coriolix_sealog.aux_data_record_builder import SealogCORIOLIXAuxDataRecordBuilder - -# ----------------------------------------------------------------------------- # - -INLINE_CONFIG = ''' -- data_source: 'vesselPosition' - query_measurements: - - gnss_gga_bow - - sensor_float_1 - aux_record_lookup: - sensor_float_1__p1: - name: heading - uom: deg - round: 3 - gnss_gga_bow__latitude: - name: latitude - uom: ddeg - round: 6 - gnss_gga_bow__longitude: - name: longitude - uom: ddeg - round: 6 -- data_source: 'meteorologicalData' - query_measurements: - - sensor_mixed_5 - aux_record_lookup: - sensor_mixed_5__p4: - name: 'air temp' - uom: 'C' - round: 2 - sensor_mixed_5__p3: - name: 'air pres' - uom: 'mBar' - round: 2 - modify: - - operation: - - multiply: 1000 - sensor_mixed_5__p5: - name: 'relHumidity' - uom: '%' - round: 1 -- data_source: 'courseAndSpeed' - query_measurements: - - gnss_vtg_bow - aux_record_lookup: - gnss_vtg_bow__cog: - name: course - uom: deg - round: 2 - gnss_vtg_bow__sog: - name: speed - uom: kts - round: 1 - modify: - - operation: - - multiply: 0.539957 -''' - -# set of events to ignore -EXCLUDE_SET = set() - -# needs to be unique for all currently active dataInserter scripts. -CLIENT_WSID = 'auxData-dataInserter-coriolix' - -HELLO = { - 'type': 'hello', - 'id': CLIENT_WSID, - 'auth': { - 'headers': HEADERS - }, - 'version': '2', - 'subs': ['/ws/status/newEvents'] -} - -PING = { - 'type': 'ping', - 'id': CLIENT_WSID -} - - -def parse_event_ids(event_id_file): - ''' - Builds list of event uid from csv-formatted file. - ''' - event_ids_from_file = [] - with open(event_id_file, 'r', encoding='utf-8') as event_id_fp: - for line in event_id_fp: - line = line.rstrip('\n') - logging.debug(line) - event_ids_from_file += line.split(',') - - event_ids_from_file = [event_id.strip() for event_id in event_ids_from_file] - - for event_id in event_ids_from_file: - if re.match(r"^[a-f\d]{24}$", event_id) is None: - logging.error("\"%s\" is an invalid event_id... quiting", event_id) - raise ValueError(f'"{event_id}" is an invalid event_id... quiting') - - return event_ids_from_file - - -def insert_aux_data(aux_data_builders, event, dry_run=False): - ''' - Add aux_data records for only the specified event - ''' - for builder in aux_data_builders: - logging.debug("Building aux data record") - record = builder.build_aux_data_record(event) - if record: - try: - logging.debug("Submitting aux data record to Sealog Server") - logging.debug(json.dumps(record)) - if not dry_run: - create_event_aux_data(record) - - except Exception as exc: # pylint:disable=W0718 - logging.warning("Error submitting aux data record") - logging.debug(str(exc)) - else: - logging.debug("No aux data for data_source: %s", builder.data_source) - - -def insert_aux_data_from_list(aux_data_builders, event_ids_from_list, dry_run=False): - ''' - Add aux_data records for only the events in the specified list - ''' - for event_id in event_ids_from_list: - try: - logging.debug("Retrieving event record from Sealog Server") - event = get_event(event_id) - logging.debug("Event: %s", event) - - except Exception as exc: - logging.warning("Error submitting aux data record") - logging.debug(str(exc)) - raise exc - - insert_aux_data(aux_data_builders, event, dry_run) - - -def insert_aux_data_for_cruise(aux_data_builders, cruise_id, dry_run=False): - ''' - Add aux_data records for only the events in the specified cruise - ''' - cruise_uid = get_cruise_uid_by_id(cruise_id) - - # exit if no cruise found - if not cruise_uid: - logging.error("cruise not found") - return - - # retrieve events for cruise - cruise_events = get_events_by_cruise(cruise_uid) - - # exit if no cruise found - if not cruise_events: - logging.error("no events found for cruise") - return - - for event in cruise_events: - insert_aux_data(aux_data_builders, event, dry_run) - - -def insert_aux_data_for_lowering(aux_data_builders, lowering_id, dry_run=False): - ''' - Add aux_data records for only the events in the specified lowering - ''' - lowering_uid = get_lowering_uid_by_id(lowering_id) - - # exit if no lowering found - if not lowering_uid: - logging.error("lowering not found") - return - - # retrieve events for lowering - lowering_events = get_events_by_lowering(lowering_uid) - - # exit if no lowering found - if not lowering_events: - logging.error("no events found for lowering") - return - - for event in lowering_events: - insert_aux_data(aux_data_builders, event, dry_run) - - -async def insert_aux_data_from_ws(aux_data_builders, dry_run=False): - ''' - Use the aux_data_builder and the coriolix_sealog wrapper to submit aux_data - records built from CORIOLIX data to the sealog-server API - ''' - try: - async with websockets.connect(WS_SERVER_URL) as websocket: - - await websocket.send(json.dumps(HELLO)) - - while True: - - event = await websocket.recv() - event_obj = json.loads(event) - - if event_obj['type'] and event_obj['type'] == 'ping': - await websocket.send(json.dumps(PING)) - elif event_obj['type'] and event_obj['type'] == 'pub': - - if event_obj['message']['event_value'] in EXCLUDE_SET: - logging.debug("Skipping because event value is in the exclude set") - continue - - logging.debug("Event: %s", event_obj['message']) - - insert_aux_data(aux_data_builders, event_obj['message'], dry_run) - - except Exception as exc: - logging.error(str(exc)) - raise exc - -# ------------------------------------------------------------------------------------- -# The main loop of the utility -# ------------------------------------------------------------------------------------- -if __name__ == '__main__': - - import argparse - import os - - parser = argparse.ArgumentParser(description='Aux Data Inserter Service - CORIOLIX') - parser.add_argument('-v', '--verbosity', dest='verbosity', - default=0, action='count', - help='Increase output verbosity') - parser.add_argument('-f', '--config_file', help='use the specifed configuration file') - parser.add_argument( - '-n', '--dry_run', action='store_true', - help='compile the aux_data records but do not submit to server API' - ) - parser.add_argument('-e', '--events', help='list of event_ids to apply the CORIOLIX data') - parser.add_argument('-c', '--cruise_id', help='cruise_id to fix aux_data for') - parser.add_argument('-l', '--lowering_id', help='lowering_id to fix aux_data for') - - parsed_args = parser.parse_args() - - ############################ - # Set up logging before we do any other argument parsing (so that we - # can log problems with argument parsing). - - LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s' - logging.basicConfig(format=LOGGING_FORMAT) - - LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} - parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS)) - logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity]) - - AUX_DATA_CONFIGS = None - - if parsed_args.config_file: - - config_file, data_source = ( - parsed_args.config_file.split(":") - if ":" in parsed_args.config_file - else [parsed_args.config_file, None] - ) - - try: - with open(config_file, 'r', encoding='utf-8') as config_fp: - AUX_DATA_CONFIGS = yaml.safe_load(config_fp) - - if data_source: - AUX_DATA_CONFIGS = [ - config for config in AUX_DATA_CONFIGS if config['data_source'] == data_source - ] - - except yaml.parser.ParserError: - logging.error("Invalid YAML syntax") - sys.exit(1) - else: - try: - AUX_DATA_CONFIGS = yaml.safe_load(INLINE_CONFIG) - except yaml.parser.ParserError: - logging.error("Invalid YAML syntax") - sys.exit(1) - - logging.debug(json.dumps(AUX_DATA_CONFIGS, indent=2)) - - # Create the Aux Data Record Builders - aux_data_builder_list = [ - SealogCORIOLIXAuxDataRecordBuilder(config) for config in AUX_DATA_CONFIGS - ] - - if parsed_args.events: - logging.debug("Processing list of event ids") - - event_ids = parse_event_ids(parsed_args.events) - logging.info("Event IDs:\n%s", json.dumps(event_ids, indent=2)) - - insert_aux_data_from_list(aux_data_builder_list, event_ids, parsed_args.dry_run) - - sys.exit(0) - - if parsed_args.cruise_id: - logging.debug("Processing events for an entire cruise") - - insert_aux_data_for_cruise( - aux_data_builder_list, - parsed_args.cruise_id, - parsed_args.dry_run - ) - - sys.exit(0) - - if parsed_args.lowering_id: - logging.debug("Processing events for an entire lowering") - - insert_aux_data_for_lowering( - aux_data_builder_list, - parsed_args.lowering_id, - parsed_args.dry_run - ) - - sys.exit(0) - - # Run the main loop - while True: - - # Wait 5 seconds for the server to complete startup - time.sleep(5) - - try: - logging.debug("Connecting to event websocket feed...") - asyncio.get_event_loop().run_until_complete( - insert_aux_data_from_ws(aux_data_builder_list, parsed_args.dry_run) - ) - except KeyboardInterrupt: - logging.error('Keyboard Interrupted') - try: - sys.exit(0) - except SystemExit: - os._exit(0) - except Exception as err: # pylint:disable=W0718 - logging.debug(str(err)) - logging.error("Lost connection to server, trying again in 5 seconds") diff --git a/misc/sealog_aux_data_inserter_framegrab.py.dist b/misc/sealog_aux_data_inserter_framegrab.py.dist deleted file mode 100644 index d4bc8e8..0000000 --- a/misc/sealog_aux_data_inserter_framegrab.py.dist +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env python3 -''' -FILE: sealog_aux_data_inserter_framegrab.py - -DESCRIPTION: This service listens for new events submitted to Sealog, copies - frame grab file(s) from a source dir, renames/copies the file - to the sealog-files/images directory and creates an aux_data - record containing the specified real-time data and associates - the aux data record with the newly created event. However if - the realtime data is older than 20 seconds this service will - consider the data stale and will not associate it with the - newly created event. - -BUGS: -NOTES: -AUTHOR: Webb Pinner -COMPANY: OceanDataTools.org -VERSION: 1.1 -CREATED: 2020-01-27 -REVISION: 2023-02-10 - -LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) - Copyright (C) OceanDataTools.org 2025 -''' - -import os -import sys -import asyncio -import json -import time -import shutil -import logging -from datetime import datetime, timedelta -import requests -import websockets - -from os.path import dirname, realpath -sys.path.append(dirname(dirname(realpath(__file__)))) - -try: - from paramiko import RSAKey, SSHClient, SFTPClient, Transport # noqa:F401 pylint:disable=W0611 - from paramiko.sftp import SFTPError # noqa:F401 pylint:disable=W0611 - PARAMIKO_ENABLED = True -except ImportError: - PARAMIKO_ENABLED = False - -from misc.python_sealog.settings import WS_SERVER_URL, HEADERS -from misc.python_sealog.event_aux_data import create_event_aux_data - -# The data_source to use for the auxData records -AUX_DATA_DATASOURCE = 'vehicleRealtimeFramegrabberData' - -# set of events to ignore -EXCLUDE_SET = () - -# needs to be unique for all currently active dataInserter scripts. -CLIENT_WSID = f'aux_data_inserter_{AUX_DATA_DATASOURCE}' - -THRESHOLD = 20 # seconds - -# ------------ only needed for scp transfers -------------- -# if not PARAMIKO_ENABLED: -# raise ModuleNotFoundError('paramiko module is not installed. Please try -# "pip3 install paramiko" prior to use.') -# user = 'survey' -# host = '192.168.1.42' -# port = 22 -# key_file = '/home/sealog/.ssh/id_rsa' -# my_key = RSAKey.from_private_key_file(key_file) -# t = Transport(host, port) - -# ------------ only needed for local transfers ------------ -SOURCE_DIR = '/mnt/ramdisk' - -# --------------------------------------------------------- - -# This needs to match the FILEPATH_ROOT variable in ../config/server_setting.js -DEST_DIR = '/opt/sealog-server/sealog-files/images' - -sources = [ - { - 'source_url': 'http://192.168.1.42/images/', - 'source_filename': 'camera1.jpg', - 'source_name': 'CAMERA_1', - 'filename_prefix': '', - 'filename_suffix': '.jpg' - } -] - -HELLO = { - 'type': 'hello', - 'id': CLIENT_WSID, - 'auth': { - 'headers': HEADERS - }, - 'version': '2', - 'subs': ['/ws/status/newEvents'] -} - -PING = { - 'type': 'ping', - 'id': CLIENT_WSID -} - - -async def aux_data_inserter(): - ''' - Connect to the websocket feed for new events. When new events arrive, - build aux_data records and submit them to the sealog-server. - ''' - - logging.debug("Connecting to event websocket feed...") - try: - async with websockets.connect(WS_SERVER_URL) as websocket: - - await websocket.send(json.dumps(HELLO)) - - while True: - - event = await websocket.recv() - event_obj = json.loads(event) - - if event_obj['type'] and event_obj['type'] == 'ping': - await websocket.send(json.dumps(PING)) - - elif event_obj['type'] and event_obj['type'] == 'pub': - - if event_obj['message']['event_value'] in EXCLUDE_SET: - logging.debug( - "Skipping because event value is in the exclude set" - ) - continue - - if datetime.strptime( - event_obj['message']['ts'], - '%Y-%m-%dT%H:%M:%S.%fZ' - ) < datetime.utcnow()-timedelta(seconds=THRESHOLD): - logging.debug("Skipping because event ts is older than thresold") - continue - - aux_data_record = { - 'event_id': event_obj['message']['id'], - 'data_source': AUX_DATA_DATASOURCE, - 'data_array': [] - } - - for source in sources: - - filename_date = datetime.date(datetime.strptime( - event_obj['message']['ts'], - '%Y-%m-%dT%H:%M:%S.%fZ') - ) - filename_time = datetime.time( - datetime.strptime( - event_obj['message']['ts'], - '%Y-%m-%dT%H:%M:%S.%fZ' - ) - ) - filename_middle = datetime.combine( - filename_date, filename_time - ).strftime("%Y%m%d_%H%M%S%f")[:-3] - - dst = os.path.join( - DEST_DIR, - source['filename_prefix'] + filename_middle + source['filename_suffix'] - ) - - logging.debug("dst: %s", dst) - - # ------------ only needed for scp transfers ------------- - # try: - # latest_file = os.path.join(SOURCE_DIR, source['source_filename']) - # src = os.path.join(SOURCE_DIR, latest_file) - # sftp = SFTPClient.from_transport(t) - # sftp.put(src, dst) - # sftp.close() - - # except SFTPError as exc: - # logging.error("Unable to copy image to server") - # logging.error(exc) - - # except OSError as exc: - # logging.error("Unable to copy image to server") - # logging.error(exc) - # ------------ only needed for scp transfers ------------- - - # ------------ only needed for http transfers ------------- - try: - res = requests.get( - source['source_url'] + source['source_filename'], - stream=True, - timeout=(2, None) - ) - - if res.status_code != 200: - logging.error( - "Unable to retrieve image from: %s", - source['source_url'] + source['source_filename'] - ) - continue - - except requests.exceptions.RequestException as exc: - logging.error("Unable to retrieve image from remote server") - logging.error(exc) - - try: - with open(dst, 'wb') as f: - shutil.copyfileobj(res.raw, f) - - except shutil.Error as exc: - logging.error("Unable to save image to server") - logging.error(exc) - # ------------ only needed for http transfers ------------- - - # ----------- only needed for local transfers ------------ - # latest_file = os.path.join(SOURCE_DIR, source['source_filename']) - # src = os.path.join(SOURCE_DIR, latest_file) - # try: - # shutil.copyfile(src,dst) - - # except shutil.Error as exc: - # logging.error("Unable to save image to server") - # logging.error(exc) - # ----------- only needed for local transfers ------------ - - aux_data_record['data_array'].append( - {'data_name': "camera_name", 'data_value': source['source_name']} - ) - aux_data_record['data_array'].append( - {'data_name': "filename", 'data_value': dst} - ) - - if len(aux_data_record['data_array']) > 0: - create_event_aux_data(aux_data_record) - - except Exception as exc: - logging.error(str(exc)) - raise exc - -# ------------------------------------------------------------------------------------- -# Required python code for running the script as a stand-alone utility -# ------------------------------------------------------------------------------------- -if __name__ == '__main__': - - import argparse - - parser = argparse.ArgumentParser( - description='Aux Data Inserter Service - ' + AUX_DATA_DATASOURCE - ) - parser.add_argument( - '-v', '--verbosity', dest='verbosity', default=0, action='count', - help='Increase output verbosity') - - parsed_args = parser.parse_args() - - ############################ - # Set up logging before we do any other argument parsing (so that we - # can log problems with argument parsing). - - LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s' - logging.basicConfig(format=LOGGING_FORMAT) - - LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} - parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS)) - logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity]) - - # Run the main loop - while True: - - # Wait 5 seconds for the server to complete startup - time.sleep(5) - - try: - # t.connect(username=user, pkey=my_key) # only needed for scp transfers - asyncio.get_event_loop().run_until_complete(aux_data_inserter()) - except KeyboardInterrupt: - logging.error('Keyboard Interrupted') - try: - sys.exit(0) - except SystemExit: - os._exit(0) - except Exception as exc: # pylint:disable=W0718 - logging.error("Lost connection to server, trying again in 5 seconds") - logging.debug(str(exc)) diff --git a/misc/sealog_aux_data_inserter_influx.py.dist b/misc/sealog_aux_data_inserter_influx.py.dist deleted file mode 100644 index 56455e5..0000000 --- a/misc/sealog_aux_data_inserter_influx.py.dist +++ /dev/null @@ -1,389 +0,0 @@ -#!/usr/bin/env python3 -''' -FILE: sealog_aux_data_inserter_influx.py - -DESCRIPTION: This service listens for new events submitted to Sealog, create - aux_data records containing the specified real-time data and - associates the aux data records with the newly created event. - - This script leverages the OpenRVDAS/InfluxDB integration so that - if can add ancillary data from any point in time so long as the - data is availble from the InfluxDB. In the event the data is not - available the script will NOT add the corresponding aux_data - records. - - This script can also add influx data to a list of event ids - (comma-separated) using the -e flag, or all the events for a given - lowering using the -l flag, or all the events for a given - cruise using the -c flag - -BUGS: -NOTES: -AUTHOR: Webb Pinner -COMPANY: OceanDataTools.org -VERSION: 1.1 -CREATED: 2021-04-21 -REVISION: 2023-02-10 - -LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) - Copyright (C) OceanDataTools.org 2025 -''' - -import re -import sys -import json -import time -import logging -import asyncio -import websockets -import yaml -from influxdb_client import InfluxDBClient - -from os.path import dirname, realpath -sys.path.append(dirname(dirname(realpath(__file__)))) - -from misc.python_sealog.events import get_event, get_events_by_cruise, get_events_by_lowering -from misc.python_sealog.event_aux_data import create_event_aux_data -from misc.python_sealog.lowerings import get_lowering_uid_by_id -from misc.python_sealog.cruises import get_cruise_uid_by_id - -from misc.python_sealog.settings import WS_SERVER_URL, HEADERS -from misc.influx_sealog.settings import ( - INFLUXDB_URL, - INFLUXDB_AUTH_TOKEN, - INFLUXDB_ORG, - INFLUXDB_VERIFY_SSL -) -from misc.influx_sealog.aux_data_record_builder import SealogInfluxAuxDataRecordBuilder - -# ----------------------------------------------------------------------------- # - -INLINE_CONFIG = ''' -- data_source: realtimeVesselPosition - # query_bucket: openrvdas - query_measurements: - - s330 - aux_record_lookup: - S330HeadingTrue: - name: heading - uom: deg - round: 3 - S330Latitude: - name: latitude - uom: ddeg - round: 6 - modify: - - test: - - field: S330NorS - eq: S - operation: - - multiply: -1 - S330Longitude: - name: longitude - uom: ddeg - round: 6 - modify: - - test: - - field: S330EorW - eq: W - operation: - - multiply: -1 - S330NorS: - no_output: true - S330EorW: - no_output: true -''' - -# set of events to ignore -EXCLUDE_SET = set() - -# needs to be unique for all currently active dataInserter scripts. -CLIENT_WSID = 'auxData-dataInserter-influx' - -HELLO = { - 'type': 'hello', - 'id': CLIENT_WSID, - 'auth': { - 'headers': HEADERS - }, - 'version': '2', - 'subs': ['/ws/status/newEvents'] -} - -PING = { - 'type': 'ping', - 'id': CLIENT_WSID -} - - -def parse_event_ids(event_id_file): - ''' - Builds list of event uid from csv-formatted file. - ''' - event_ids_from_file = [] - with open(event_id_file, 'r', encoding='utf-8') as event_id_fp: - for line in event_id_fp: - line = line.rstrip('\n') - logging.debug(line) - event_ids_from_file += line.split(',') - - event_ids_from_file = [event_id.strip() for event_id in event_ids_from_file] - - for event_id in event_ids_from_file: - if re.match(r"^[a-f\d]{24}$", event_id) is None: - logging.error("\"%s\" is an invalid event_id... quiting", event_id) - raise ValueError(f'"{event_id}" is an invalid event_id... quiting') - - return event_ids_from_file - - -def insert_aux_data(aux_data_builders, event, dry_run=False): - ''' - Add aux_data records for only the specified event - ''' - for builder in aux_data_builders: - logging.debug("Building aux data record") - record = builder.build_aux_data_record(event) - if record: - try: - logging.debug("Submitting aux data record to Sealog Server") - logging.debug(json.dumps(record)) - if not dry_run: - create_event_aux_data(record) - - except Exception as exc: # pylint:disable=W0718 - logging.warning("Error submitting aux data record") - logging.debug(str(exc)) - else: - logging.debug("No aux data for data_source: %s", builder.data_source) - - -def insert_aux_data_from_list(aux_data_builders, event_ids_from_list, dry_run=False): - ''' - Add aux_data records for only the events in the specified list - ''' - for event_id in event_ids_from_list: - try: - logging.debug("Retrieving event record from Sealog Server") - event = get_event(event_id) - logging.debug("Event: %s", event) - - except Exception as exc: - logging.warning("Error submitting aux data record") - logging.debug(str(exc)) - raise exc - - insert_aux_data(aux_data_builders, event, dry_run) - - -def insert_aux_data_for_cruise(aux_data_builders, cruise_id, dry_run=False): - ''' - Add aux_data records for only the events in the specified cruise - ''' - cruise_uid = get_cruise_uid_by_id(cruise_id) - - # exit if no cruise found - if not cruise_uid: - logging.error("cruise not found") - return - - # retrieve events for cruise - cruise_events = get_events_by_cruise(cruise_uid) - - # exit if no cruise found - if not cruise_events: - logging.error("no events found for cruise") - return - - for event in cruise_events: - insert_aux_data(aux_data_builders, event, dry_run) - - -def insert_aux_data_for_lowering(aux_data_builders, lowering_id, dry_run=False): - ''' - Add aux_data records for only the events in the specified lowering - ''' - lowering_uid = get_lowering_uid_by_id(lowering_id) - - # exit if no lowering found - if not lowering_uid: - logging.error("lowering not found") - return - - # retrieve events for lowering - lowering_events = get_events_by_lowering(lowering_uid) - - # exit if no lowering found - if not lowering_events: - logging.error("no events found for lowering") - return - - for event in lowering_events: - insert_aux_data(aux_data_builders, event, dry_run) - - -async def insert_aux_data_from_ws(aux_data_builders, dry_run=False): - ''' - Use the aux_data_builder and the influx_sealog wrapper to submit aux_data - records built from influxDB data to the sealog-server API - ''' - try: - async with websockets.connect(WS_SERVER_URL) as websocket: - - await websocket.send(json.dumps(HELLO)) - - while True: - - event = await websocket.recv() - event_obj = json.loads(event) - - if event_obj['type'] and event_obj['type'] == 'ping': - await websocket.send(json.dumps(PING)) - elif event_obj['type'] and event_obj['type'] == 'pub': - - if event_obj['message']['event_value'] in EXCLUDE_SET: - logging.debug("Skipping because event value is in the exclude set") - continue - - logging.debug("Event: %s", event_obj['message']) - - insert_aux_data(aux_data_builders, event_obj['message'], dry_run) - - except Exception as exc: - logging.error(str(exc)) - raise exc - -# ------------------------------------------------------------------------------------- -# The main loop of the utility -# ------------------------------------------------------------------------------------- -if __name__ == '__main__': - - import argparse - import os - - parser = argparse.ArgumentParser(description='Aux Data Inserter Service - InfluxDB') - parser.add_argument( - '-v', '--verbosity', dest='verbosity', default=0, - action='count', help='Increase output verbosity' - ) - parser.add_argument('-f', '--config_file', help='use the specifed configuration file') - parser.add_argument( - '-n', '--dry_run', action='store_true', - help='compile the aux_data records but do not submit to server API' - ) - parser.add_argument('-e', '--events', help='list of event_ids to apply the influx data') - parser.add_argument('-c', '--cruise_id', help='cruise_id to fix aux_data for') - parser.add_argument('-l', '--lowering_id', help='lowering_id to fix aux_data for') - - parsed_args = parser.parse_args() - - ############################ - # Set up logging before we do any other argument parsing (so that we - # can log problems with argument parsing). - - LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s' - logging.basicConfig(format=LOGGING_FORMAT) - - LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} - parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS)) - logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity]) - - AUX_DATA_CONFIGS = None - - if parsed_args.config_file: - - config_file, data_source = ( - parsed_args.config_file.split(":") - if ":" in parsed_args.config_file - else [parsed_args.config_file, None] - ) - - try: - with open(config_file, 'r', encoding='utf-8') as config_fp: - AUX_DATA_CONFIGS = yaml.safe_load(config_fp) - - if data_source: - AUX_DATA_CONFIGS = [ - config for config in AUX_DATA_CONFIGS if config['data_source'] == data_source - ] - - except yaml.parser.ParserError: - logging.error("Invalid YAML syntax") - sys.exit(1) - else: - try: - AUX_DATA_CONFIGS = yaml.safe_load(INLINE_CONFIG) - except yaml.parser.ParserError: - logging.error("Invalid YAML syntax") - sys.exit(1) - - logging.debug(json.dumps(AUX_DATA_CONFIGS, indent=2)) - - # create an influxDB Client - use_ssl = INFLUXDB_URL.find('https:') == 0 - client = InfluxDBClient(url=INFLUXDB_URL, - token=INFLUXDB_AUTH_TOKEN, - org=INFLUXDB_ORG, - ssl=use_ssl, - verify_ssl=INFLUXDB_VERIFY_SSL) - - # Create the Aux Data Record Builders - aux_data_builder_list = list( - map( - lambda config: SealogInfluxAuxDataRecordBuilder(client, config), - AUX_DATA_CONFIGS - ) - ) - - if parsed_args.events: - logging.debug("Processing list of event ids") - - event_ids = parse_event_ids(parsed_args.events) - logging.info("Event IDs:\n%s", json.dumps(event_ids, indent=2)) - - insert_aux_data_from_list(aux_data_builder_list, event_ids, parsed_args.dry_run) - - sys.exit(0) - - if parsed_args.cruise_id: - logging.debug("Processing events for an entire cruise") - - insert_aux_data_for_cruise( - aux_data_builder_list, - parsed_args.cruise_id, - parsed_args.dry_run - ) - - sys.exit(0) - - if parsed_args.lowering_id: - logging.debug("Processing events for an entire lowering") - - insert_aux_data_for_lowering( - aux_data_builder_list, - parsed_args.lowering_id, - parsed_args.dry_run - ) - - sys.exit(0) - - # Run the main loop - while True: - - # Wait 5 seconds for the server to complete startup - time.sleep(5) - - try: - logging.debug("Connecting to event websocket feed...") - asyncio.get_event_loop().run_until_complete( - insert_aux_data_from_ws(aux_data_builder_list, parsed_args.dry_run) - ) - except KeyboardInterrupt: - logging.error('Keyboard Interrupted') - try: - sys.exit(0) - except SystemExit: - os._exit(0) - except Exception as err: # pylint:disable=W0718 - logging.debug(str(err)) - logging.error("Lost connection to server, trying again in 5 seconds") diff --git a/misc/sealog_aux_data_manager_coriolix.py.dist b/misc/sealog_aux_data_manager_coriolix.py.dist new file mode 100644 index 0000000..d6ab523 --- /dev/null +++ b/misc/sealog_aux_data_manager_coriolix.py.dist @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +''' +FILE: sealog_aux_data_manager_coriolix.py + +DESCRIPTION: This service listens for new events submitted to Sealog, creates + aux_data records containing the specified real-time data and + associates the aux data records with the newly created event. + + This script leverages the CORILOX database/API so that + ancillary data can be added to events from any point in time so + long as the data is availble from the CORIOLIX API. In the event + the data is not available the script will NOT add the + corresponding aux_data records. + + This script can also add CORIOLIX data to a list of event ids + (comma-separated) using the -e flag, or all the events for a given + lowering using the -l flag, or all the events for a given + cruise using the -c flag + +BUGS: +NOTES: +AUTHOR: Webb Pinner +COMPANY: OceanDataTools.org +VERSION: 2.0 +CREATED: 2025-02-09 +REVISION: + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' + +import sys +from os.path import dirname, realpath +sys.path.append(dirname(dirname(realpath(__file__)))) + +from misc.python_sealog.settings import WS_SERVER_URL, HEADERS +from misc.coriolix_sealog.aux_data_record_builder import SealogCORIOLIXAuxDataRecordBuilder +from misc.aux_data_file_cleaners.do_nothing_aux_data_file_cleaner import DoNothingAuxDataFileCleaner +from misc.aux_data_manager_runner import run_aux_data_manager + +# ----------------------------------------------------------------------------- # + +INLINE_CONFIG = ''' +- data_source: 'vesselPosition' + query_measurements: + - gnss_gga_bow + - sensor_float_1 + aux_record_lookup: + sensor_float_1__p1: + name: heading + uom: deg + round: 3 + gnss_gga_bow__latitude: + name: latitude + uom: ddeg + round: 6 + gnss_gga_bow__longitude: + name: longitude + uom: ddeg + round: 6 +- data_source: 'meteorologicalData' + query_measurements: + - sensor_mixed_5 + aux_record_lookup: + sensor_mixed_5__p4: + name: 'air temp' + uom: 'C' + round: 2 + sensor_mixed_5__p3: + name: 'air pres' + uom: 'mBar' + round: 2 + modify: + - operation: + - multiply: 1000 + sensor_mixed_5__p5: + name: 'relHumidity' + uom: '%' + round: 1 +- data_source: 'courseAndSpeed' + query_measurements: + - gnss_vtg_bow + aux_record_lookup: + gnss_vtg_bow__cog: + name: course + uom: deg + round: 2 + gnss_vtg_bow__sog: + name: speed + uom: kts + round: 1 + modify: + - operation: + - multiply: 0.539957 +''' + +# needs to be unique for all currently active dataManager scripts. +CLIENT_WSID = 'auxData-dataManager-coriolix' + +# ------------------------------------------------------------------------------------- +# The main loop of the utility +# ------------------------------------------------------------------------------------- +if __name__ == '__main__': + + def builder_factory(aux_data_config): + '''Build an AuxDataRecordBuilder''' + return SealogCORIOLIXAuxDataRecordBuilder(aux_data_config) + + def cleaner_factory(aux_data_config): + '''Build an AuxDataFileCleaner''' + return DoNothingAuxDataFileCleaner(aux_data_config) + + run_aux_data_manager( + builder_factory=builder_factory, + cleaner_factory=cleaner_factory, + inline_config=INLINE_CONFIG, + ws_server_url=WS_SERVER_URL, + headers=HEADERS, + client_wsid=CLIENT_WSID + ) diff --git a/misc/sealog_aux_data_manager_framegrab.py.dist b/misc/sealog_aux_data_manager_framegrab.py.dist new file mode 100644 index 0000000..3240dde --- /dev/null +++ b/misc/sealog_aux_data_manager_framegrab.py.dist @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +''' +FILE: sealog_aux_data_manager_framegrab.py + +DESCRIPTION: This service listens for new events submitted to Sealog, copies + frame grab file(s) from a source dir, renames/copies the file + to the sealog-files/images directory and creates an aux_data + record containing the specified real-time data and associates + the aux data record with the newly created event. However if + the realtime data is older than 20 seconds this service will + consider the data stale and will not associate it with the + newly created event. + +BUGS: +NOTES: +AUTHOR: Webb Pinner +COMPANY: OceanDataTools.org +VERSION: 1.1 +CREATED: 2020-01-27 +REVISION: 2023-02-10 + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' + +import sys +from os.path import dirname, realpath +sys.path.append(dirname(dirname(realpath(__file__)))) + +from misc.python_sealog.settings import WS_SERVER_URL, HEADERS +from misc.framegrab_aux.aux_data_record_builder_framegrab_http import ( + FramegrabHTTPAuxDataRecordBuilder + ) +from misc.aux_data_file_cleaners.delete_files_aux_data_file_cleaner import ( + DeleteFilesAuxDataFileCleaner + ) +from misc.aux_data_manager_runner import run_aux_data_manager + +# The data_source to use for the auxData records +INLINE_CONFIG = ''' +- data_source: 'vehicleRealtimeFramegrabberData' +''' + +# set of events to ignore +EXCLUDE_SET = () + +# needs to be unique for all currently active dataInserter scripts. +CLIENT_WSID = 'aux_data_inserter_vehicleRealtimeFramegrabberData' + + +# ------------------------------------------------------------------------------------- +# Required python code for running the script as a stand-alone utility +# ------------------------------------------------------------------------------------- +if __name__ == '__main__': + + def builder_factory(aux_data_config): + '''Build an AuxDataRecordBuilder''' + return FramegrabHTTPAuxDataRecordBuilder(aux_data_config) + + def cleaner_factory(aux_data_config): + '''Build an AuxDataFileCleaner''' + return DeleteFilesAuxDataFileCleaner(aux_data_config) + + run_aux_data_manager( + builder_factory=builder_factory, + cleaner_factory=cleaner_factory, + inline_config=INLINE_CONFIG, + ws_server_url=WS_SERVER_URL, + headers=HEADERS, + client_wsid=CLIENT_WSID, + exclude_set=EXCLUDE_SET + ) diff --git a/misc/sealog_aux_data_manager_influx.py.dist b/misc/sealog_aux_data_manager_influx.py.dist new file mode 100644 index 0000000..c7674fe --- /dev/null +++ b/misc/sealog_aux_data_manager_influx.py.dist @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +''' +FILE: sealog_aux_data_manager_influx.py + +DESCRIPTION: This service listens for new events submitted to Sealog, create + aux_data records containing the specified real-time data and + associates the aux data records with the newly created event. + + This script leverages the OpenRVDAS/InfluxDB integration so that + if can add ancillary data from any point in time so long as the + data is availble from the InfluxDB. In the event the data is not + available the script will NOT add the corresponding aux_data + records. + + This script can also add influx data to a list of event ids + (comma-separated) using the -e flag, or all the events for a given + lowering using the -l flag, or all the events for a given + cruise using the -c flag + +BUGS: +NOTES: +AUTHOR: Webb Pinner +COMPANY: OceanDataTools.org +VERSION: 1.1 +CREATED: 2021-04-21 +REVISION: 2023-02-10 + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' + +import sys +from influxdb_client import InfluxDBClient + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(realpath(__file__)))) + +from misc.python_sealog.settings import WS_SERVER_URL, HEADERS +from misc.influx_sealog.settings import ( + INFLUXDB_URL, + INFLUXDB_AUTH_TOKEN, + INFLUXDB_ORG, + INFLUXDB_VERIFY_SSL +) +from misc.influx_sealog.aux_data_record_builder import SealogInfluxAuxDataRecordBuilder +from misc.aux_data_file_cleaners.do_nothing_aux_data_file_cleaner import DoNothingAuxDataFileCleaner +from misc.aux_data_manager_runner import run_aux_data_manager + +# ----------------------------------------------------------------------------- # + +INLINE_CONFIG = ''' +- data_source: realtimeVesselPosition + # query_bucket: openrvdas + query_measurements: + - s330 + aux_record_lookup: + S330HeadingTrue: + name: heading + uom: deg + round: 3 + S330Latitude: + name: latitude + uom: ddeg + round: 6 + modify: + - test: + - field: S330NorS + eq: S + operation: + - multiply: -1 + S330Longitude: + name: longitude + uom: ddeg + round: 6 + modify: + - test: + - field: S330EorW + eq: W + operation: + - multiply: -1 + S330NorS: + no_output: true + S330EorW: + no_output: true +''' + +# set of events to ignore +EXCLUDE_SET = set() + +# needs to be unique for all currently active dataManager scripts. +CLIENT_WSID = 'auxData-dataManager-influx' + +# ------------------------------------------------------------------------------------- +# The main loop of the utility +# ------------------------------------------------------------------------------------- +if __name__ == '__main__': + # create an influxDB Client + use_ssl = INFLUXDB_URL.find('https:') == 0 + client = InfluxDBClient(url=INFLUXDB_URL, + token=INFLUXDB_AUTH_TOKEN, + org=INFLUXDB_ORG, + ssl=use_ssl, + verify_ssl=INFLUXDB_VERIFY_SSL) + + # builder factory that injects the client + def builder_factory(aux_data_config): + '''Build an AuxDataRecordBuilder''' + return SealogInfluxAuxDataRecordBuilder(client, aux_data_config) + + def cleaner_factory(aux_data_config): + '''Build an AuxDataFileCleaner''' + return DoNothingAuxDataFileCleaner(aux_data_config) + + run_aux_data_manager( + builder_factory=builder_factory, + cleaner_factory=cleaner_factory, + inline_config=INLINE_CONFIG, + ws_server_url=WS_SERVER_URL, + headers=HEADERS, + client_wsid=CLIENT_WSID + ) diff --git a/misc/sealog_aux_data_manager_influx_v1.py.dist b/misc/sealog_aux_data_manager_influx_v1.py.dist new file mode 100644 index 0000000..bee105c --- /dev/null +++ b/misc/sealog_aux_data_manager_influx_v1.py.dist @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +''' +FILE: sealog_aux_data_manager_influx_v1.py + +DESCRIPTION: This service listens for new events submitted to Sealog, create + aux_data records containing the specified real-time data and + associates the aux data records with the newly created event. + + This script leverages the OpenRVDAS/InfluxDB integration so that + if can add ancillary data from any point in time so long as the + data is availble from the InfluxDB. In the event the data is not + available the script will NOT add the corresponding aux_data + records. + + This script can also add influx data to a list of event ids + (comma-separated) using the -e flag, or all the events for a given + lowering using the -l flag, or all the events for a given + cruise using the -c flag + +BUGS: +NOTES: +AUTHOR: Webb Pinner +COMPANY: OceanDataTools.org +VERSION: 1.1 +CREATED: 2021-04-21 +REVISION: 2023-02-10 + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' + +import sys +from influxdb import InfluxDBClient + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(realpath(__file__)))) + +from misc.python_sealog.settings import WS_SERVER_URL, HEADERS +from misc.influx_sealog.settings import ( + INFLUX_HOST, + INFLUX_PORT, + INFLUX_USER, + INFLUX_PASS +) +from misc.influx_sealog.aux_data_record_builder_v1 import SealogInfluxV1AuxDataRecordBuilder +from misc.aux_data_file_cleaners.do_nothing_aux_data_file_cleaner import DoNothingAuxDataFileCleaner +from misc.aux_data_manager_runner import run_aux_data_manager + +# ----------------------------------------------------------------------------- # + +INLINE_CONFIG = ''' +- data_source: hercRealtimeNavData + query_measurements: + - sonardyne_nav + query_filters: + - |- + "name" = 'Herc 2412' + aux_record_lookup: + latitude: + name: latitude + uom: ddeg + round: 6 + longitude: + name: longitude + uom: ddeg + round: 6 + depth: + name: depth + uom: m + round: 2 + name: + no_output: true +''' + +# set of events to ignore +EXCLUDE_SET = set() + +# needs to be unique for all currently active dataManager scripts. +CLIENT_WSID = 'auxData-dataManager-influx-v1' + +# ------------------------------------------------------------------------------------- +# The main loop of the utility +# ------------------------------------------------------------------------------------- +if __name__ == '__main__': + # create an influxDB Client + client = InfluxDBClient(host=INFLUX_HOST, + port=INFLUX_PORT, + username=INFLUX_USER, + password=INFLUX_PASS + ) + + # builder factory that injects the client + def builder_factory(aux_data_config): + '''Build an AuxDataRecordBuilder''' + return SealogInfluxV1AuxDataRecordBuilder(client, aux_data_config) + + def cleaner_factory(aux_data_config): + '''Build an AuxDataFileCleaner''' + return DoNothingAuxDataFileCleaner(aux_data_config) + + run_aux_data_manager( + builder_factory=builder_factory, + cleaner_factory=cleaner_factory, + inline_config=INLINE_CONFIG, + ws_server_url=WS_SERVER_URL, + headers=HEADERS, + client_wsid=CLIENT_WSID + ) diff --git a/misc/sealog_create_cruise_from_openvdm.py.dist b/misc/sealog_create_cruise_from_openvdm.py.dist index 345d3c7..19ffe05 100644 --- a/misc/sealog_create_cruise_from_openvdm.py.dist +++ b/misc/sealog_create_cruise_from_openvdm.py.dist @@ -92,9 +92,9 @@ def main(force=False): # pylint:disable=R0915 if req.status_code == 200: cruise_config = json.loads(req.text) cruise['cruise_id'] = cruise_config['cruiseID'] - cruise['cruise_additional_meta']['cruise_name'] = cruise_config['cruiseName'] - cruise['cruise_location'] = cruise_config['cruiseLocation'] - cruise['cruise_additional_meta']['cruise_pi'] = cruise_config['cruisePI'] + cruise['cruise_additional_meta']['cruise_name'] = cruise_config.get('cruiseName', 'FIX ME') + cruise['cruise_location'] = cruise_config.get('cruiseLocation', 'FIX ME') + cruise['cruise_additional_meta']['cruise_pi'] = cruise_config.get('cruisePI', 'FIX ME') cruise['start_ts'] = datetime.strptime( cruise_config['cruiseStartDate'], diff --git a/plugins/auth.js b/plugins/auth.js index 75cf85c..e54e011 100644 --- a/plugins/auth.js +++ b/plugins/auth.js @@ -1,6 +1,9 @@ +const Boom = require('@hapi/boom'); +const Bcrypt = require('bcryptjs'); const SECRET_KEY = require('../config/secret'); const { + apiKeysTable, usersTable } = require('../config/db_constants'); @@ -9,10 +12,10 @@ exports.plugin = { dependencies: ['hapi-mongodb', 'hapi-auth-jwt2'], register: (server, options) => { - const validateFunction = async (decoded, request) => { + const db = server.mongo.db; + const ObjectID = server.mongo.ObjectID; - const db = request.mongo.db; - const ObjectID = request.mongo.ObjectID; + const validateJWT = async (decoded, request) => { try { const result = await db.collection(usersTable).findOne({ _id: new ObjectID(decoded.id) }); @@ -26,7 +29,10 @@ exports.plugin = { return { isValid: false }; } - await db.collection(usersTable).updateOne({ _id: new ObjectID(decoded.id) }, { $set: { last_login: new Date() } }); + // Update last_login no more than once every 10 minutes + if (!result.last_login || Date.now() - result.last_login.getTime() > 600000) { + await db.collection(usersTable).updateOne({ _id: new ObjectID(decoded.id) }, { $set: { last_login: new Date() } }); + } return { isValid: true }; @@ -44,7 +50,85 @@ exports.plugin = { algorithms: ['HS256'] }, // Implement validation function - validate: validateFunction + validate: validateJWT }); + + // ---------------- API KEY VALIDATION ---------------- + + const validateApiKey = async (providedKey) => { + + if (!providedKey) { + return null; + } + + // Fetch only keys that are not disabled or deleted + const keys = await db.collection(apiKeysTable).find({ disabled: { $ne: true } }).toArray(); + console.error('keys:',keys); + + for (const keyRecord of keys) { + // Compare raw key to hashed key (correct order!!) + const isMatch = await Bcrypt.compare(providedKey, keyRecord.key_hash); + + if (!isMatch) { + continue; // try next key + } + + // Check expiration + if (keyRecord.expiresAt && keyRecord.expiresAt < new Date()) { + // Key exists but is expired → treat as invalid + return null; + } + + // Key is valid and not expired + return keyRecord; + } + + // Nothing matched + return null; + }; + + const apiKeyScheme = () => ({ + authenticate: async (request, h) => { + + const apiKey = request.headers['x-api-key']; + + if (!apiKey) { + throw Boom.unauthorized('Missing API Key'); + } + + const keyRecord = await validateApiKey(apiKey); + + if (!keyRecord) { + throw Boom.unauthorized('Invalid API Key'); + } + + const { user_id, scope, expires } = keyRecord; + + // Check expiration + if (expires && new Date() > expires) { + throw Boom.unauthorized('API Key expired'); + } + + // Verify user still exists + const user = await db.collection(usersTable).findOne({ _id: new ObjectID(user_id) }); + if (!user || user.disabled) { + throw Boom.unauthorized('User disabled or missing'); + } + + // Log usage + await db.collection(apiKeysTable).updateOne( + { _id: keyRecord._id }, + { $set: { last_used: new Date() } } + ); + + return h.authenticated({ + credentials: { id: user_id, scope, type: 'api-key' } + }); + } + }); + + + server.auth.scheme('api-key', apiKeyScheme); + server.auth.strategy('api-key', 'api-key'); } }; diff --git a/plugins/db_api_keys.js b/plugins/db_api_keys.js new file mode 100644 index 0000000..de8058b --- /dev/null +++ b/plugins/db_api_keys.js @@ -0,0 +1,69 @@ +const { hashedApiKey } = require('../lib/utils'); + +const { + apiKeysTable +} = require('../config/db_constants'); + +exports.plugin = { + name: 'db_populate_apikeys', + dependencies: ['hapi-mongodb'], + register: async (server, options) => { + + const db = server.mongo.db; + const ObjectID = server.mongo.ObjectID; + + const expiresAt = new Date(); + expiresAt.setDate(expiresAt.getDate() + 30); + + const init_data = [ + { + _id: ObjectID('5981f167212b348ae32fa9f5'), + user_id: ObjectID('5981f167212b348aed7fa9f5'), // Reference to users collection + key_hash: await hashedApiKey('5981f167212b348ae32fa9f5'), // We store a hash, never raw key + label: 'Default Key', + scope: ['read_cruises'], // Optional: can match user scopes or add more granular scopes + created: new Date(), + last_used: null, + disabled: false, + expires: expiresAt + } + ]; + + console.log('Searching for API Keys Collection'); + const result = await db.listCollections({ name: apiKeysTable }).toArray(); + + if (result.length) { + if (process.env.NODE_ENV !== 'development') { + console.log('API Keys Collection already exists... we\'re done here.'); + return; + } + + console.log('API Keys Collection exists... dropping it!'); + try { + await db.dropCollection(apiKeysTable); + } + catch (err) { + console.log('DROP ERROR:', err.code); + throw (err); + } + } + + console.log('Creating API Keys Collection'); + try { + const collection = await db.createCollection(apiKeysTable); + + console.log('Creating API Key indexes'); + await collection.createIndex({ key_hash: 1 }, { unique: true }); + await collection.createIndex({ user_id: 1 }); + await collection.createIndex({ disabled: 1 }); + await collection.createIndex({ expires: 1 }); // for fast expiry checks + + console.log('Populating API Keys Collection'); + await collection.insertMany(init_data); + } + catch (err) { + console.log('CREATE ERROR:', err.code); + throw (err); + } + } +}; diff --git a/routes/api/v1/api_keys.js b/routes/api/v1/api_keys.js new file mode 100644 index 0000000..3d04f18 --- /dev/null +++ b/routes/api/v1/api_keys.js @@ -0,0 +1,289 @@ +// const Joi = require('joi'); +const Boom = require('@hapi/boom'); +const { hashApiKey, randomAsciiString } = require('../../../lib/utils'); + +const { + apiKeysTable +} = require('../../../config/db_constants'); + +const { + apiKeyCreatePayload, + apiKeyParam, + apiKeyQuery, + apiKeyUpdatePayload, + apiKeySuccessResponse, + authorizationHeader, + databaseInsertResponse +} = require('../../../lib/validations'); + +exports.plugin = { + name: 'apiKeys', + dependencies: ['hapi-auth-jwt2', 'hapi-mongodb'], + register: (server) => { + + const db = server.mongo.db; + const ObjectID = server.mongo.ObjectID; + + const _renameAndClearFields = (doc) => { + + //rename id + doc.id = doc._id; + delete doc._id; + + //remove fields entirely + delete doc.key_hash; + + return doc; + }; + + // ---------------- LIST API KEYS ---------------- + server.route({ + method: 'GET', + path: '/api_keys', + handler: async (request, h) => { + + // if the request includes a user_id that is not the current user's ID and the user id + if (request.params.user_id) { + if (!request.auth.credentials.roles.includes('admin') || request.auth.credentials.id !== request.params.user_id) { + return Boom.badRequest('The requesting user is unauthorized to make that request'); + } + } + + const userId = request.params.user_id || request.auth.credentials.id; + const result = await db.collection(apiKeysTable) + .find({ user_id: ObjectID(userId) }) + .toArray(); + + console.error(result); + + result.forEach(_renameAndClearFields); + + return h.response(result).code(200); + }, + config: { + auth: { + strategy: 'jwt', + scope: ['admin', 'read_apikeys'] + }, + validate: { + headers: authorizationHeader, + query: apiKeyQuery + }, + response: { + status: { + 200: apiKeySuccessResponse + } + }, + description: 'Return the API Keys based on query parameters', + notes: '
Requires authorization via: JWT token
\ +Available to: admin
', + tags: ['api_keys','api'] + } + }); + + server.route({ + method: 'GET', + path: '/api_keys/{id}', + handler: async (request, h) => { + + const { id } = request.params; + + const result = await db.collection(apiKeysTable).findOne({ _id: new ObjectID(id) }); + + if (!result) { + throw Boom.notFound('API Key not found'); + } + + if (!request.auth.credentials.roles.includes('admin') && request.auth.credentials.id !== result.user_id) { + return Boom.badRequest('The requesting user is unauthorized to make that request'); + } + + const cleanedResult = _renameAndClearFields(result); + + return h.response(cleanedResult).code(200); + }, + config: { + auth: { + strategy: 'jwt', + scope: ['admin', 'read_apikeys'] + }, + validate: { + headers: authorizationHeader, + params: apiKeyQuery + }, + response: { + status: { + 200: apiKeySuccessResponse + } + }, + description: 'Return the API Keys based on query parameters', + notes: 'Requires authorization via: JWT token
\ +Available to: admin
', + tags: ['api_keys','api'] + } + }); + + // ---------------- CREATE API KEY ---------------- + server.route({ + method: 'POST', + path: '/api_keys', + handler: async (request, h) => { + + const { label, roles = [], expires } = request.payload; + const apiKeyPlain = randomAsciiString(20); + const keyHash = await hashApiKey(apiKeyPlain); + + const result = await db.collection(apiKeysTable).insertOne({ + user_id: ObjectID(request.auth.credentials.id), + key_hash: keyHash, + label, + roles, + created: new Date(), + last_used: null, + disabled: false, + expires: expires ? new Date(expires) : null + }); + + return h.response({ + id: result.insertedId, + key: apiKeyPlain // show plain key ONCE + }).code(201); + }, + config: { + auth: { + strategy: 'jwt', + scope: ['admin', 'create_api_keys'] + }, + validate: { + headers: authorizationHeader, + payload: apiKeyCreatePayload, + failAction: (request, h, err) => { + + throw Boom.badRequest(err.message); + } + }, + response: { + status: { + 201: databaseInsertResponse + } + }, + + description: 'Create a new API key', + notes: 'Requires authorization via: JWT token
\ +Available to: admin
', + tags: ['api_keys','api'] + } + }); + + + // ---------------- PATCH / UPDATE LABEL OR EXPIRATION ---------------- + server.route({ + method: 'PATCH', + path: '/api_keys/{id}', + handler: async (request, h) => { + + const { id } = request.params; + + const result = await db.collection(apiKeysTable).findOne({ _id: new ObjectID(id) }); + + if (!result) { + throw Boom.notFound('API Key not found'); + } + + if (!request.auth.credentials.roles.includes('admin') && request.auth.credentials.id !== result.user_id) { + return Boom.badRequest('The requesting user is unauthorized to make that request'); + } + + const updates = request.payload; + + const updateDoc = {}; + if (updates.label !== undefined) { + updateDoc.label = updates.label; + } + + if (updates.expires !== undefined) { + updateDoc.expires = updates.expires ? new Date(updates.expires) : null; + } + + if (updates.disabled !== undefined) { + updateDoc.disabled = updates.disabled; + } + + await db.collection(apiKeysTable).updateOne( + { _id: new ObjectID(id) }, + { $set: updateDoc } + ); + + return h.response({ message: 'API Key updated' }).code(200); + }, + config: { + auth: { + strategy: 'jwt', + scope: ['admin', 'write_api_keys'] + }, + validate: { + headers: authorizationHeader, + params: apiKeyParam, + payload: apiKeyUpdatePayload, + failAction: (request, h, err) => { + + throw Boom.badRequest(err.message); + } + }, + response: { + status: { } + }, + description: 'Update an API key', + notes: 'Requires authorization via: JWT token
\ +Available to: admin
', + tags: ['api_keys','api'] + } + }); + + + // ---------------- DELETE / REVOKE API KEY ---------------- + server.route({ + method: 'DELETE', + path: '/api_keys/{id}', + handler: async (request, h) => { + + const { id } = request.params; + + const result = await db.collection(apiKeysTable).findOne({ _id: new ObjectID(id) }); + + if (!result) { + throw Boom.notFound('API Key not found'); + } + + if (!request.auth.credentials.roles.includes('admin') && request.auth.credentials.id !== result.user_id) { + return Boom.badRequest('The requesting user is unauthorized to make that request'); + } + + await db.collection(apiKeysTable).updateOne( + { _id: new ObjectID(id) }, + { $set: { disabled: true } } // soft delete + ); + + return h.response({ message: 'API Key disabled' }).code(200); + }, + config: { + auth: { + strategy: 'jwt', + scope: ['admin', 'create_api_keys'] + }, + validate: { + headers: authorizationHeader, + params: apiKeyParam + }, + response: { + status: {} + }, + description: 'Delete an API key', + notes: 'Requires authorization via: JWT token
\ +Available to: admin
', + tags: ['api_keys','api'] + } + }); + } +}; + diff --git a/routes/api/v1/auth.js b/routes/api/v1/auth.js index e1fe41e..b9602b7 100644 --- a/routes/api/v1/auth.js +++ b/routes/api/v1/auth.js @@ -65,6 +65,9 @@ const _rolesToScope = (roles) => { else if (role === 'cruise_manager') { return scope_accumulator.concat(['read_events', 'write_events', 'read_event_templates', 'write_event_templates', 'read_cruises', 'write_cruises', 'read_lowerings', 'write_lowerings', 'read_users', 'write_users']); } + else if (role === 'apikey_manager') { + return scope_accumulator.concat(['read_events', 'write_events', 'read_event_templates', 'write_event_templates', 'read_cruises', 'write_cruises', 'read_lowerings', 'write_lowerings', 'read_users', 'write_users', 'read_api_keys', 'write_api_keys']); + } return scope_accumulator; }, []); diff --git a/routes/api/v1/cruises.js b/routes/api/v1/cruises.js index 92c1484..36a7ffd 100644 --- a/routes/api/v1/cruises.js +++ b/routes/api/v1/cruises.js @@ -199,35 +199,37 @@ exports.plugin = { const cruises = await db.collection(cruisesTable).find(query).sort( { start_ts: -1 } ).skip(offset).limit(limit).toArray(); // console.log("cruises:", cruises); - if (cruises.length > 0) { + if (cruises.length === 0) { + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } - const mod_cruises = cruises.map((cruise) => { + return h.response([]).code(200); + } - try { - cruise.cruise_additional_meta.cruise_files = Fs.readdirSync(cruisePath + '/' + cruise._id); - } - catch (error) { - cruise.cruise_additional_meta.cruise_files = []; - } + const mod_cruises = cruises.map((cruise) => { - return _renameAndClearFields(cruise); - }); + try { + cruise.cruise_additional_meta.cruise_files = Fs.readdirSync(cruisePath + '/' + cruise._id); + } + catch (error) { + cruise.cruise_additional_meta.cruise_files = []; + } - if (request.query.format && request.query.format === 'csv') { + return _renameAndClearFields(cruise); + }); - const flat_cruises = flattenCruiseObjs(mod_cruises); - const csv_headers = buildCruiseCSVHeaders(flat_cruises); - const parser = new AsyncParser({ fields: csv_headers }, {}, {}); - const csv_results = await parser.parse(flat_cruises).promise(); + if (request.query.format && request.query.format === 'csv') { - return h.response(csv_results).code(200); - } + const flat_cruises = flattenCruiseObjs(mod_cruises); + const csv_headers = buildCruiseCSVHeaders(flat_cruises); + const parser = new AsyncParser({ fields: csv_headers }, {}, {}); + const csv_results = await parser.parse(flat_cruises).promise(); - return h.response(mod_cruises).code(200); + return h.response(csv_results).code(200); } - return Boom.notFound('No records found'); - + return h.response(mod_cruises).code(200); } catch (err) { return Boom.serverUnavailable('database error', err); @@ -235,7 +237,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_cruises'] }, validate: { @@ -329,7 +331,7 @@ exports.plugin = { return h.response(_renameAndClearFields(cruise)).code(200); } - return Boom.notFound('No records found'); + return Boom.notFound('No cruise record found'); } catch (err) { @@ -339,7 +341,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_cruises'] }, validate: { @@ -423,7 +425,7 @@ exports.plugin = { return h.response(_renameAndClearFields(cruise)).code(200); } - return Boom.notFound('No records found'); + return Boom.notFound('No cruise record found'); } catch (err) { @@ -433,7 +435,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_cruises'] }, validate: { @@ -476,11 +478,11 @@ exports.plugin = { try { const result = await db.collection(cruisesTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No cruise record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && result.cruise_hidden && (useAccessControl && typeof result.cruise_access_list !== 'undefined' && !result.cruise_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this cruise'); + return Boom.unauthorized('User not authorized to retrieve this cruise record'); } cruise = result; @@ -511,7 +513,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_cruises'] }, validate: { @@ -554,11 +556,11 @@ exports.plugin = { try { const result = await db.collection(cruisesTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No cruise record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && result.cruise_hidden && (useAccessControl && typeof result.cruise_access_list !== 'undefined' && !result.cruise_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this cruise'); + return Boom.unauthorized('User not authorized to retrieve this cruise record'); } cruise = result; @@ -575,7 +577,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_cruises'] }, validate: { @@ -695,7 +697,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'create_cruises'] }, validate: { @@ -712,7 +714,7 @@ exports.plugin = { } }, - description: 'Create a new event template', + description: 'Create a new cruise', notes: 'Requires authorization via: JWT token
\Available to: admin
', tags: ['cruises','api'] @@ -756,11 +758,11 @@ exports.plugin = { const result = await db.collection(cruisesTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No cruise record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && result.cruise_hidden && ( useAccessControl && typeof result.cruise_access_list !== 'undefined' && !result.cruise_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to edit this cruise'); + return Boom.unauthorized('User not authorized to edit this cruise record'); } // if a start date and/or stop date is provided, ensure the new date works with the existing date @@ -872,9 +874,8 @@ exports.plugin = { const loweringQuery = { start_ts: { '$gte': updatedCruise.start_ts }, stop_ts: { '$lt': updatedCruise.stop_ts } }; try { - console.error('here 2'); const cruiseLowerings = await db.collection(loweringsTable).find(loweringQuery).toArray(); - // console.log(cruiseLowerings); + cruiseLowerings.forEach((lowering) => { lowering.id = lowering._id; @@ -891,7 +892,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_cruises'] }, validate: { @@ -907,8 +908,12 @@ exports.plugin = { status: { } }, description: 'Update a cruise record', - notes: 'Requires authorization via: JWT token
\ -Available to: admin
', + notes: 'Requires authorization using either:
\ +Available to roles: admin, cruise_manager
', tags: ['cruises','api'] } }); @@ -940,7 +945,7 @@ exports.plugin = { cruise = await db.collection(cruisesTable).findOne(query); if (!cruise) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No cruise record found for id: ' + request.params.id); } } @@ -1015,7 +1020,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_cruises'] }, validate: { @@ -1058,7 +1063,7 @@ exports.plugin = { const result = await db.collection(cruisesTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No cruise record found for id: ' + request.params.id); } } catch (err) { @@ -1080,7 +1085,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'create_cruises'] }, validate: { @@ -1130,7 +1135,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin'] }, validate: { diff --git a/routes/api/v1/custom_vars.js b/routes/api/v1/custom_vars.js index 916a183..d4d009d 100644 --- a/routes/api/v1/custom_vars.js +++ b/routes/api/v1/custom_vars.js @@ -49,14 +49,13 @@ exports.plugin = { try { const results = await db.collection(customVarsTable).find(query).toArray(); - if (results.length > 0) { - - results.forEach(_renameAndClearFields); - - return h.response(results).code(200); + if (results.length === 0) { + return h.response([]).code(200); } - return Boom.notFound('No records found'); + results.forEach(_renameAndClearFields); + + return h.response(results).code(200); } catch (err) { @@ -65,7 +64,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -104,7 +103,7 @@ exports.plugin = { try { const result = await db.collection(customVarsTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No custom variable record found for id: ' + request.params.id); } const mod_result = _renameAndClearFields(result); @@ -116,7 +115,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -157,7 +156,7 @@ exports.plugin = { const result = await db.collection(customVarsTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No custom variable record found for id: ' + request.params.id); } custom_var_name = result.custom_var_name; @@ -182,7 +181,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_events'] }, validate: { diff --git a/routes/api/v1/event_aux_data.js b/routes/api/v1/event_aux_data.js index 8026afb..2ba0e51 100644 --- a/routes/api/v1/event_aux_data.js +++ b/routes/api/v1/event_aux_data.js @@ -69,11 +69,11 @@ exports.plugin = { const cruiseResult = await db.collection(cruisesTable).findOne({ _id: cruise_id }); if (!cruiseResult) { - return Boom.badRequest('Cruise not found for id' + request.params.id); + return Boom.badRequest('No cruise record found for id' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && cruiseResult.cruise_hidden && (useAccessControl && typeof cruiseResult.cruise_access_list !== 'undefined' && !cruiseResult.cruise_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this cruise'); + return Boom.unauthorized('User not authorized to retrieve this cruise record'); } cruise = cruiseResult; @@ -89,46 +89,45 @@ exports.plugin = { const results = await db.collection(eventsTable).find(eventQuery, { _id: 1 }).sort( { ts: 1 } ).toArray(); // EventID Filtering - if (results.length > 0) { - const query = {}; + if (results.length === 0) { + return h.response([]).code(200); + } - const eventIDs = results.map((event) => { + const query = {}; - return event._id; - }); - query.event_id = { $in: eventIDs }; + const eventIDs = results.map((event) => { - // Datasource Filtering - if (request.query.datasource) { - if (Array.isArray(request.query.datasource)) { - query.data_source = { $in: request.query.datasource }; - } - else { - query.data_source = request.query.datasource; - } + return event._id; + }); + query.event_id = { $in: eventIDs }; + + // Datasource Filtering + if (request.query.datasource) { + if (Array.isArray(request.query.datasource)) { + query.data_source = { $in: request.query.datasource }; + } + else { + query.data_source = request.query.datasource; } + } - const limit = (request.query.limit) ? request.query.limit : 0; - const offset = (request.query.offset) ? request.query.offset : 0; + const limit = (request.query.limit) ? request.query.limit : 0; + const offset = (request.query.offset) ? request.query.offset : 0; - try { - const eventAuxDataResults = await db.collection(eventAuxDataTable).find(query).skip(offset).limit(limit).toArray(); + try { + const eventAuxDataResults = await db.collection(eventAuxDataTable).find(query).skip(offset).limit(limit).toArray(); - if (eventAuxDataResults.length > 0) { - eventAuxDataResults.forEach(_renameAndClearFields); + if (eventAuxDataResults.length === 0) { + return h.response([]).code(200); + } - return h.response(eventAuxDataResults).code(200); - } + eventAuxDataResults.forEach(_renameAndClearFields); - return Boom.notFound('No records found'); + return h.response(eventAuxDataResults).code(200); - } - catch (err) { - return Boom.serverUnavailable('database error', err); - } } - else { - return Boom.notFound('No records found'); + catch (err) { + return Boom.serverUnavailable('database error', err); } } catch (err) { @@ -137,7 +136,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -171,11 +170,11 @@ exports.plugin = { const loweringResult = await db.collection(loweringsTable).findOne({ _id: ObjectID(request.params.id) }); if (!loweringResult) { - return Boom.notFound('lowering not found for that id'); + return Boom.notFound('No lowering record found for id' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && loweringResult.lowering_hidden && (useAccessControl && typeof loweringResult.lowering_access_list !== 'undefined' && !loweringResult.lowering_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this lowering'); + return Boom.unauthorized('User not authorized to retrieve this lowering record'); } lowering = loweringResult; @@ -190,46 +189,45 @@ exports.plugin = { const results = await db.collection(eventsTable).find(eventQuery, { _id: 1 }).sort( { ts: 1 } ).toArray(); // EventID Filtering - if (results.length > 0) { - const query = {}; - const eventIDs = results.map((event) => { + if (results.length === 0) { + return h.response([]).code(200); + } - return event._id; - }); - query.event_id = { $in: eventIDs }; + const query = {}; + const eventIDs = results.map((event) => { - // Datasource Filtering - if (request.query.datasource) { - if (Array.isArray(request.query.datasource)) { - query.data_source = { $in: request.query.datasource }; - } - else { - query.data_source = request.query.datasource; - } + return event._id; + }); + query.event_id = { $in: eventIDs }; + + // Datasource Filtering + if (request.query.datasource) { + if (Array.isArray(request.query.datasource)) { + query.data_source = { $in: request.query.datasource }; } + else { + query.data_source = request.query.datasource; + } + } - // Limiting & Offset - const limit = (request.query.limit) ? request.query.limit : 0; - const offset = (request.query.offset) ? request.query.offset : 0; + // Limiting & Offset + const limit = (request.query.limit) ? request.query.limit : 0; + const offset = (request.query.offset) ? request.query.offset : 0; - try { - const auxDataResults = await db.collection(eventAuxDataTable).find(query).skip(offset).limit(limit).toArray(); + try { + const auxDataResults = await db.collection(eventAuxDataTable).find(query).skip(offset).limit(limit).toArray(); - if (auxDataResults.length > 0) { - auxDataResults.forEach(_renameAndClearFields); + if (auxDataResults.length === 0) { + return h.response([]).code(200); + } - return h.response(auxDataResults).code(200); - } + auxDataResults.forEach(_renameAndClearFields); - return Boom.notFound('No records found'); + return h.response(auxDataResults).code(200); - } - catch (err) { - return Boom.serverUnavailable('database error', err); - } } - else { - return Boom.notFound('No records found'); + catch (err) { + return Boom.serverUnavailable('database error', err); } } catch (err) { @@ -238,7 +236,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -278,47 +276,45 @@ exports.plugin = { const results = await db.collection(eventsTable).find(eventQuery, { _id: 1 }).sort( { ts: 1 } ).toArray(); // EventID Filtering - if (results.length > 0) { - const query = {}; + if (results.length === 0) { + return h.response([]).code(200); + } - const eventIDs = results.map((event) => { + const query = {}; - return new ObjectID(event._id); - }); - query.event_id = { $in: eventIDs }; + const eventIDs = results.map((event) => { - // Datasource Filtering - if (request.query.datasource) { - if (Array.isArray(request.query.datasource)) { - query.data_source = { $in: request.query.datasource }; - } - else { - query.data_source = request.query.datasource; - } - } + return new ObjectID(event._id); + }); + query.event_id = { $in: eventIDs }; - const limit = (request.query.limit) ? request.query.limit : 0; - const offset = (request.query.offset) ? request.query.offset : 0; + // Datasource Filtering + if (request.query.datasource) { + if (Array.isArray(request.query.datasource)) { + query.data_source = { $in: request.query.datasource }; + } + else { + query.data_source = request.query.datasource; + } + } - try { - const auxDataResults = await db.collection(eventAuxDataTable).find(query).skip(offset).limit(limit).toArray(); + const limit = (request.query.limit) ? request.query.limit : 0; + const offset = (request.query.offset) ? request.query.offset : 0; - if (auxDataResults.length > 0) { + try { + const auxDataResults = await db.collection(eventAuxDataTable).find(query).skip(offset).limit(limit).toArray(); - auxDataResults.forEach(_renameAndClearFields); + if (auxDataResults.length === 0) { + return h.response([]).code(200); + } - return h.response(auxDataResults).code(200); - } + auxDataResults.forEach(_renameAndClearFields); - return Boom.notFound('No records found'); + return h.response(auxDataResults).code(200); - } - catch (err) { - return Boom.serverUnavailable('database error', err); - } } - else { - return Boom.notFound('No records found'); + catch (err) { + return Boom.serverUnavailable('database error', err); } } catch (err) { @@ -360,13 +356,13 @@ exports.plugin = { try { const results = await db.collection(eventAuxDataTable).find(query).skip(offset).limit(limit).toArray(); - if (results.length > 0) { - results.forEach(_renameAndClearFields); - - return h.response(results).code(200); + if (results.length === 0) { + return h.response([]).code(200); } - return Boom.notFound('No records found'); + results.forEach(_renameAndClearFields); + + return h.response(results).code(200); } catch (err) { @@ -376,7 +372,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -408,7 +404,7 @@ exports.plugin = { try { const result = await db.collection(eventAuxDataTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No aux_data record found for id: ' + request.params.id); } const mod_result = _renameAndClearFields(result); @@ -420,7 +416,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -495,7 +491,7 @@ exports.plugin = { const queryResult = await db.collection(eventsTable).findOne(query); if (!queryResult) { - return Boom.badRequest('event not found'); + return Boom.badRequest('event record not found'); } query = { event_id: event_aux_data.event_id, data_source: event_aux_data.data_source }; @@ -557,7 +553,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_events'] }, validate: { @@ -600,7 +596,7 @@ exports.plugin = { result = await db.collection(eventAuxDataTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No aux_data record found for id: ' + request.params.id); } event_aux_data = request.payload; @@ -651,7 +647,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_events'] }, validate: { @@ -689,7 +685,7 @@ exports.plugin = { try { const auxData = await db.collection(eventAuxDataTable).findOne(query); if (!auxData) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No aux_data record found for id: ' + request.params.id); } } catch (err) { @@ -707,7 +703,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_events'] }, validate: { diff --git a/routes/api/v1/event_exports.js b/routes/api/v1/event_exports.js index 1dc1d9d..1366d64 100644 --- a/routes/api/v1/event_exports.js +++ b/routes/api/v1/event_exports.js @@ -59,7 +59,7 @@ exports.plugin = { const cruiseResult = await db.collection(cruisesTable).findOne({ _id: ObjectID(request.params.id) }); if (!cruiseResult) { - return Boom.notFound('cruise not found for that id'); + return Boom.notFound('No cruise record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && cruiseResult.cruise_hidden && (useAccessControl && typeof cruiseResult.cruise_access_list !== 'undefined' && !cruiseResult.cruise_access_list.includes(request.auth.credentials.id))) { @@ -104,66 +104,70 @@ exports.plugin = { return Boom.serverUnavailable('database error'); } - if (results.length > 0) { + if (results.length === 0) { + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } - // datasource filtering - if (request.query.datasource) { + return h.response([]).code(200); + } - const datasource_query = {}; + // datasource filtering + if (request.query.datasource) { - const eventIDs = results.map((event) => event._id); + const datasource_query = {}; - datasource_query.event_id = { $in: eventIDs }; + const eventIDs = results.map((event) => event._id); - if (Array.isArray(request.query.datasource)) { - datasource_query.data_source = { $in: request.query.datasource }; - } - else { - datasource_query.data_source = request.query.datasource; - } + datasource_query.event_id = { $in: eventIDs }; - let aux_data_results = []; - try { - aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } + if (Array.isArray(request.query.datasource)) { + datasource_query.data_source = { $in: request.query.datasource }; + } + else { + datasource_query.data_source = request.query.datasource; + } - const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); + let aux_data_results = []; + try { + aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); + } + catch (err) { + console.log(err); + return Boom.serverUnavailable('database error'); + } - results = results.filter((event) => { + const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); - return (aux_data_eventID_set.has(String(event._id))) ? event : null; - }); + results = results.filter((event) => { - } + return (aux_data_eventID_set.has(String(event._id))) ? event : null; + }); - results.forEach(_renameAndClearFields); + } - if (request.query.add_record_ids) { - results = await addEventRecordIDs(request, results); - } + results.forEach(_renameAndClearFields); - if (request.query.format && request.query.format === 'csv') { + if (request.query.add_record_ids) { + results = await addEventRecordIDs(request, results); + } - const flat_events = flattenEventObjs(results); - const csv_headers = buildEventCSVHeaders(flat_events); - const parser = new AsyncParser({ fields: csv_headers }, {}, {}); - const csv_results = await parser.parse(flat_events).promise(); + if (request.query.format && request.query.format === 'csv') { - return h.response(csv_results).code(200); - } + const flat_events = flattenEventObjs(results); + const csv_headers = buildEventCSVHeaders(flat_events); + const parser = new AsyncParser({ fields: csv_headers }, {}, {}); + const csv_results = await parser.parse(flat_events).promise(); - return h.response(results).code(200); + return h.response(csv_results).code(200); } - return Boom.notFound('No records found'); + return h.response(results).code(200); + }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -197,11 +201,11 @@ exports.plugin = { const loweringResult = await db.collection(loweringsTable).findOne({ _id: ObjectID(request.params.id) }); if (!loweringResult) { - return Boom.notFound('lowering not found for that id'); + return Boom.notFound('No lowering record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && loweringResult.lowering_hidden && (useAccessControl && typeof loweringResult.lowering_access_list !== 'undefined' && !loweringResult.lowering_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this lowering'); + return Boom.unauthorized('User not authorized to retrieve this lowering record'); } lowering = loweringResult; @@ -213,7 +217,7 @@ exports.plugin = { } if (lowering.lowering_hidden && !request.auth.credentials.scope.includes('admin')) { - return Boom.unauthorized('User not authorized to retrieve hidden lowerings'); + return Boom.unauthorized('User not authorized to retrieve hidden lowering records'); } const query = buildEventsQuery(request, lowering.start_ts, lowering.stop_ts); @@ -246,65 +250,65 @@ exports.plugin = { return Boom.serverUnavailable('database error'); } - if (results.length > 0) { - - // datasource filtering - if (request.query.datasource) { + if (results.length === 0) { + return h.response([]).code(200); + } - const datasource_query = {}; + // datasource filtering + if (request.query.datasource) { - const eventIDs = results.map((event) => event._id); + const datasource_query = {}; - datasource_query.event_id = { $in: eventIDs }; + const eventIDs = results.map((event) => event._id); - if (Array.isArray(request.query.datasource)) { - datasource_query.data_source = { $in: request.query.datasource }; - } - else { - datasource_query.data_source = request.query.datasource; - } + datasource_query.event_id = { $in: eventIDs }; - let aux_data_results = []; - try { - aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } + if (Array.isArray(request.query.datasource)) { + datasource_query.data_source = { $in: request.query.datasource }; + } + else { + datasource_query.data_source = request.query.datasource; + } - const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); + let aux_data_results = []; + try { + aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); + } + catch (err) { + console.log(err); + return Boom.serverUnavailable('database error'); + } - results = results.filter((event) => { + const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); - return (aux_data_eventID_set.has(String(event._id))) ? event : null; - }); - } + results = results.filter((event) => { - results.forEach(_renameAndClearFields); + return (aux_data_eventID_set.has(String(event._id))) ? event : null; + }); + } - if (request.query.add_record_ids) { - results = await addEventRecordIDs(request, results); - } + results.forEach(_renameAndClearFields); - if (request.query.format && request.query.format === 'csv') { + if (request.query.add_record_ids) { + results = await addEventRecordIDs(request, results); + } - const flat_events = flattenEventObjs(results); - const csv_headers = buildEventCSVHeaders(flat_events); - const parser = new AsyncParser({ fields: csv_headers }, {}, {}); - const csv_results = await parser.parse(flat_events).promise(); + if (request.query.format && request.query.format === 'csv') { - return h.response(csv_results).code(200); - } + const flat_events = flattenEventObjs(results); + const csv_headers = buildEventCSVHeaders(flat_events); + const parser = new AsyncParser({ fields: csv_headers }, {}, {}); + const csv_results = await parser.parse(flat_events).promise(); - return h.response(results).code(200); + return h.response(csv_results).code(200); } - return Boom.notFound('No records found'); + return h.response(results).code(200); + }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -378,12 +382,12 @@ exports.plugin = { try { const results = await db.collection(eventsTable).aggregate(aggregate, { allowDiskUse: true }).skip(offset).toArray(); - if (results.length > 0) { - results.forEach(_renameAndClearFields); - return h.response(results).code(200); + if (results.length === 0) { + return h.response([]).code(200); } - return Boom.notFound('No records found'); + results.forEach(_renameAndClearFields); + return h.response(results).code(200); } catch (err) { @@ -417,28 +421,33 @@ exports.plugin = { try { let results = await db.collection(eventsTable).aggregate(aggregate, { allowDiskUse: true }).skip(offset).toArray(); - if (results.length > 0) { - results.forEach(_renameAndClearFields); - - if (request.query.add_record_ids) { - results = await addEventRecordIDs(request, results); + if (results.length === 0) { + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); } - if (request.query.format && request.query.format === 'csv') { + return h.response([]).code(200); + } - const flat_events = flattenEventObjs(results); - const csv_headers = buildEventCSVHeaders(flat_events); - const parser = new AsyncParser({ fields: csv_headers }, {}, {}); - const csv_results = await parser.parse(flat_events).promise(); + results.forEach(_renameAndClearFields); + if (request.query.add_record_ids) { + results = await addEventRecordIDs(request, results); + } - return h.response(csv_results).code(200); - } + if (request.query.format && request.query.format === 'csv') { + + const flat_events = flattenEventObjs(results); + const csv_headers = buildEventCSVHeaders(flat_events); + const parser = new AsyncParser({ fields: csv_headers }, {}, {}); + const csv_results = await parser.parse(flat_events).promise(); - return h.response(results).code(200); + + return h.response(csv_results).code(200); } - return Boom.notFound('No records found'); + return h.response(results).code(200); + } catch (err) { console.log(err); @@ -448,7 +457,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -494,7 +503,7 @@ exports.plugin = { let results = await db.collection(eventsTable).aggregate(aggregate, { allowDiskUse: true }).toArray(); if (results.length === 0) { - return Boom.notFound('No records found'); + return Boom.notFound('No event records found'); } results.forEach(_renameAndClearFields); @@ -522,7 +531,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { diff --git a/routes/api/v1/event_templates.js b/routes/api/v1/event_templates.js index a417ed5..87de0f7 100644 --- a/routes/api/v1/event_templates.js +++ b/routes/api/v1/event_templates.js @@ -59,16 +59,16 @@ exports.plugin = { try { const results = await db.collection(eventTemplatesTable).find(query).sort(sort).skip(offset).limit(limit).toArray(); - if (results.length > 0) { - results.forEach((result) => { + if (results.length === 0) { + return h.response([]).code(200); + } - return _renameAndClearFields(result, request.auth.credentials.scope.some((role) => ['admin', 'write_event_templates'].includes(role))); - }); + results.forEach((result) => { - return h.response(results).code(200); - } + return _renameAndClearFields(result, request.auth.credentials.scope.some((role) => ['admin', 'write_event_templates'].includes(role))); + }); - return Boom.notFound('No records found'); + return h.response(results).code(200); } catch (err) { @@ -78,7 +78,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_event_templates'] }, validate: { @@ -118,11 +118,11 @@ exports.plugin = { const result = await db.collection(eventTemplatesTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No event template record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && result.admin_only) { - return Boom.notFound('template only available to admin users'); + return Boom.notFound('template record only available to admin users'); } return h.response(_renameAndClearFields(result, request.auth.credentials.scope.some((role) => ['admin', 'write_event_templates'].includes(role)))).code(200); @@ -134,7 +134,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_event_templates'] }, validate: { @@ -211,7 +211,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_event_templates'] }, validate: { @@ -257,7 +257,7 @@ exports.plugin = { const result = await db.collection(eventTemplatesTable).findOne(query); if (!result) { - return Boom.badRequest('No record found for id: ' + request.params.id ); + return Boom.badRequest('No event template record found for id: ' + request.params.id ); } event_template = { ...result, ...request.payload }; @@ -284,7 +284,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_event_templates'] }, validate: { @@ -326,11 +326,11 @@ exports.plugin = { try { const result = await db.collection(eventTemplatesTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id ); + return Boom.notFound('No event template record found for id: ' + request.params.id ); } if (result.system_template && !request.auth.credentials.scope.includes('admin')) { - return Boom.unauthorized('user does not have permission to delete system templates'); + return Boom.unauthorized('user does not have permission to delete system template records'); } } catch (err) { @@ -351,7 +351,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_event_templates'] }, validate: { diff --git a/routes/api/v1/events.js b/routes/api/v1/events.js index 193fe78..159e065 100644 --- a/routes/api/v1/events.js +++ b/routes/api/v1/events.js @@ -49,6 +49,50 @@ const _renameAndClearFields = (doc) => { return doc; }; +const _deleteEventsWithAuxData = async (db, server, query = {}, limit = 0, offset = 0, sort = { ts: 1 }) => { + + // Find the events + const eventsToDelete = await db.collection(eventsTable) + .find(query) + .sort(sort) + .skip(offset) + .limit(limit) + .toArray(); + + if (eventsToDelete.length === 0) { + return { deletedCount: 0 }; + } + + const eventIDs = eventsToDelete.map((x) => x._id); + + // Fetch and group aux_data + const allAuxData = await db.collection(eventAuxDataTable) + .find({ event_id: { $in: eventIDs } }) + .toArray(); + + const eventIDToAuxData = allAuxData.reduce((dict, auxData) => { + + const eventId = auxData.event_id.toString(); + if (!dict[eventId]) { + dict[eventId] = []; + } + + dict[eventId].push(auxData); + return dict; + }, {}); + + // Publish deleteEvents message for each event with its aux data + // Let the aux data managers handle aux data deletion + for (const event of eventsToDelete) { + event.aux_data = eventIDToAuxData[event._id.toString()] || []; + server.publish('/ws/status/deleteEvents', _renameAndClearFields(event)); + } + + // Delete event records + const results = await db.collection(eventsTable).deleteMany({ _id: { $in: eventIDs } }); + + return { deletedCount: results.deletedCount }; +}; exports.plugin = { name: 'routes-api-events', @@ -77,7 +121,7 @@ exports.plugin = { } if (!request.auth.credentials.scope.includes('admin') && cruiseResult.cruise_hidden && (useAccessControl && typeof cruiseResult.cruise_access_list !== 'undefined' && !cruiseResult.cruise_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this cruise'); + return Boom.unauthorized('User not authorized to retrieve this cruise record'); } cruise = cruiseResult; @@ -104,7 +148,11 @@ exports.plugin = { } if (results.length === 0) { - return Boom.notFound('No records found' ); + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } + + return h.response([]).code(200); } // --------- Data source filtering @@ -166,7 +214,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -201,11 +249,11 @@ exports.plugin = { const cruiseResult = await db.collection(cruisesTable).findOne({ _id: ObjectID(request.params.id) }); if (!cruiseResult) { - return Boom.badRequest('No record cruise found for id: ' + request.params.id ); + return Boom.badRequest('No cruise record found for id: ' + request.params.id ); } if (!request.auth.credentials.scope.includes('admin') && cruiseResult.cruise_hidden && (useAccessControl && typeof cruiseResult.cruise_access_list !== 'undefined' && !cruiseResult.cruise_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this cruise'); + return Boom.unauthorized('User not authorized to retrieve this cruise record'); } cruise = cruiseResult; @@ -216,7 +264,7 @@ exports.plugin = { } if (cruise.cruise_hidden && !request.auth.credentials.scope.includes('admin')) { - return Boom.unauthorized('User not authorized to retrieve hidden cruises'); + return Boom.unauthorized('User not authorized to retrieve hidden cruise records'); } const query = buildEventsQuery(request, cruise.start_ts, cruise.stop_ts); @@ -232,56 +280,52 @@ exports.plugin = { return Boom.serverUnavailable('database error'); } - if (results.length > 0) { - - // --------- Data source filtering - if (request.query.datasource) { - - const datasource_query = {}; + if (typeof request.query.datasource === 'undefined') { + return h.response({ events: results.length }).code(200); + } - const eventIDs = results.map((event) => event._id); + // --------- Data source filtering + const datasource_query = {}; - datasource_query.event_id = { $in: eventIDs }; + const eventIDs = results.map((event) => event._id); - if (Array.isArray(request.query.datasource)) { - const regex_query = request.query.datasource.map((datasource) => { + datasource_query.event_id = { $in: eventIDs }; - const return_regex = new RegExp(datasource, 'i'); - return return_regex; - }); + if (Array.isArray(request.query.datasource)) { + const regex_query = request.query.datasource.map((datasource) => { - datasource_query.data_source = { $in: regex_query }; - } - else { - datasource_query.data_source = RegExp(request.query.datasource, 'i'); - } - - let aux_data_results = []; - try { - aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } + const return_regex = new RegExp(datasource, 'i'); + return return_regex; + }); - const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); + datasource_query.data_source = { $in: regex_query }; + } + else { + datasource_query.data_source = RegExp(request.query.datasource, 'i'); + } - results = results.filter((event) => { + let aux_data_results = []; + try { + aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); + } + catch (err) { + console.log(err); + return Boom.serverUnavailable('database error'); + } - return (aux_data_eventID_set.has(String(event._id))) ? event : null; - }); + const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); - } + results = results.filter((event) => { - return h.response({ events: results.length }).code(200); - } + return (aux_data_eventID_set.has(String(event._id))) ? event : null; + }); return h.response({ events: results.length }).code(200); + }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -315,11 +359,11 @@ exports.plugin = { const loweringResult = await db.collection(loweringsTable).findOne({ _id: ObjectID(request.params.id) }); if (!loweringResult) { - return Boom.badRequest('No record lowering found for id: ' + request.params.id ); + return Boom.badRequest('No lowering record found for id: ' + request.params.id ); } if (!request.auth.credentials.scope.includes('admin') && loweringResult.lowering_hidden && (useAccessControl && typeof loweringResult.lowering_access_list !== 'undefined' && !loweringResult.lowering_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this lowering'); + return Boom.unauthorized('User not authorized to retrieve this lowering record'); } lowering = loweringResult; @@ -346,7 +390,11 @@ exports.plugin = { } if (results.length === 0) { - return Boom.notFound('No records found' ); + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } + + return h.response([]).code(200); } // --------- Data source filtering @@ -408,7 +456,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -443,11 +491,11 @@ exports.plugin = { const loweringResult = await db.collection(loweringsTable).findOne({ _id: ObjectID(request.params.id) }); if (!loweringResult) { - return Boom.badRequest('No record lowering found for id: ' + request.params.id ); + return Boom.badRequest('No lowering record found for id: ' + request.params.id ); } if (!request.auth.credentials.scope.includes('admin') && loweringResult.lowering_hidden && (useAccessControl && typeof loweringResult.lowering_access_list !== 'undefined' && !loweringResult.lowering_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this lowering'); + return Boom.unauthorized('User not authorized to retrieve this lowering record'); } lowering = loweringResult; @@ -470,56 +518,51 @@ exports.plugin = { return Boom.serverUnavailable('database error'); } - if (results.length > 0) { - - // --------- Data source filtering - if (request.query.datasource) { - - const datasource_query = {}; - - const eventIDs = results.map((event) => event._id); + // --------- Data source filtering + if (!request.query.datasource) { + return h.response({ events: results.length }).code(200); + } - datasource_query.event_id = { $in: eventIDs }; + const datasource_query = {}; - if (Array.isArray(request.query.datasource)) { - const regex_query = request.query.datasource.map((datasource) => { + const eventIDs = results.map((event) => event._id); - const return_regex = new RegExp(datasource, 'i'); - return return_regex; - }); + datasource_query.event_id = { $in: eventIDs }; - datasource_query.data_source = { $in: regex_query }; - } - else { - datasource_query.data_source = RegExp(request.query.datasource, 'i'); - } + if (Array.isArray(request.query.datasource)) { + const regex_query = request.query.datasource.map((datasource) => { - let aux_data_results = []; - try { - aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } + const return_regex = new RegExp(datasource, 'i'); + return return_regex; + }); - const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); + datasource_query.data_source = { $in: regex_query }; + } + else { + datasource_query.data_source = RegExp(request.query.datasource, 'i'); + } - results = results.filter((event) => { + let aux_data_results = []; + try { + aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); + } + catch (err) { + console.log(err); + return Boom.serverUnavailable('database error'); + } - return (aux_data_eventID_set.has(String(event._id))) ? event : null; - }); + const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); - } + results = results.filter((event) => { - return h.response({ events: results.length }).code(200); - } + return (aux_data_eventID_set.has(String(event._id))) ? event : null; + }); return h.response({ events: results.length }).code(200); }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -593,12 +636,16 @@ exports.plugin = { const results = await db.collection(eventsTable).find(query).sort(sort).skip(offset).limit(limit).toArray(); // console.log("results:", results); - if (results.length > 0) { - results.forEach(_renameAndClearFields); - return h.response(results).code(200); + if (results.length === 0) { + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } + + return h.response([]).code(200); } - return Boom.notFound('No records found' ); + results.forEach(_renameAndClearFields); + return h.response(results).code(200); } catch (err) { @@ -619,7 +666,11 @@ exports.plugin = { // console.log("results:", results); if (results.length === 0) { - return Boom.notFound('No records found' ); + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } + + return h.response([]).code(200); } results.forEach(_renameAndClearFields); @@ -647,7 +698,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -741,7 +792,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -761,7 +812,6 @@ exports.plugin = { }); - server.route({ method: 'GET', path: '/events/{id}', @@ -776,7 +826,7 @@ exports.plugin = { let result = await db.collection(eventsTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id ); + return Boom.notFound('No event record found for id: ' + request.params.id ); } result = _renameAndClearFields(result); @@ -804,7 +854,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -909,7 +959,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_events'] }, validate: { @@ -1005,14 +1055,13 @@ exports.plugin = { const updatedEvent = _renameAndClearFields(result.value); if (time_change) { - server.publish('/ws/status/deleteEvents', updatedEvent); - // delete any aux_data const aux_data_query = { event_id: updatedEvent.id }; - // console.log(result.value); - // console.log(aux_data_query); - await db.collection(eventAuxDataTable).deleteMany(aux_data_query); + const aux_data_result = await db.collection(eventAuxDataTable).find(aux_data_query).toArray(); + updatedEvent.aux_data = aux_data_result; + server.publish('/ws/status/deleteEvents', _renameAndClearFields(updatedEvent)); + // console.log(del_results); server.publish('/ws/status/newEvents', updatedEvent); @@ -1032,7 +1081,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_events'] }, validate: { @@ -1103,42 +1152,13 @@ exports.plugin = { const offset = (request.query.offset) ? request.query.offset : 0; const sort = (request.query.sort === 'newest') ? { ts: -1 } : { ts: 1 }; - let eventIDs = []; - - // find the events - try { - const results = await db.collection(eventsTable).find(query).sort(sort).project({ _id: 1 }).skip(offset).limit(limit).toArray(); // should return just the ids - // console.log("results:",results); - - if (results.length === 0) { - return h.response({ deletedCount: 0 }).code(200); - } - - eventIDs = results.map((x) => x._id); - // console.log("eventIDs:",eventIDs); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } - - // delete the aux_data records try { - await db.collection(eventAuxDataTable).deleteMany({ event_id: { $in: eventIDs } }); + const result = await _deleteEventsWithAuxData(db, server, query, limit, offset, sort); + return h.response(result).code(200); } catch (err) { console.log(err); - return Boom.serverUnavailable('database error'); - } - - // delete the event records - try { - const results = await db.collection(eventsTable).deleteMany({ _id: { $in: eventIDs } }); - return h.response({ deletedCount: results.deletedCount }).code(200); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); + return Boom.serverUnavailable('database error', err); } } else { @@ -1148,48 +1168,20 @@ exports.plugin = { const offset = (request.query.offset) ? request.query.offset : 0; const sort = (request.query.sort === 'newest') ? { ts: -1 } : { ts: 1 }; - let eventIDs = []; - - // find the events try { - const results = await db.collection(eventsTable).find(query).sort(sort).project({ _id: 1 }).skip(offset).limit(limit).toArray(); - // console.log("results:", results); - - if (results.length === 0) { - return h.response({ deletedCount: 0 }).code(200); - } - - eventIDs = results.map((x) => x._id); - // console.log("eventIDs:",eventIDs); + const result = await _deleteEventsWithAuxData(db, server, query, limit, offset, sort); + return h.response(result).code(200); } catch (err) { console.log(err); - return Boom.serverUnavailable('database error'); + return Boom.serverUnavailable('database error', err); } - // delete the aux_data records - try { - await db.collection(eventAuxDataTable).deleteMany({ event_id: { $in: eventIDs } }); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } - - // delete the event records - try { - const results = await db.collection(eventsTable).deleteMany({ _id: { $in: eventIDs } }); - return h.response({ deletedCount: results.deletedCount }).code(200); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } } }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin'] }, validate: { @@ -1231,7 +1223,7 @@ exports.plugin = { const result = await db.collection(eventsTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id ); + return Boom.notFound('No event record found for id: ' + request.params.id ); } event = result; @@ -1260,21 +1252,13 @@ exports.plugin = { return Boom.serverUnavailable('database error'); } - try { - await db.collection(eventAuxDataTable).deleteMany({ event_id: new ObjectID(request.params.id) }); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } - server.publish('/ws/status/deleteEvents', _renameAndClearFields(event)); return h.response().code(204); }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_events'] }, validate: { @@ -1300,25 +1284,17 @@ exports.plugin = { // const ObjectID = request.mongo.ObjectID; try { - await db.collection(eventsTable).deleteMany(); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } - - try { - await db.collection(eventAuxDataTable).deleteMany(); - return h.response().code(204); + const result = await _deleteEventsWithAuxData(db, server); + return h.response(result).code(200); } catch (err) { console.log(err); - return Boom.serverUnavailable('database error'); + return Boom.serverUnavailable('database error', err); } }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin'] }, validate: { diff --git a/routes/api/v1/external_calls.js b/routes/api/v1/external_calls.js index 004ef05..0bfbe5f 100644 --- a/routes/api/v1/external_calls.js +++ b/routes/api/v1/external_calls.js @@ -146,7 +146,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin'] }, validate: { @@ -194,7 +194,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin'] }, validate: { diff --git a/routes/api/v1/lowerings.js b/routes/api/v1/lowerings.js index 3ebccd0..39047a8 100644 --- a/routes/api/v1/lowerings.js +++ b/routes/api/v1/lowerings.js @@ -122,7 +122,7 @@ exports.plugin = { query.lowering_hidden = request.query.hidden; } else if (request.query.hidden) { - return Boom.unauthorized('User not authorized to retrieve hidden lowerings'); + return Boom.unauthorized('User not authorized to retrieve hidden lowering records'); } else { query.lowering_hidden = false; @@ -187,34 +187,37 @@ exports.plugin = { try { const lowerings = await db.collection(loweringsTable).find(query).sort( { start_ts: -1 } ).skip(offset).limit(limit).toArray(); - if (lowerings.length > 0) { + if (lowerings.length === 0) { + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } - const mod_lowerings = lowerings.map((lowering) => { + return h.response([]).code(200); + } - try { - lowering.lowering_additional_meta.lowering_files = Fs.readdirSync(loweringPath + '/' + lowering._id); - } - catch (error) { - lowering.lowering_additional_meta.lowering_files = []; - } + const mod_lowerings = lowerings.map((lowering) => { - return _renameAndClearFields(lowering); - }); + try { + lowering.lowering_additional_meta.lowering_files = Fs.readdirSync(loweringPath + '/' + lowering._id); + } + catch (error) { + lowering.lowering_additional_meta.lowering_files = []; + } - if (request.query.format && request.query.format === 'csv') { + return _renameAndClearFields(lowering); + }); - const flat_lowerings = flattenLoweringObjs(mod_lowerings); - const csv_headers = buildLoweringCSVHeaders(flat_lowerings); - const parser = new AsyncParser({ fields: csv_headers }, {}, {}); - const csv_results = await parser.parse(flat_lowerings).promise(); + if (request.query.format && request.query.format === 'csv') { - return h.response(csv_results).code(200); - } + const flat_lowerings = flattenLoweringObjs(mod_lowerings); + const csv_headers = buildLoweringCSVHeaders(flat_lowerings); + const parser = new AsyncParser({ fields: csv_headers }, {}, {}); + const csv_results = await parser.parse(flat_lowerings).promise(); - return h.response(mod_lowerings).code(200); + return h.response(csv_results).code(200); } - return Boom.notFound('No records found'); + return h.response(mod_lowerings).code(200); } catch (err) { @@ -224,7 +227,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_lowerings'] }, validate: { @@ -334,34 +337,37 @@ exports.plugin = { try { const lowerings = await db.collection(loweringsTable).find(query).sort( { start_ts: -1 } ).skip(offset).limit(limit).toArray(); - if (lowerings.length > 0) { + if (lowerings.length === 0) { + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } - const mod_lowerings = lowerings.map((result) => { + return h.response([]).code(200); + } - try { - result.lowering_additional_meta.lowering_files = Fs.readdirSync(loweringPath + '/' + result._id); - } - catch (error) { - result.lowering_additional_meta.lowering_files = []; - } + const mod_lowerings = lowerings.map((result) => { - return _renameAndClearFields(result); - }); + try { + result.lowering_additional_meta.lowering_files = Fs.readdirSync(loweringPath + '/' + result._id); + } + catch (error) { + result.lowering_additional_meta.lowering_files = []; + } - if (request.query.format && request.query.format === 'csv') { + return _renameAndClearFields(result); + }); - const flat_lowerings = flattenLoweringObjs(mod_lowerings); - const csv_headers = buildLoweringCSVHeaders(flat_lowerings); - const parser = new AsyncParser({ fields: csv_headers }, {}, {}); - const csv_results = await parser.parse(flat_lowerings).promise(); + if (request.query.format && request.query.format === 'csv') { - return h.response(csv_results).code(200); - } + const flat_lowerings = flattenLoweringObjs(mod_lowerings); + const csv_headers = buildLoweringCSVHeaders(flat_lowerings); + const parser = new AsyncParser({ fields: csv_headers }, {}, {}); + const csv_results = await parser.parse(flat_lowerings).promise(); - return h.response(mod_lowerings).code(200); + return h.response(csv_results).code(200); } - return Boom.notFound('No records found'); + return h.response(mod_lowerings).code(200); } catch (err) { @@ -371,7 +377,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_lowerings'] }, validate: { @@ -465,7 +471,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_lowerings'] }, validate: { @@ -507,7 +513,7 @@ exports.plugin = { try { const result = await db.collection(loweringsTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No lowering record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && result.lowering_hidden && (useAccessControl && typeof result.lowering_access_list !== 'undefined' && !result.lowering_access_list.includes(request.auth.credentials.id))) { @@ -543,7 +549,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_lowerings'] }, validate: { @@ -586,11 +592,11 @@ exports.plugin = { try { const result = await db.collection(loweringsTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No lowering record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && result.lowering_hidden && (useAccessControl && typeof result.lowering_access_list !== 'undefined' && !result.lowering_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this lowering'); + return Boom.unauthorized('User not authorized to retrieve this lowering record'); } lowering = result; @@ -608,7 +614,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_lowerings'] }, validate: { @@ -722,7 +728,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'create_lowerings'] }, validate: { @@ -784,11 +790,11 @@ exports.plugin = { const result = await db.collection(loweringsTable).findOne(query); if (!result) { - return Boom.badRequest('No record found for id: ' + request.params.id); + return Boom.badRequest('No lowering record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && result.lowering_hidden && ( useAccessControl && typeof result.lowering_access_list !== 'undefined' && !result.lowering_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to edit this lowering'); + return Boom.unauthorized('User not authorized to edit this lowering record'); } // if a start date and/or stop date is provided, ensure the new date works with the existing date @@ -872,7 +878,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_lowerings'] }, validate: { @@ -922,7 +928,7 @@ exports.plugin = { lowering = await db.collection(loweringsTable).findOne(query); if (!lowering) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No lowering record found for id: ' + request.params.id); } } @@ -994,7 +1000,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_lowerings'] }, validate: { @@ -1037,7 +1043,7 @@ exports.plugin = { try { const result = await db.collection(loweringsTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No lowering record found for id: ' + request.params.id); } } catch (err) { @@ -1056,7 +1062,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'create_lowerings'] }, validate: { @@ -1106,7 +1112,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin'] }, validate: { diff --git a/routes/api/v1/users.js b/routes/api/v1/users.js index 724e41b..c2eebdd 100644 --- a/routes/api/v1/users.js +++ b/routes/api/v1/users.js @@ -13,6 +13,7 @@ const { } = require('../../../config/server_settings'); const { + apiKeysTable, cruisesTable, loweringsTable, usersTable @@ -406,6 +407,10 @@ exports.plugin = { try { await db.collection(usersTable).updateOne(query, { $set: user }); + if (typeof query.disabled === 'boolean' && !query.disabled) { + await db.collection(apiKeysTable).updateMany({ user_id: query._id }, { $set: { disabled: false } }); + } + return h.response().code(204); } catch (err) { @@ -473,6 +478,7 @@ exports.plugin = { try { await db.collection(usersTable).deleteOne(query); + await db.collection(apiKeysTable).deleteMany({ user_id: query._id }); } catch (err) { console.log('ERROR:', err); diff --git a/test/api_keys.test.js b/test/api_keys.test.js new file mode 100644 index 0000000..661fe9c --- /dev/null +++ b/test/api_keys.test.js @@ -0,0 +1,311 @@ +'use strict'; + +const Lab = require('@hapi/lab'); +const { expect } = require('@hapi/code'); +const { beforeEach, afterEach, describe, it } = exports.lab = Lab.script(); +const { init } = require('../lib/server'); +const { randomAsciiString, hashedApiKey } = require('../lib/utils'); + +const { apikeysTable, usersTable } = require('../config/db_constants') + +const Jwt = require('jsonwebtoken'); +const { ObjectId } = require('mongodb'); + +const SECRET = require('../config/secret'); + +describe('Users API', () => { + let server; + let db; + let users; + + const adminLoginToken = randomAsciiString(20) + const normalLoginToken = randomAsciiString(20) + const apiKey = randomAsciiString(20) + const apiKeyHash = hashedApiKey(apiKey) + + const adminUser = { + _id: ObjectId('000000000000000000000001'), + username: 'admin', + fullname: 'test_admin', + email: 'admin@example.com', + password: 'hashed', + last_login: new Date(), + roles: ['admin'], + system_user: false, + disabled: false, + loginToken: adminLoginToken + }; + + const normalUser = { + _id: ObjectId('000000000000000000000002'), + username: 'bob', + fullname: 'test_bob', + email: 'bob@example.com', + password: 'hashedbob', + roles: ['cruise_manager'], + system_user: false, + disabled: false, + loginToken: normalLoginToken + }; + + const apiKeyRecord = [ + { + _id: ObjectId('000000000000000000000003'), + user_id: ObjectId('000000000000000000000002'), // Reference to users collection + key_hash: apiKeyHash, // We store a hash, never raw key + label: 'Default Key', + scope: ['read_cruises'], // Optional: can match user scopes or add more granular scopes + created: new Date(), + last_used: null, + disabled: false, + expires: null + } + ]; + + const adminJwt = Jwt.sign( + { id: adminUser._id, roles: adminUser.roles, scope: ['admin'] }, + SECRET + ); + + const normalJwt = Jwt.sign( + { id: normalUser._id, roles: normalUser.roles, scope: ['read_users'] }, + SECRET + ); + + beforeEach(async () => { + server = await init(); + db = server.mongo.db; + + users = db.collection(usersTable); + apikeys = db.collection(apikeysTable); + + await users.deleteMany({}); + await users.insertMany([adminUser, normalUser]); + await apikeys.deleteMany({}); + await apikeys.insertMany([apiKeyRecord]); + }); + + afterEach(async () => { + await server.stop(); + }); + + // ─────────────────────────────────────────────── + // GET /users + // ─────────────────────────────────────────────── + describe('GET /api_keys', () => { + it('returns all api_keys for admin', async () => { + const res = await server.inject({ + method: 'GET', + url: '/sealog-server/api/v1/api_keys', + headers: { Authorization: 'Bearer ' + adminJwt } + }); + + expect(res.statusCode).to.equal(200); + expect(res.result.length).to.equal(1); + expect(res.result[0]).to.not.include('key_hash'); + }); + + // it('returns only non-system api_keys for non-admin', async () => { + // // mark admin as system user + // await api_keys.updateOne( + // { _id: adminUser._id }, + // { $set: { system_user: true } } + // ); + + // const res = await server.inject({ + // method: 'GET', + // url: '/sealog-server/api/v1/api_keys', + // headers: { Authorization: 'Bearer ' + normalJwt } + // }); + + // expect(res.statusCode).to.equal(200); + // expect(res.result.length).to.equal(1); + // expect(res.result[0].username).to.equal('bob'); + // }); + }); + + // ─────────────────────────────────────────────── + // GET /api_keys/{id} + // ─────────────────────────────────────────────── +// describe('GET /api_keys/{id}', () => { +// it('returns the user for admin', async () => { +// const res = await server.inject({ +// method: 'GET', +// url: `/sealog-server/api/v1/api_keys/${normalUser._id}`, +// headers: { Authorization: 'Bearer ' + adminJwt } +// }); + +// expect(res.statusCode).to.equal(200); +// expect(res.result.username).to.equal('bob'); +// }); + +// it('blocks non-admin from accessing system_user', async () => { +// await api_keys.updateOne( +// { _id: ObjectId(adminUser._id) }, +// { $set: { system_user: true } } +// ); + +// const res = await server.inject({ +// method: 'GET', +// url: `/sealog-server/api/v1/api_keys/${adminUser._id}`, +// headers: { Authorization: 'Bearer ' + normalJwt } +// }); + +// expect(res.statusCode).to.equal(400); +// }); +// }); + +// // ─────────────────────────────────────────────── +// // POST /api_keys +// // ─────────────────────────────────────────────── +// describe('POST /api_keys', () => { +// it('creates a new user', async () => { +// const payload = { +// username: 'charlie', +// password: 'pass123', +// fullname: 'test_charlie', +// email: 'charlie@example.com', +// resetURL: 'https://example/reset/', +// roles: ['event_logger'] +// }; + +// const res = await server.inject({ +// method: 'POST', +// url: '/sealog-server/api/v1/api_keys', +// headers: { Authorization: 'Bearer ' + adminJwt }, +// payload +// }); + +// expect(res.statusCode).to.equal(201); + +// const newUser = await api_keys.findOne({ username: 'charlie' }); +// expect(newUser).to.exist(); +// expect(newUser.password).to.exist(); +// }); + +// it('rejects duplicate username', async () => { +// const payload = { +// username: 'bob', +// password: 'pass123', +// fullname: 'test_bob', +// email: 'x@example.com', +// resetURL: 'https://example/reset/', +// roles: ['event_logger'] +// }; + +// const res = await server.inject({ +// method: 'POST', +// url: '/sealog-server/api/v1/api_keys', +// headers: { Authorization: 'Bearer ' + adminJwt }, +// payload +// }); + +// expect(res.statusCode).to.equal(409); +// }); +// }); + +// // ─────────────────────────────────────────────── +// // PATCH /api_keys/{id} +// // ─────────────────────────────────────────────── +// describe('PATCH /api_keys/{id}', () => { +// it('updates a user when authorized', async () => { +// const res = await server.inject({ +// method: 'PATCH', +// url: `/sealog-server/api/v1/api_keys/${normalUser._id}`, +// headers: { Authorization: 'Bearer ' + normalJwt }, +// payload: { fullname: 'new_bob' } +// }); + +// expect(res.statusCode).to.equal(204); + +// const updated = await api_keys.findOne({ _id: normalUser._id }); +// expect(updated.fullname).to.equal('new_bob'); +// }); + +// it('disallows non-admin modifying system_user account', async () => { +// await api_keys.updateOne( +// { _id: adminUser._id }, +// { $set: { system_user: true } } +// ); + +// const res = await server.inject({ +// method: 'PATCH', +// url: `/sealog-server/api/v1/api_keys/${adminUser._id}`, +// headers: { Authorization: 'Bearer ' + normalJwt }, +// payload: { fullname: 'new_admin' } +// }); + +// expect(res.statusCode).to.equal(400); +// }); +// }); + +// // ─────────────────────────────────────────────── +// // DELETE /api_keys/{id} +// // ─────────────────────────────────────────────── +// describe('DELETE /api_keys/{id}', () => { +// it('deletes a user as admin', async () => { +// const res = await server.inject({ +// method: 'DELETE', +// url: `/sealog-server/api/v1/api_keys/${normalUser._id}`, +// headers: { Authorization: 'Bearer ' + adminJwt } +// }); + +// expect(res.statusCode).to.equal(204); + +// const gone = await api_keys.findOne({ _id: normalUser._id }); +// expect(gone).to.not.exist(); +// }); + +// it('prevents user from deleting self', async () => { +// const res = await server.inject({ +// method: 'DELETE', +// url: `/sealog-server/api/v1/api_keys/${adminUser._id}`, +// headers: { Authorization: 'Bearer ' + adminJwt } +// }); + +// expect(res.statusCode).to.equal(400); +// }); +// }); + +// // ─────────────────────────────────────────────── +// // GET /api_keys/{id}/token +// // ─────────────────────────────────────────────── +// describe('GET /api_keys/{id}/token', () => { +// it('returns JWT for owner', async () => { +// const res = await server.inject({ +// method: 'GET', +// url: `/sealog-server/api/v1/api_keys/${normalUser._id}/token`, +// headers: { Authorization: 'Bearer ' + normalJwt } +// }); + +// expect(res.statusCode).to.equal(200); +// expect(res.result.token).to.exist(); +// }); + +// it('blocks others unless admin', async () => { +// const res = await server.inject({ +// method: 'GET', +// url: `/sealog-server/api/v1/api_keys/${normalUser._id}/token`, +// headers: { Authorization: 'Bearer ' + adminJwt } +// }); + +// expect(res.statusCode).to.equal(200); +// }); +// }); + +// // ─────────────────────────────────────────────── +// // GET /api_keys/{id}/loginToken +// // ─────────────────────────────────────────────── +// describe('GET /api_keys/{id}/loginToken', () => { +// it('returns loginToken for user', async () => { +// const res = await server.inject({ +// method: 'GET', +// url: `/sealog-server/api/v1/api_keys/${normalUser._id}/loginToken`, +// headers: { Authorization: 'Bearer ' + normalJwt } +// }); + +// expect(res.statusCode).to.equal(200); +// expect(res.result.loginToken).to.equal(normalLoginToken); +// }); +// }); +});