Aux Data Cleaner logging bugfix

demo/FKt230303_S0492_eventTemplates.json

@@ -385,7 +385,15 @@
           "Deployed"
         ],
         "event_option_required": true,
-        "event_option_allow_freeform": false
+        "event_option_allow_freeform": false,
+        "event_option_visibility": {
+          "show_hide": "show if",
+          "event_option_name": "Type",
+          "event_option_values": [
+            "Push Core",
+            "Water Sample"
+          ]
+        }
       },
       {
         "event_option_name": "Sample Description",

lib/utils.js

@@ -376,6 +376,12 @@ const hashedPassword = (password) => {
 
 };
 
+const hashedApiKey = (apiKey) => {
+
+  return hashSync( apiKey, saltRounds );
+
+};
+
 const mvFilesToDir = (sourcePath, destPath, createIfMissing = false) => {
 
   try {
@@ -484,6 +490,7 @@ module.exports = {
   filePreProcessor,
   flattenEventObjs,
   hashedPassword,
+  hashedApiKey,
   mvFilesToDir,
   randomAsciiString,
   randomString,

lib/validations.js

@@ -1,6 +1,7 @@
 const Joi = require('joi');
 
 const {
+  roles,
   useAccessControl
 } = require('../config/server_settings');
 
@@ -360,6 +361,12 @@ const eventTemplateQuery = Joi.object({
   sort: Joi.string().valid('event_name').optional()
 }).optional().label('eventTemplateQuery');
 
+const eventTemplateVisibility = Joi.object({
+  show_hide: Joi.string().required(),
+  event_option_name: Joi.string().required(),
+  event_option_values: Joi.array().items(Joi.string()).required()
+});
+
 const eventTemplateResponse = Joi.object({
   id: Joi.object(),
   event_name: Joi.string(),
@@ -376,7 +383,8 @@ const eventTemplateResponse = Joi.object({
     event_option_default_value: Joi.string().allow(''),
     event_option_values: Joi.array().items(Joi.string()),
     event_option_allow_freeform: Joi.boolean(),
-    event_option_required: Joi.boolean()
+    event_option_required: Joi.boolean(),
+    event_option_visibility: eventTemplateVisibility.optional()
   }))
 }).label('eventTemplateResponse');
 
@@ -396,7 +404,8 @@ const eventTemplateCreatePayload = Joi.object({
     event_option_default_value: Joi.string().allow('').optional(),
     event_option_values: Joi.array().items(Joi.string()).required(),
     event_option_allow_freeform: Joi.boolean().required(),
-    event_option_required: Joi.boolean().required()
+    event_option_required: Joi.boolean().required(),
+    event_option_visibility: eventTemplateVisibility.optional()
   })).optional()
 }).label('eventTemplateCreatePayload');
 
@@ -415,7 +424,8 @@ const eventTemplateUpdatePayload = Joi.object({
     event_option_default_value: Joi.string().allow('').optional(),
     event_option_values: Joi.array().items(Joi.string()).required(),
     event_option_allow_freeform: Joi.boolean().required(),
-    event_option_required: Joi.boolean().required()
+    event_option_required: Joi.boolean().required(),
+    event_option_visibility: eventTemplateVisibility.optional()
   })).optional()
 }).required().min(1).label('eventTemplateUpdatePayload');
 
@@ -626,7 +636,7 @@ const userCreatePayload = Joi.object({
   fullname: Joi.string().min(1).max(100).required(),
   email: Joi.string().email().required(),
   password: Joi.string().allow('').max(50).required(),
-  roles: Joi.array().items(Joi.string()).min(1).required(),
+  roles: Joi.array().items(Joi.string().valid(...roles)).min(1).required(),
   system_user: Joi.boolean().optional(),
   disabled: Joi.boolean().optional(),
   resetURL: Joi.string().uri().required()
@@ -637,7 +647,7 @@ const userUpdatePayload = Joi.object({
   fullname: Joi.string().min(1).max(100).optional(),
   // email: Joi.string().email().optional(),
   password: Joi.string().allow('').max(50).optional(),
-  roles: Joi.array().items(Joi.string()).min(1).optional(),
+  roles: Joi.array().items(Joi.string().valid(...roles)).min(1).optional(),
   system_user: Joi.boolean().optional(),
   disabled: Joi.boolean().optional()
 }).required().min(1).label('userUpdatePayload');
@@ -649,7 +659,7 @@ const userResponse = Joi.object({
   last_login: Joi.date(),
   username: Joi.string(),
   fullname: Joi.string(),
-  roles: Joi.array().items(Joi.string()),
+  roles: Joi.array().items(Joi.string().valid(...roles)),
   disabled: Joi.boolean()
 }).label('userResponse');
 
@@ -658,6 +668,46 @@ const userSuccessResponse = Joi.alternatives().try(
   Joi.array().items(userResponse)
 ).label('userSuccessResponse');
 
+
+// api keys
+// ----------------------------------------------------------------------------
+const apiKeyParam = Joi.object({
+  id: Joi.string().length(24).required()
+}).label('apiKeyParam');
+
+const apiKeyQuery = Joi.object({
+  id: Joi.string().length(24).optional(),
+  label: Joi.string().max(100).optional()
+}).label('apiKeyQuery');
+
+const apiKeyCreatePayload = Joi.object({
+  label: Joi.string().max(100).required(),
+  scope: Joi.array().items(Joi.string()).optional(),
+  expires: Joi.date().optional()
+}).label('apiKeyCreatePayload');
+
+const apiKeyResponse = Joi.object({
+  id: Joi.object(),
+  user_id: Joi.object(),
+  label: Joi.string(),
+  scope: Joi.array().items(Joi.string()),
+  created: Joi.date(),
+  last_used: Joi.date().allow(null),
+  disabled: Joi.boolean(),
+  expires: Joi.date().allow(null)
+}).label('apiKeyResponse');
+
+const apiKeySuccessResponse = Joi.alternatives().try(
+  apiKeyResponse,
+  Joi.array().items(apiKeyResponse)
+).label('apiKeySuccessResponse');
+
+const apiKeyUpdatePayload = Joi.object({
+  label: Joi.string().max(100).optional(),
+  scope: Joi.array().items(Joi.string()).optional(),
+  expires: Joi.date().optional()
+}).label('apiKeyUpdatePayload');
+
 module.exports = {
   authorizationHeader,
   autoLoginPayload,
@@ -733,5 +783,10 @@ module.exports = {
   userQuery,
   userSuccessResponse,
   userToken,
-  userUpdatePayload
+  userUpdatePayload,
+  apiKeyCreatePayload,
+  apiKeyParam,
+  apiKeyQuery,
+  apiKeySuccessResponse,
+  apiKeyUpdatePayload
 };

misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+'''
+FILE:           base_aux_data_file_cleaner.py
+
+DESCRIPTION:    Base class for cleaning up any changes made by aux_data_inserters outside
+                of the aux data mongo collection.
+
+BUGS:
+NOTES:
+AUTHOR:     Lindsey Jones
+COMPANY:    OET
+VERSION:    1.0
+CREATED:    2026-02-20
+REVISION:
+
+LICENSE INFO:   This code is licensed under MIT license (see LICENSE.txt for details)
+                Copyright (C) OceanDataTools.org 2025
+'''
+import logging
+
+from abc import ABC, abstractmethod
+
+
+class AuxDataFileCleaner(ABC):
+    '''
+    Abstract base class for building sealog aux_data records from various data sources.
+    Handles common functionality for querying external data sources and building
+    aux_data records with transformations.
+    '''
+
+    def __init__(self, aux_data_config):
+        '''
+        Initialize the base builder with configuration.
+        '''
+        self._data_source = aux_data_config['data_source']
+        self.logger = logging.getLogger(__name__)
+
+    def _get_aux_data_for_source(self, event):
+        '''
+        Get the aux data record for the given event based on the data source.
+
+        Args:
+            event (dict): Event dictionary containing 'id' and 'ts' keys
+
+        Returns:
+            str or None: ID of cleaned aux data record, if there is one
+        '''
+        all_aux_data = event.get('aux_data', {})
+        if not all_aux_data:
+            self.logger.debug("No aux data found for event %s", event['id'])
+            return None
+
+        # Find the first aux_data record that matches the data source
+        # There should only ever be one record per data source
+        aux_data_for_source = next(
+            (aux_data for aux_data in all_aux_data
+             if aux_data["data_source"] == self._data_source),
+            None  # default if not found
+        )
+
+        if not aux_data_for_source:
+            self.logger.debug("No %s aux data found for event %s", self._data_source, event['id'])
+
+        return aux_data_for_source
+
+    @abstractmethod
+    def open_connections(self):
+        '''
+        Open any necessary connections to external data sources.
+        Must be implemented by subclasses.
+        '''
+
+    @abstractmethod
+    def close_connections(self):
+        '''
+        Close any open connections to external data sources.
+        Must be implemented by subclasses.
+        '''
+
+    @abstractmethod
+    def clean_aux_data_record(self, event, dry_run):
+        '''
+        Do any clean up required for the given event
+        Must be implemented by subclasses.
+
+        Args:
+            event (dict): Event dictionary containing 'id' and 'ts' keys
+            dry_run (bool): If True, do not actually delete any aux data records
+
+        Returns:
+            dict or None: Aux data record or None if no data available
+        '''
+
+    @property
+    def data_source(self):
+        '''
+        Getter method for the data_source property
+        '''
+        return self._data_source

misc/aux_data_file_cleaners/delete_files_aux_data_file_cleaner.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+'''
+FILE:           delete_files_aux_data_file_cleaner.py
+
+DESCRIPTION:    Cleans up the files created by the aux data creation process.
+
+BUGS:
+NOTES:
+AUTHOR:     Lindsey Jones
+COMPANY:    OET
+VERSION:    1.0
+CREATED:    2026-02-20
+REVISION:
+
+LICENSE INFO:   This code is licensed under MIT license (see LICENSE.txt for details)
+                Copyright (C) OceanDataTools.org 2025
+'''
+import os
+import sys
+
+from os.path import dirname, realpath
+sys.path.append(dirname(dirname(dirname(realpath(__file__)))))
+
+from misc.aux_data_file_cleaners.base_aux_data_file_cleaner import AuxDataFileCleaner
+
+
+class DeleteFilesAuxDataFileCleaner(AuxDataFileCleaner):
+    '''
+    Cleans up the files created by the stillcap_ffmpeg aux data creation process
+    '''
+
+    def open_connections(self):
+        '''
+        No connections to open.
+        '''
+
+    def close_connections(self):
+        '''
+        No connections to close.
+        '''
+
+    def clean_aux_data_record(self, event, dry_run):
+        '''
+        Do any clean up required for the given event
+
+        Args:
+            event (dict): Event dictionary containing 'id' and 'ts' keys
+            dry_run (bool): If True, do not actually delete any aux data records
+
+        Returns:
+            str or None: ID of cleaned aux data record, if there is one
+        '''
+        aux_data = self._get_aux_data_for_source(event)
+
+        if not aux_data:
+            return None
+
+        file_paths = [
+            data["data_value"]
+            for data in aux_data["data_array"]
+            if data["data_name"] == "filename"
+        ]
+        for file_path in file_paths:
+            try:
+                self.logger.debug("Deleting file %s for event %s", file_path, event['id'])
+                if not dry_run:
+                    os.remove(file_path)
+            except Exception as e:  # pylint: disable=W0718
+                self.logger.error(
+                    "Error deleting file %s for event %s: %s",
+                    file_path,
+                    event['id'],
+                    e
+                )
+
+        return aux_data["_id"]