From 4dece1960094d5365d9b97aca0fe692a2e2cdbe6 Mon Sep 17 00:00:00 2001 From: Webb Pinner Date: Wed, 13 Aug 2025 10:45:56 -0400 Subject: [PATCH 01/17] update cruise_create script to take advantage of 2.11 --- misc/sealog_create_cruise_from_openvdm.py.dist | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misc/sealog_create_cruise_from_openvdm.py.dist b/misc/sealog_create_cruise_from_openvdm.py.dist index 345d3c7..19ffe05 100644 --- a/misc/sealog_create_cruise_from_openvdm.py.dist +++ b/misc/sealog_create_cruise_from_openvdm.py.dist @@ -92,9 +92,9 @@ def main(force=False): # pylint:disable=R0915 if req.status_code == 200: cruise_config = json.loads(req.text) cruise['cruise_id'] = cruise_config['cruiseID'] - cruise['cruise_additional_meta']['cruise_name'] = cruise_config['cruiseName'] - cruise['cruise_location'] = cruise_config['cruiseLocation'] - cruise['cruise_additional_meta']['cruise_pi'] = cruise_config['cruisePI'] + cruise['cruise_additional_meta']['cruise_name'] = cruise_config.get('cruiseName', 'FIX ME') + cruise['cruise_location'] = cruise_config.get('cruiseLocation', 'FIX ME') + cruise['cruise_additional_meta']['cruise_pi'] = cruise_config.get('cruisePI', 'FIX ME') cruise['start_ts'] = datetime.strptime( cruise_config['cruiseStartDate'], From 751dcae368f7e6ce7b5e226847934ba53a5ccb9a Mon Sep 17 00:00:00 2001 From: Webb Pinner Date: Tue, 24 Feb 2026 09:16:34 -0400 Subject: [PATCH 02/17] Adds event option visibility control Adds the ability to control the visibility of event options based on the values of other event options. This allows administrators to create more dynamic and context-aware event templates, improving the user experience and reducing the likelihood of errors. --- demo/FKt230303_S0492_eventTemplates.json | 10 +++++++++- lib/validations.js | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/demo/FKt230303_S0492_eventTemplates.json b/demo/FKt230303_S0492_eventTemplates.json index ea2537d..f5e798e 100644 --- a/demo/FKt230303_S0492_eventTemplates.json +++ b/demo/FKt230303_S0492_eventTemplates.json @@ -385,7 +385,15 @@ "Deployed" ], "event_option_required": true, - "event_option_allow_freeform": false + "event_option_allow_freeform": false, + "event_option_visibility": { + "show_hide": "show if", + "event_option_name": "Type", + "event_option_values": [ + "Push Core", + "Water Sample" + ] + } }, { "event_option_name": "Sample Description", diff --git a/lib/validations.js b/lib/validations.js index 285cbd0..7442fa6 100644 --- a/lib/validations.js +++ b/lib/validations.js @@ -360,6 +360,12 @@ const eventTemplateQuery = Joi.object({ sort: Joi.string().valid('event_name').optional() }).optional().label('eventTemplateQuery'); +const eventTemplateVisibility = Joi.object({ + show_hide: Joi.string().required(), + event_option_name: Joi.string().required(), + event_option_values: Joi.array().items(Joi.string()).required() +}); + const eventTemplateResponse = Joi.object({ id: Joi.object(), event_name: Joi.string(), @@ -376,7 +382,8 @@ const eventTemplateResponse = Joi.object({ event_option_default_value: Joi.string().allow(''), event_option_values: Joi.array().items(Joi.string()), event_option_allow_freeform: Joi.boolean(), - event_option_required: Joi.boolean() + event_option_required: Joi.boolean(), + event_option_visibility: eventTemplateVisibility.optional() })) }).label('eventTemplateResponse'); @@ -396,7 +403,8 @@ const eventTemplateCreatePayload = Joi.object({ event_option_default_value: Joi.string().allow('').optional(), event_option_values: Joi.array().items(Joi.string()).required(), event_option_allow_freeform: Joi.boolean().required(), - event_option_required: Joi.boolean().required() + event_option_required: Joi.boolean().required(), + event_option_visibility: eventTemplateVisibility.optional() })).optional() }).label('eventTemplateCreatePayload'); @@ -415,7 +423,8 @@ const eventTemplateUpdatePayload = Joi.object({ event_option_default_value: Joi.string().allow('').optional(), event_option_values: Joi.array().items(Joi.string()).required(), event_option_allow_freeform: Joi.boolean().required(), - event_option_required: Joi.boolean().required() + event_option_required: Joi.boolean().required(), + event_option_visibility: eventTemplateVisibility.optional() })).optional() }).required().min(1).label('eventTemplateUpdatePayload'); From d9e5581bc1a2cc8917856a2fcd8540dcb140cfff Mon Sep 17 00:00:00 2001 From: Webb Pinner Date: Sat, 8 Nov 2025 09:05:03 -0500 Subject: [PATCH 03/17] implementing changes for issue #64 --- routes/api/v1/cruises.js | 65 +++++----- routes/api/v1/custom_vars.js | 15 ++- routes/api/v1/event_aux_data.js | 200 +++++++++++++++-------------- routes/api/v1/event_exports.js | 209 ++++++++++++++++--------------- routes/api/v1/event_templates.js | 24 ++-- routes/api/v1/events.js | 181 +++++++++++++------------- routes/api/v1/lowerings.js | 98 ++++++++------- 7 files changed, 403 insertions(+), 389 deletions(-) diff --git a/routes/api/v1/cruises.js b/routes/api/v1/cruises.js index 92c1484..3e2029e 100644 --- a/routes/api/v1/cruises.js +++ b/routes/api/v1/cruises.js @@ -199,35 +199,37 @@ exports.plugin = { const cruises = await db.collection(cruisesTable).find(query).sort( { start_ts: -1 } ).skip(offset).limit(limit).toArray(); // console.log("cruises:", cruises); - if (cruises.length > 0) { + if (cruises.length === 0) { + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } - const mod_cruises = cruises.map((cruise) => { + return h.response([]).code(200); + } - try { - cruise.cruise_additional_meta.cruise_files = Fs.readdirSync(cruisePath + '/' + cruise._id); - } - catch (error) { - cruise.cruise_additional_meta.cruise_files = []; - } + const mod_cruises = cruises.map((cruise) => { - return _renameAndClearFields(cruise); - }); + try { + cruise.cruise_additional_meta.cruise_files = Fs.readdirSync(cruisePath + '/' + cruise._id); + } + catch (error) { + cruise.cruise_additional_meta.cruise_files = []; + } - if (request.query.format && request.query.format === 'csv') { + return _renameAndClearFields(cruise); + }); - const flat_cruises = flattenCruiseObjs(mod_cruises); - const csv_headers = buildCruiseCSVHeaders(flat_cruises); - const parser = new AsyncParser({ fields: csv_headers }, {}, {}); - const csv_results = await parser.parse(flat_cruises).promise(); + if (request.query.format && request.query.format === 'csv') { - return h.response(csv_results).code(200); - } + const flat_cruises = flattenCruiseObjs(mod_cruises); + const csv_headers = buildCruiseCSVHeaders(flat_cruises); + const parser = new AsyncParser({ fields: csv_headers }, {}, {}); + const csv_results = await parser.parse(flat_cruises).promise(); - return h.response(mod_cruises).code(200); + return h.response(csv_results).code(200); } - return Boom.notFound('No records found'); - + return h.response(mod_cruises).code(200); } catch (err) { return Boom.serverUnavailable('database error', err); @@ -329,7 +331,7 @@ exports.plugin = { return h.response(_renameAndClearFields(cruise)).code(200); } - return Boom.notFound('No records found'); + return Boom.notFound('No cruise record found'); } catch (err) { @@ -423,7 +425,7 @@ exports.plugin = { return h.response(_renameAndClearFields(cruise)).code(200); } - return Boom.notFound('No records found'); + return Boom.notFound('No cruise record found'); } catch (err) { @@ -476,11 +478,11 @@ exports.plugin = { try { const result = await db.collection(cruisesTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No cruise record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && result.cruise_hidden && (useAccessControl && typeof result.cruise_access_list !== 'undefined' && !result.cruise_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this cruise'); + return Boom.unauthorized('User not authorized to retrieve this cruise record'); } cruise = result; @@ -554,11 +556,11 @@ exports.plugin = { try { const result = await db.collection(cruisesTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No cruise record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && result.cruise_hidden && (useAccessControl && typeof result.cruise_access_list !== 'undefined' && !result.cruise_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this cruise'); + return Boom.unauthorized('User not authorized to retrieve this cruise record'); } cruise = result; @@ -756,11 +758,11 @@ exports.plugin = { const result = await db.collection(cruisesTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No cruise record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && result.cruise_hidden && ( useAccessControl && typeof result.cruise_access_list !== 'undefined' && !result.cruise_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to edit this cruise'); + return Boom.unauthorized('User not authorized to edit this cruise record'); } // if a start date and/or stop date is provided, ensure the new date works with the existing date @@ -872,9 +874,8 @@ exports.plugin = { const loweringQuery = { start_ts: { '$gte': updatedCruise.start_ts }, stop_ts: { '$lt': updatedCruise.stop_ts } }; try { - console.error('here 2'); const cruiseLowerings = await db.collection(loweringsTable).find(loweringQuery).toArray(); - // console.log(cruiseLowerings); + cruiseLowerings.forEach((lowering) => { lowering.id = lowering._id; @@ -940,7 +941,7 @@ exports.plugin = { cruise = await db.collection(cruisesTable).findOne(query); if (!cruise) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No cruise record found for id: ' + request.params.id); } } @@ -1058,7 +1059,7 @@ exports.plugin = { const result = await db.collection(cruisesTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No cruise record found for id: ' + request.params.id); } } catch (err) { diff --git a/routes/api/v1/custom_vars.js b/routes/api/v1/custom_vars.js index 916a183..dc129da 100644 --- a/routes/api/v1/custom_vars.js +++ b/routes/api/v1/custom_vars.js @@ -49,14 +49,13 @@ exports.plugin = { try { const results = await db.collection(customVarsTable).find(query).toArray(); - if (results.length > 0) { - - results.forEach(_renameAndClearFields); - - return h.response(results).code(200); + if (results.length === 0) { + return h.response([]).code(200); } - return Boom.notFound('No records found'); + results.forEach(_renameAndClearFields); + + return h.response(results).code(200); } catch (err) { @@ -104,7 +103,7 @@ exports.plugin = { try { const result = await db.collection(customVarsTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No custom variable record found for id: ' + request.params.id); } const mod_result = _renameAndClearFields(result); @@ -157,7 +156,7 @@ exports.plugin = { const result = await db.collection(customVarsTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No custom variable record found for id: ' + request.params.id); } custom_var_name = result.custom_var_name; diff --git a/routes/api/v1/event_aux_data.js b/routes/api/v1/event_aux_data.js index 8026afb..705ef03 100644 --- a/routes/api/v1/event_aux_data.js +++ b/routes/api/v1/event_aux_data.js @@ -69,11 +69,11 @@ exports.plugin = { const cruiseResult = await db.collection(cruisesTable).findOne({ _id: cruise_id }); if (!cruiseResult) { - return Boom.badRequest('Cruise not found for id' + request.params.id); + return Boom.badRequest('No cruise record found for id' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && cruiseResult.cruise_hidden && (useAccessControl && typeof cruiseResult.cruise_access_list !== 'undefined' && !cruiseResult.cruise_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this cruise'); + return Boom.unauthorized('User not authorized to retrieve this cruise record'); } cruise = cruiseResult; @@ -89,46 +89,45 @@ exports.plugin = { const results = await db.collection(eventsTable).find(eventQuery, { _id: 1 }).sort( { ts: 1 } ).toArray(); // EventID Filtering - if (results.length > 0) { - const query = {}; + if (results.length === 0) { + return h.response([]).code(200); + } - const eventIDs = results.map((event) => { + const query = {}; - return event._id; - }); - query.event_id = { $in: eventIDs }; + const eventIDs = results.map((event) => { - // Datasource Filtering - if (request.query.datasource) { - if (Array.isArray(request.query.datasource)) { - query.data_source = { $in: request.query.datasource }; - } - else { - query.data_source = request.query.datasource; - } + return event._id; + }); + query.event_id = { $in: eventIDs }; + + // Datasource Filtering + if (request.query.datasource) { + if (Array.isArray(request.query.datasource)) { + query.data_source = { $in: request.query.datasource }; + } + else { + query.data_source = request.query.datasource; } + } - const limit = (request.query.limit) ? request.query.limit : 0; - const offset = (request.query.offset) ? request.query.offset : 0; + const limit = (request.query.limit) ? request.query.limit : 0; + const offset = (request.query.offset) ? request.query.offset : 0; - try { - const eventAuxDataResults = await db.collection(eventAuxDataTable).find(query).skip(offset).limit(limit).toArray(); + try { + const eventAuxDataResults = await db.collection(eventAuxDataTable).find(query).skip(offset).limit(limit).toArray(); - if (eventAuxDataResults.length > 0) { - eventAuxDataResults.forEach(_renameAndClearFields); + if (eventAuxDataResults.length === 0) { + return h.response([]).code(200); + } - return h.response(eventAuxDataResults).code(200); - } + eventAuxDataResults.forEach(_renameAndClearFields); - return Boom.notFound('No records found'); + return h.response(eventAuxDataResults).code(200); - } - catch (err) { - return Boom.serverUnavailable('database error', err); - } } - else { - return Boom.notFound('No records found'); + catch (err) { + return Boom.serverUnavailable('database error', err); } } catch (err) { @@ -171,11 +170,11 @@ exports.plugin = { const loweringResult = await db.collection(loweringsTable).findOne({ _id: ObjectID(request.params.id) }); if (!loweringResult) { - return Boom.notFound('lowering not found for that id'); + return Boom.notFound('No lowering record found for id' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && loweringResult.lowering_hidden && (useAccessControl && typeof loweringResult.lowering_access_list !== 'undefined' && !loweringResult.lowering_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this lowering'); + return Boom.unauthorized('User not authorized to retrieve this lowering record'); } lowering = loweringResult; @@ -190,46 +189,45 @@ exports.plugin = { const results = await db.collection(eventsTable).find(eventQuery, { _id: 1 }).sort( { ts: 1 } ).toArray(); // EventID Filtering - if (results.length > 0) { - const query = {}; - const eventIDs = results.map((event) => { + if (results.length === 0) { + return h.response([]).code(200); + } - return event._id; - }); - query.event_id = { $in: eventIDs }; + const query = {}; + const eventIDs = results.map((event) => { - // Datasource Filtering - if (request.query.datasource) { - if (Array.isArray(request.query.datasource)) { - query.data_source = { $in: request.query.datasource }; - } - else { - query.data_source = request.query.datasource; - } + return event._id; + }); + query.event_id = { $in: eventIDs }; + + // Datasource Filtering + if (request.query.datasource) { + if (Array.isArray(request.query.datasource)) { + query.data_source = { $in: request.query.datasource }; } + else { + query.data_source = request.query.datasource; + } + } - // Limiting & Offset - const limit = (request.query.limit) ? request.query.limit : 0; - const offset = (request.query.offset) ? request.query.offset : 0; + // Limiting & Offset + const limit = (request.query.limit) ? request.query.limit : 0; + const offset = (request.query.offset) ? request.query.offset : 0; - try { - const auxDataResults = await db.collection(eventAuxDataTable).find(query).skip(offset).limit(limit).toArray(); + try { + const auxDataResults = await db.collection(eventAuxDataTable).find(query).skip(offset).limit(limit).toArray(); - if (auxDataResults.length > 0) { - auxDataResults.forEach(_renameAndClearFields); + if (auxDataResults.length === 0) { + return h.response([]).code(200); + } - return h.response(auxDataResults).code(200); - } + auxDataResults.forEach(_renameAndClearFields); - return Boom.notFound('No records found'); + return h.response(auxDataResults).code(200); - } - catch (err) { - return Boom.serverUnavailable('database error', err); - } } - else { - return Boom.notFound('No records found'); + catch (err) { + return Boom.serverUnavailable('database error', err); } } catch (err) { @@ -278,47 +276,45 @@ exports.plugin = { const results = await db.collection(eventsTable).find(eventQuery, { _id: 1 }).sort( { ts: 1 } ).toArray(); // EventID Filtering - if (results.length > 0) { - const query = {}; + if (results.length === 0) { + return h.response([]).code(200); + } - const eventIDs = results.map((event) => { + const query = {}; - return new ObjectID(event._id); - }); - query.event_id = { $in: eventIDs }; + const eventIDs = results.map((event) => { - // Datasource Filtering - if (request.query.datasource) { - if (Array.isArray(request.query.datasource)) { - query.data_source = { $in: request.query.datasource }; - } - else { - query.data_source = request.query.datasource; - } - } + return new ObjectID(event._id); + }); + query.event_id = { $in: eventIDs }; - const limit = (request.query.limit) ? request.query.limit : 0; - const offset = (request.query.offset) ? request.query.offset : 0; + // Datasource Filtering + if (request.query.datasource) { + if (Array.isArray(request.query.datasource)) { + query.data_source = { $in: request.query.datasource }; + } + else { + query.data_source = request.query.datasource; + } + } - try { - const auxDataResults = await db.collection(eventAuxDataTable).find(query).skip(offset).limit(limit).toArray(); + const limit = (request.query.limit) ? request.query.limit : 0; + const offset = (request.query.offset) ? request.query.offset : 0; - if (auxDataResults.length > 0) { + try { + const auxDataResults = await db.collection(eventAuxDataTable).find(query).skip(offset).limit(limit).toArray(); - auxDataResults.forEach(_renameAndClearFields); + if (auxDataResults.length === 0) { + return h.response([]).code(200); + } - return h.response(auxDataResults).code(200); - } + auxDataResults.forEach(_renameAndClearFields); - return Boom.notFound('No records found'); + return h.response(auxDataResults).code(200); - } - catch (err) { - return Boom.serverUnavailable('database error', err); - } } - else { - return Boom.notFound('No records found'); + catch (err) { + return Boom.serverUnavailable('database error', err); } } catch (err) { @@ -360,13 +356,13 @@ exports.plugin = { try { const results = await db.collection(eventAuxDataTable).find(query).skip(offset).limit(limit).toArray(); - if (results.length > 0) { - results.forEach(_renameAndClearFields); - - return h.response(results).code(200); + if (results.length === 0) { + return h.response([]).code(200); } - return Boom.notFound('No records found'); + results.forEach(_renameAndClearFields); + + return h.response(results).code(200); } catch (err) { @@ -408,7 +404,7 @@ exports.plugin = { try { const result = await db.collection(eventAuxDataTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No aux_data record found for id: ' + request.params.id); } const mod_result = _renameAndClearFields(result); @@ -495,7 +491,7 @@ exports.plugin = { const queryResult = await db.collection(eventsTable).findOne(query); if (!queryResult) { - return Boom.badRequest('event not found'); + return Boom.badRequest('event record not found'); } query = { event_id: event_aux_data.event_id, data_source: event_aux_data.data_source }; @@ -600,7 +596,7 @@ exports.plugin = { result = await db.collection(eventAuxDataTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No aux_data record found for id: ' + request.params.id); } event_aux_data = request.payload; @@ -689,7 +685,7 @@ exports.plugin = { try { const auxData = await db.collection(eventAuxDataTable).findOne(query); if (!auxData) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No aux_data record found for id: ' + request.params.id); } } catch (err) { diff --git a/routes/api/v1/event_exports.js b/routes/api/v1/event_exports.js index 1dc1d9d..cb4d09d 100644 --- a/routes/api/v1/event_exports.js +++ b/routes/api/v1/event_exports.js @@ -59,7 +59,7 @@ exports.plugin = { const cruiseResult = await db.collection(cruisesTable).findOne({ _id: ObjectID(request.params.id) }); if (!cruiseResult) { - return Boom.notFound('cruise not found for that id'); + return Boom.notFound('No cruise record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && cruiseResult.cruise_hidden && (useAccessControl && typeof cruiseResult.cruise_access_list !== 'undefined' && !cruiseResult.cruise_access_list.includes(request.auth.credentials.id))) { @@ -104,62 +104,66 @@ exports.plugin = { return Boom.serverUnavailable('database error'); } - if (results.length > 0) { + if (results.length === 0) { + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } - // datasource filtering - if (request.query.datasource) { + return h.response([]).code(200); + } - const datasource_query = {}; + // datasource filtering + if (request.query.datasource) { - const eventIDs = results.map((event) => event._id); + const datasource_query = {}; - datasource_query.event_id = { $in: eventIDs }; + const eventIDs = results.map((event) => event._id); - if (Array.isArray(request.query.datasource)) { - datasource_query.data_source = { $in: request.query.datasource }; - } - else { - datasource_query.data_source = request.query.datasource; - } + datasource_query.event_id = { $in: eventIDs }; - let aux_data_results = []; - try { - aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } + if (Array.isArray(request.query.datasource)) { + datasource_query.data_source = { $in: request.query.datasource }; + } + else { + datasource_query.data_source = request.query.datasource; + } - const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); + let aux_data_results = []; + try { + aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); + } + catch (err) { + console.log(err); + return Boom.serverUnavailable('database error'); + } - results = results.filter((event) => { + const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); - return (aux_data_eventID_set.has(String(event._id))) ? event : null; - }); + results = results.filter((event) => { - } + return (aux_data_eventID_set.has(String(event._id))) ? event : null; + }); - results.forEach(_renameAndClearFields); + } - if (request.query.add_record_ids) { - results = await addEventRecordIDs(request, results); - } + results.forEach(_renameAndClearFields); - if (request.query.format && request.query.format === 'csv') { + if (request.query.add_record_ids) { + results = await addEventRecordIDs(request, results); + } - const flat_events = flattenEventObjs(results); - const csv_headers = buildEventCSVHeaders(flat_events); - const parser = new AsyncParser({ fields: csv_headers }, {}, {}); - const csv_results = await parser.parse(flat_events).promise(); + if (request.query.format && request.query.format === 'csv') { - return h.response(csv_results).code(200); - } + const flat_events = flattenEventObjs(results); + const csv_headers = buildEventCSVHeaders(flat_events); + const parser = new AsyncParser({ fields: csv_headers }, {}, {}); + const csv_results = await parser.parse(flat_events).promise(); - return h.response(results).code(200); + return h.response(csv_results).code(200); } - return Boom.notFound('No records found'); + return h.response(results).code(200); + }, config: { auth: { @@ -197,11 +201,11 @@ exports.plugin = { const loweringResult = await db.collection(loweringsTable).findOne({ _id: ObjectID(request.params.id) }); if (!loweringResult) { - return Boom.notFound('lowering not found for that id'); + return Boom.notFound('No lowering record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && loweringResult.lowering_hidden && (useAccessControl && typeof loweringResult.lowering_access_list !== 'undefined' && !loweringResult.lowering_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this lowering'); + return Boom.unauthorized('User not authorized to retrieve this lowering record'); } lowering = loweringResult; @@ -213,7 +217,7 @@ exports.plugin = { } if (lowering.lowering_hidden && !request.auth.credentials.scope.includes('admin')) { - return Boom.unauthorized('User not authorized to retrieve hidden lowerings'); + return Boom.unauthorized('User not authorized to retrieve hidden lowering records'); } const query = buildEventsQuery(request, lowering.start_ts, lowering.stop_ts); @@ -246,61 +250,61 @@ exports.plugin = { return Boom.serverUnavailable('database error'); } - if (results.length > 0) { - - // datasource filtering - if (request.query.datasource) { + if (results.length === 0) { + return h.response([]).code(200); + } - const datasource_query = {}; + // datasource filtering + if (request.query.datasource) { - const eventIDs = results.map((event) => event._id); + const datasource_query = {}; - datasource_query.event_id = { $in: eventIDs }; + const eventIDs = results.map((event) => event._id); - if (Array.isArray(request.query.datasource)) { - datasource_query.data_source = { $in: request.query.datasource }; - } - else { - datasource_query.data_source = request.query.datasource; - } + datasource_query.event_id = { $in: eventIDs }; - let aux_data_results = []; - try { - aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } + if (Array.isArray(request.query.datasource)) { + datasource_query.data_source = { $in: request.query.datasource }; + } + else { + datasource_query.data_source = request.query.datasource; + } - const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); + let aux_data_results = []; + try { + aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); + } + catch (err) { + console.log(err); + return Boom.serverUnavailable('database error'); + } - results = results.filter((event) => { + const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); - return (aux_data_eventID_set.has(String(event._id))) ? event : null; - }); - } + results = results.filter((event) => { - results.forEach(_renameAndClearFields); + return (aux_data_eventID_set.has(String(event._id))) ? event : null; + }); + } - if (request.query.add_record_ids) { - results = await addEventRecordIDs(request, results); - } + results.forEach(_renameAndClearFields); - if (request.query.format && request.query.format === 'csv') { + if (request.query.add_record_ids) { + results = await addEventRecordIDs(request, results); + } - const flat_events = flattenEventObjs(results); - const csv_headers = buildEventCSVHeaders(flat_events); - const parser = new AsyncParser({ fields: csv_headers }, {}, {}); - const csv_results = await parser.parse(flat_events).promise(); + if (request.query.format && request.query.format === 'csv') { - return h.response(csv_results).code(200); - } + const flat_events = flattenEventObjs(results); + const csv_headers = buildEventCSVHeaders(flat_events); + const parser = new AsyncParser({ fields: csv_headers }, {}, {}); + const csv_results = await parser.parse(flat_events).promise(); - return h.response(results).code(200); + return h.response(csv_results).code(200); } - return Boom.notFound('No records found'); + return h.response(results).code(200); + }, config: { auth: { @@ -378,12 +382,12 @@ exports.plugin = { try { const results = await db.collection(eventsTable).aggregate(aggregate, { allowDiskUse: true }).skip(offset).toArray(); - if (results.length > 0) { - results.forEach(_renameAndClearFields); - return h.response(results).code(200); + if (results.length === 0) { + return h.response([]).code(200); } - return Boom.notFound('No records found'); + results.forEach(_renameAndClearFields); + return h.response(results).code(200); } catch (err) { @@ -417,28 +421,33 @@ exports.plugin = { try { let results = await db.collection(eventsTable).aggregate(aggregate, { allowDiskUse: true }).skip(offset).toArray(); - if (results.length > 0) { - results.forEach(_renameAndClearFields); - - if (request.query.add_record_ids) { - results = await addEventRecordIDs(request, results); + if (results.length === 0) { + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); } - if (request.query.format && request.query.format === 'csv') { + return h.response([]).code(200); + } - const flat_events = flattenEventObjs(results); - const csv_headers = buildEventCSVHeaders(flat_events); - const parser = new AsyncParser({ fields: csv_headers }, {}, {}); - const csv_results = await parser.parse(flat_events).promise(); + results.forEach(_renameAndClearFields); + if (request.query.add_record_ids) { + results = await addEventRecordIDs(request, results); + } - return h.response(csv_results).code(200); - } + if (request.query.format && request.query.format === 'csv') { + + const flat_events = flattenEventObjs(results); + const csv_headers = buildEventCSVHeaders(flat_events); + const parser = new AsyncParser({ fields: csv_headers }, {}, {}); + const csv_results = await parser.parse(flat_events).promise(); - return h.response(results).code(200); + + return h.response(csv_results).code(200); } - return Boom.notFound('No records found'); + return h.response(results).code(200); + } catch (err) { console.log(err); @@ -494,7 +503,7 @@ exports.plugin = { let results = await db.collection(eventsTable).aggregate(aggregate, { allowDiskUse: true }).toArray(); if (results.length === 0) { - return Boom.notFound('No records found'); + return Boom.notFound('No event records found'); } results.forEach(_renameAndClearFields); diff --git a/routes/api/v1/event_templates.js b/routes/api/v1/event_templates.js index a417ed5..c21ef43 100644 --- a/routes/api/v1/event_templates.js +++ b/routes/api/v1/event_templates.js @@ -59,16 +59,16 @@ exports.plugin = { try { const results = await db.collection(eventTemplatesTable).find(query).sort(sort).skip(offset).limit(limit).toArray(); - if (results.length > 0) { - results.forEach((result) => { + if (results.length === 0) { + return h.response([]).code(200); + } - return _renameAndClearFields(result, request.auth.credentials.scope.some((role) => ['admin', 'write_event_templates'].includes(role))); - }); + results.forEach((result) => { - return h.response(results).code(200); - } + return _renameAndClearFields(result, request.auth.credentials.scope.some((role) => ['admin', 'write_event_templates'].includes(role))); + }); - return Boom.notFound('No records found'); + return h.response(results).code(200); } catch (err) { @@ -118,11 +118,11 @@ exports.plugin = { const result = await db.collection(eventTemplatesTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No event template record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && result.admin_only) { - return Boom.notFound('template only available to admin users'); + return Boom.notFound('template record only available to admin users'); } return h.response(_renameAndClearFields(result, request.auth.credentials.scope.some((role) => ['admin', 'write_event_templates'].includes(role)))).code(200); @@ -257,7 +257,7 @@ exports.plugin = { const result = await db.collection(eventTemplatesTable).findOne(query); if (!result) { - return Boom.badRequest('No record found for id: ' + request.params.id ); + return Boom.badRequest('No event template record found for id: ' + request.params.id ); } event_template = { ...result, ...request.payload }; @@ -326,11 +326,11 @@ exports.plugin = { try { const result = await db.collection(eventTemplatesTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id ); + return Boom.notFound('No event template record found for id: ' + request.params.id ); } if (result.system_template && !request.auth.credentials.scope.includes('admin')) { - return Boom.unauthorized('user does not have permission to delete system templates'); + return Boom.unauthorized('user does not have permission to delete system template records'); } } catch (err) { diff --git a/routes/api/v1/events.js b/routes/api/v1/events.js index 193fe78..ce14245 100644 --- a/routes/api/v1/events.js +++ b/routes/api/v1/events.js @@ -77,7 +77,7 @@ exports.plugin = { } if (!request.auth.credentials.scope.includes('admin') && cruiseResult.cruise_hidden && (useAccessControl && typeof cruiseResult.cruise_access_list !== 'undefined' && !cruiseResult.cruise_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this cruise'); + return Boom.unauthorized('User not authorized to retrieve this cruise record'); } cruise = cruiseResult; @@ -104,7 +104,11 @@ exports.plugin = { } if (results.length === 0) { - return Boom.notFound('No records found' ); + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } + + return h.response([]).code(200); } // --------- Data source filtering @@ -201,11 +205,11 @@ exports.plugin = { const cruiseResult = await db.collection(cruisesTable).findOne({ _id: ObjectID(request.params.id) }); if (!cruiseResult) { - return Boom.badRequest('No record cruise found for id: ' + request.params.id ); + return Boom.badRequest('No cruise record found for id: ' + request.params.id ); } if (!request.auth.credentials.scope.includes('admin') && cruiseResult.cruise_hidden && (useAccessControl && typeof cruiseResult.cruise_access_list !== 'undefined' && !cruiseResult.cruise_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this cruise'); + return Boom.unauthorized('User not authorized to retrieve this cruise record'); } cruise = cruiseResult; @@ -216,7 +220,7 @@ exports.plugin = { } if (cruise.cruise_hidden && !request.auth.credentials.scope.includes('admin')) { - return Boom.unauthorized('User not authorized to retrieve hidden cruises'); + return Boom.unauthorized('User not authorized to retrieve hidden cruise records'); } const query = buildEventsQuery(request, cruise.start_ts, cruise.stop_ts); @@ -232,52 +236,48 @@ exports.plugin = { return Boom.serverUnavailable('database error'); } - if (results.length > 0) { - - // --------- Data source filtering - if (request.query.datasource) { - - const datasource_query = {}; - - const eventIDs = results.map((event) => event._id); + if (typeof request.query.datasource === 'undefined') { + return h.response({ events: results.length }).code(200); + } - datasource_query.event_id = { $in: eventIDs }; + // --------- Data source filtering + const datasource_query = {}; - if (Array.isArray(request.query.datasource)) { - const regex_query = request.query.datasource.map((datasource) => { + const eventIDs = results.map((event) => event._id); - const return_regex = new RegExp(datasource, 'i'); - return return_regex; - }); + datasource_query.event_id = { $in: eventIDs }; - datasource_query.data_source = { $in: regex_query }; - } - else { - datasource_query.data_source = RegExp(request.query.datasource, 'i'); - } + if (Array.isArray(request.query.datasource)) { + const regex_query = request.query.datasource.map((datasource) => { - let aux_data_results = []; - try { - aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } + const return_regex = new RegExp(datasource, 'i'); + return return_regex; + }); - const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); + datasource_query.data_source = { $in: regex_query }; + } + else { + datasource_query.data_source = RegExp(request.query.datasource, 'i'); + } - results = results.filter((event) => { + let aux_data_results = []; + try { + aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); + } + catch (err) { + console.log(err); + return Boom.serverUnavailable('database error'); + } - return (aux_data_eventID_set.has(String(event._id))) ? event : null; - }); + const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); - } + results = results.filter((event) => { - return h.response({ events: results.length }).code(200); - } + return (aux_data_eventID_set.has(String(event._id))) ? event : null; + }); return h.response({ events: results.length }).code(200); + }, config: { auth: { @@ -315,11 +315,11 @@ exports.plugin = { const loweringResult = await db.collection(loweringsTable).findOne({ _id: ObjectID(request.params.id) }); if (!loweringResult) { - return Boom.badRequest('No record lowering found for id: ' + request.params.id ); + return Boom.badRequest('No lowering record found for id: ' + request.params.id ); } if (!request.auth.credentials.scope.includes('admin') && loweringResult.lowering_hidden && (useAccessControl && typeof loweringResult.lowering_access_list !== 'undefined' && !loweringResult.lowering_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this lowering'); + return Boom.unauthorized('User not authorized to retrieve this lowering record'); } lowering = loweringResult; @@ -346,7 +346,11 @@ exports.plugin = { } if (results.length === 0) { - return Boom.notFound('No records found' ); + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } + + return h.response([]).code(200); } // --------- Data source filtering @@ -443,11 +447,11 @@ exports.plugin = { const loweringResult = await db.collection(loweringsTable).findOne({ _id: ObjectID(request.params.id) }); if (!loweringResult) { - return Boom.badRequest('No record lowering found for id: ' + request.params.id ); + return Boom.badRequest('No lowering record found for id: ' + request.params.id ); } if (!request.auth.credentials.scope.includes('admin') && loweringResult.lowering_hidden && (useAccessControl && typeof loweringResult.lowering_access_list !== 'undefined' && !loweringResult.lowering_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this lowering'); + return Boom.unauthorized('User not authorized to retrieve this lowering record'); } lowering = loweringResult; @@ -470,50 +474,45 @@ exports.plugin = { return Boom.serverUnavailable('database error'); } - if (results.length > 0) { - - // --------- Data source filtering - if (request.query.datasource) { - - const datasource_query = {}; - - const eventIDs = results.map((event) => event._id); + // --------- Data source filtering + if (!request.query.datasource) { + return h.response({ events: results.length }).code(200); + } - datasource_query.event_id = { $in: eventIDs }; + const datasource_query = {}; - if (Array.isArray(request.query.datasource)) { - const regex_query = request.query.datasource.map((datasource) => { + const eventIDs = results.map((event) => event._id); - const return_regex = new RegExp(datasource, 'i'); - return return_regex; - }); + datasource_query.event_id = { $in: eventIDs }; - datasource_query.data_source = { $in: regex_query }; - } - else { - datasource_query.data_source = RegExp(request.query.datasource, 'i'); - } + if (Array.isArray(request.query.datasource)) { + const regex_query = request.query.datasource.map((datasource) => { - let aux_data_results = []; - try { - aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } + const return_regex = new RegExp(datasource, 'i'); + return return_regex; + }); - const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); + datasource_query.data_source = { $in: regex_query }; + } + else { + datasource_query.data_source = RegExp(request.query.datasource, 'i'); + } - results = results.filter((event) => { + let aux_data_results = []; + try { + aux_data_results = await db.collection(eventAuxDataTable).find(datasource_query, { _id: 0, event_id: 1 }).toArray(); + } + catch (err) { + console.log(err); + return Boom.serverUnavailable('database error'); + } - return (aux_data_eventID_set.has(String(event._id))) ? event : null; - }); + const aux_data_eventID_set = new Set(aux_data_results.map((aux_data) => String(aux_data.event_id))); - } + results = results.filter((event) => { - return h.response({ events: results.length }).code(200); - } + return (aux_data_eventID_set.has(String(event._id))) ? event : null; + }); return h.response({ events: results.length }).code(200); }, @@ -593,12 +592,16 @@ exports.plugin = { const results = await db.collection(eventsTable).find(query).sort(sort).skip(offset).limit(limit).toArray(); // console.log("results:", results); - if (results.length > 0) { - results.forEach(_renameAndClearFields); - return h.response(results).code(200); + if (results.length === 0) { + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } + + return h.response([]).code(200); } - return Boom.notFound('No records found' ); + results.forEach(_renameAndClearFields); + return h.response(results).code(200); } catch (err) { @@ -619,7 +622,11 @@ exports.plugin = { // console.log("results:", results); if (results.length === 0) { - return Boom.notFound('No records found' ); + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } + + return h.response([]).code(200); } results.forEach(_renameAndClearFields); @@ -761,7 +768,6 @@ exports.plugin = { }); - server.route({ method: 'GET', path: '/events/{id}', @@ -776,7 +782,7 @@ exports.plugin = { let result = await db.collection(eventsTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id ); + return Boom.notFound('No event record found for id: ' + request.params.id ); } result = _renameAndClearFields(result); @@ -1009,11 +1015,8 @@ exports.plugin = { // delete any aux_data const aux_data_query = { event_id: updatedEvent.id }; - // console.log(result.value); - // console.log(aux_data_query); await db.collection(eventAuxDataTable).deleteMany(aux_data_query); - // console.log(del_results); server.publish('/ws/status/newEvents', updatedEvent); @@ -1231,7 +1234,7 @@ exports.plugin = { const result = await db.collection(eventsTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id ); + return Boom.notFound('No event record found for id: ' + request.params.id ); } event = result; diff --git a/routes/api/v1/lowerings.js b/routes/api/v1/lowerings.js index 3ebccd0..296a9eb 100644 --- a/routes/api/v1/lowerings.js +++ b/routes/api/v1/lowerings.js @@ -122,7 +122,7 @@ exports.plugin = { query.lowering_hidden = request.query.hidden; } else if (request.query.hidden) { - return Boom.unauthorized('User not authorized to retrieve hidden lowerings'); + return Boom.unauthorized('User not authorized to retrieve hidden lowering records'); } else { query.lowering_hidden = false; @@ -187,34 +187,37 @@ exports.plugin = { try { const lowerings = await db.collection(loweringsTable).find(query).sort( { start_ts: -1 } ).skip(offset).limit(limit).toArray(); - if (lowerings.length > 0) { + if (lowerings.length === 0) { + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } - const mod_lowerings = lowerings.map((lowering) => { + return h.response([]).code(200); + } - try { - lowering.lowering_additional_meta.lowering_files = Fs.readdirSync(loweringPath + '/' + lowering._id); - } - catch (error) { - lowering.lowering_additional_meta.lowering_files = []; - } + const mod_lowerings = lowerings.map((lowering) => { - return _renameAndClearFields(lowering); - }); + try { + lowering.lowering_additional_meta.lowering_files = Fs.readdirSync(loweringPath + '/' + lowering._id); + } + catch (error) { + lowering.lowering_additional_meta.lowering_files = []; + } - if (request.query.format && request.query.format === 'csv') { + return _renameAndClearFields(lowering); + }); - const flat_lowerings = flattenLoweringObjs(mod_lowerings); - const csv_headers = buildLoweringCSVHeaders(flat_lowerings); - const parser = new AsyncParser({ fields: csv_headers }, {}, {}); - const csv_results = await parser.parse(flat_lowerings).promise(); + if (request.query.format && request.query.format === 'csv') { - return h.response(csv_results).code(200); - } + const flat_lowerings = flattenLoweringObjs(mod_lowerings); + const csv_headers = buildLoweringCSVHeaders(flat_lowerings); + const parser = new AsyncParser({ fields: csv_headers }, {}, {}); + const csv_results = await parser.parse(flat_lowerings).promise(); - return h.response(mod_lowerings).code(200); + return h.response(csv_results).code(200); } - return Boom.notFound('No records found'); + return h.response(mod_lowerings).code(200); } catch (err) { @@ -334,34 +337,37 @@ exports.plugin = { try { const lowerings = await db.collection(loweringsTable).find(query).sort( { start_ts: -1 } ).skip(offset).limit(limit).toArray(); - if (lowerings.length > 0) { + if (lowerings.length === 0) { + if (request.query.format && request.query.format === 'csv') { + return h.response('').code(200); + } - const mod_lowerings = lowerings.map((result) => { + return h.response([]).code(200); + } - try { - result.lowering_additional_meta.lowering_files = Fs.readdirSync(loweringPath + '/' + result._id); - } - catch (error) { - result.lowering_additional_meta.lowering_files = []; - } + const mod_lowerings = lowerings.map((result) => { - return _renameAndClearFields(result); - }); + try { + result.lowering_additional_meta.lowering_files = Fs.readdirSync(loweringPath + '/' + result._id); + } + catch (error) { + result.lowering_additional_meta.lowering_files = []; + } - if (request.query.format && request.query.format === 'csv') { + return _renameAndClearFields(result); + }); - const flat_lowerings = flattenLoweringObjs(mod_lowerings); - const csv_headers = buildLoweringCSVHeaders(flat_lowerings); - const parser = new AsyncParser({ fields: csv_headers }, {}, {}); - const csv_results = await parser.parse(flat_lowerings).promise(); + if (request.query.format && request.query.format === 'csv') { - return h.response(csv_results).code(200); - } + const flat_lowerings = flattenLoweringObjs(mod_lowerings); + const csv_headers = buildLoweringCSVHeaders(flat_lowerings); + const parser = new AsyncParser({ fields: csv_headers }, {}, {}); + const csv_results = await parser.parse(flat_lowerings).promise(); - return h.response(mod_lowerings).code(200); + return h.response(csv_results).code(200); } - return Boom.notFound('No records found'); + return h.response(mod_lowerings).code(200); } catch (err) { @@ -507,7 +513,7 @@ exports.plugin = { try { const result = await db.collection(loweringsTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No lowering record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && result.lowering_hidden && (useAccessControl && typeof result.lowering_access_list !== 'undefined' && !result.lowering_access_list.includes(request.auth.credentials.id))) { @@ -586,11 +592,11 @@ exports.plugin = { try { const result = await db.collection(loweringsTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No lowering record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && result.lowering_hidden && (useAccessControl && typeof result.lowering_access_list !== 'undefined' && !result.lowering_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to retrieve this lowering'); + return Boom.unauthorized('User not authorized to retrieve this lowering record'); } lowering = result; @@ -784,11 +790,11 @@ exports.plugin = { const result = await db.collection(loweringsTable).findOne(query); if (!result) { - return Boom.badRequest('No record found for id: ' + request.params.id); + return Boom.badRequest('No lowering record found for id: ' + request.params.id); } if (!request.auth.credentials.scope.includes('admin') && result.lowering_hidden && ( useAccessControl && typeof result.lowering_access_list !== 'undefined' && !result.lowering_access_list.includes(request.auth.credentials.id))) { - return Boom.unauthorized('User not authorized to edit this lowering'); + return Boom.unauthorized('User not authorized to edit this lowering record'); } // if a start date and/or stop date is provided, ensure the new date works with the existing date @@ -922,7 +928,7 @@ exports.plugin = { lowering = await db.collection(loweringsTable).findOne(query); if (!lowering) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No lowering record found for id: ' + request.params.id); } } @@ -1037,7 +1043,7 @@ exports.plugin = { try { const result = await db.collection(loweringsTable).findOne(query); if (!result) { - return Boom.notFound('No record found for id: ' + request.params.id); + return Boom.notFound('No lowering record found for id: ' + request.params.id); } } catch (err) { From 887a8b10f272c78015e27689f3d19eed4f356ef0 Mon Sep 17 00:00:00 2001 From: Webb Pinner Date: Thu, 27 Nov 2025 10:56:53 -0500 Subject: [PATCH 04/17] initial import to implement issue #66 --- lib/utils.js | 7 + lib/validations.js | 54 +++++- plugins/auth.js | 94 +++++++++- plugins/db_api_keys.js | 69 ++++++++ routes/api/v1/api_keys.js | 289 +++++++++++++++++++++++++++++++ routes/api/v1/auth.js | 3 + routes/api/v1/cruises.js | 30 ++-- routes/api/v1/custom_vars.js | 6 +- routes/api/v1/event_aux_data.js | 14 +- routes/api/v1/event_exports.js | 8 +- routes/api/v1/event_templates.js | 10 +- routes/api/v1/events.js | 24 +-- routes/api/v1/external_calls.js | 4 +- routes/api/v1/lowerings.js | 20 +-- routes/api/v1/users.js | 6 + 15 files changed, 573 insertions(+), 65 deletions(-) create mode 100644 plugins/db_api_keys.js create mode 100644 routes/api/v1/api_keys.js diff --git a/lib/utils.js b/lib/utils.js index 4c2a7af..d812183 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -376,6 +376,12 @@ const hashedPassword = (password) => { }; +const hashedApiKey = (apiKey) => { + + return hashSync( apiKey, saltRounds ); + +}; + const mvFilesToDir = (sourcePath, destPath, createIfMissing = false) => { try { @@ -484,6 +490,7 @@ module.exports = { filePreProcessor, flattenEventObjs, hashedPassword, + hashedApiKey, mvFilesToDir, randomAsciiString, randomString, diff --git a/lib/validations.js b/lib/validations.js index 7442fa6..746c420 100644 --- a/lib/validations.js +++ b/lib/validations.js @@ -1,6 +1,7 @@ const Joi = require('joi'); const { + roles, useAccessControl } = require('../config/server_settings'); @@ -635,7 +636,7 @@ const userCreatePayload = Joi.object({ fullname: Joi.string().min(1).max(100).required(), email: Joi.string().email().required(), password: Joi.string().allow('').max(50).required(), - roles: Joi.array().items(Joi.string()).min(1).required(), + roles: Joi.array().items(Joi.string().valid(...roles)).min(1).required(), system_user: Joi.boolean().optional(), disabled: Joi.boolean().optional(), resetURL: Joi.string().uri().required() @@ -646,7 +647,7 @@ const userUpdatePayload = Joi.object({ fullname: Joi.string().min(1).max(100).optional(), // email: Joi.string().email().optional(), password: Joi.string().allow('').max(50).optional(), - roles: Joi.array().items(Joi.string()).min(1).optional(), + roles: Joi.array().items(Joi.string().valid(...roles)).min(1).optional(), system_user: Joi.boolean().optional(), disabled: Joi.boolean().optional() }).required().min(1).label('userUpdatePayload'); @@ -658,7 +659,7 @@ const userResponse = Joi.object({ last_login: Joi.date(), username: Joi.string(), fullname: Joi.string(), - roles: Joi.array().items(Joi.string()), + roles: Joi.array().items(Joi.string().valid(...roles)), disabled: Joi.boolean() }).label('userResponse'); @@ -667,6 +668,46 @@ const userSuccessResponse = Joi.alternatives().try( Joi.array().items(userResponse) ).label('userSuccessResponse'); + +// api keys +// ---------------------------------------------------------------------------- +const apiKeyParam = Joi.object({ + id: Joi.string().length(24).required() +}).label('apiKeyParam'); + +const apiKeyQuery = Joi.object({ + id: Joi.string().length(24).optional(), + label: Joi.string().max(100).optional() +}).label('apiKeyQuery'); + +const apiKeyCreatePayload = Joi.object({ + label: Joi.string().max(100).required(), + scope: Joi.array().items(Joi.string()).optional(), + expires: Joi.date().optional() +}).label('apiKeyCreatePayload'); + +const apiKeyResponse = Joi.object({ + id: Joi.object(), + user_id: Joi.object(), + label: Joi.string(), + scope: Joi.array().items(Joi.string()), + created: Joi.date(), + last_used: Joi.date().allow(null), + disabled: Joi.boolean(), + expires: Joi.date().allow(null) +}).label('apiKeyResponse'); + +const apiKeySuccessResponse = Joi.alternatives().try( + apiKeyResponse, + Joi.array().items(apiKeyResponse) +).label('apiKeySuccessResponse'); + +const apiKeyUpdatePayload = Joi.object({ + label: Joi.string().max(100).optional(), + scope: Joi.array().items(Joi.string()).optional(), + expires: Joi.date().optional() +}).label('apiKeyUpdatePayload'); + module.exports = { authorizationHeader, autoLoginPayload, @@ -742,5 +783,10 @@ module.exports = { userQuery, userSuccessResponse, userToken, - userUpdatePayload + userUpdatePayload, + apiKeyCreatePayload, + apiKeyParam, + apiKeyQuery, + apiKeySuccessResponse, + apiKeyUpdatePayload }; diff --git a/plugins/auth.js b/plugins/auth.js index 75cf85c..e54e011 100644 --- a/plugins/auth.js +++ b/plugins/auth.js @@ -1,6 +1,9 @@ +const Boom = require('@hapi/boom'); +const Bcrypt = require('bcryptjs'); const SECRET_KEY = require('../config/secret'); const { + apiKeysTable, usersTable } = require('../config/db_constants'); @@ -9,10 +12,10 @@ exports.plugin = { dependencies: ['hapi-mongodb', 'hapi-auth-jwt2'], register: (server, options) => { - const validateFunction = async (decoded, request) => { + const db = server.mongo.db; + const ObjectID = server.mongo.ObjectID; - const db = request.mongo.db; - const ObjectID = request.mongo.ObjectID; + const validateJWT = async (decoded, request) => { try { const result = await db.collection(usersTable).findOne({ _id: new ObjectID(decoded.id) }); @@ -26,7 +29,10 @@ exports.plugin = { return { isValid: false }; } - await db.collection(usersTable).updateOne({ _id: new ObjectID(decoded.id) }, { $set: { last_login: new Date() } }); + // Update last_login no more than once every 10 minutes + if (!result.last_login || Date.now() - result.last_login.getTime() > 600000) { + await db.collection(usersTable).updateOne({ _id: new ObjectID(decoded.id) }, { $set: { last_login: new Date() } }); + } return { isValid: true }; @@ -44,7 +50,85 @@ exports.plugin = { algorithms: ['HS256'] }, // Implement validation function - validate: validateFunction + validate: validateJWT }); + + // ---------------- API KEY VALIDATION ---------------- + + const validateApiKey = async (providedKey) => { + + if (!providedKey) { + return null; + } + + // Fetch only keys that are not disabled or deleted + const keys = await db.collection(apiKeysTable).find({ disabled: { $ne: true } }).toArray(); + console.error('keys:',keys); + + for (const keyRecord of keys) { + // Compare raw key to hashed key (correct order!!) + const isMatch = await Bcrypt.compare(providedKey, keyRecord.key_hash); + + if (!isMatch) { + continue; // try next key + } + + // Check expiration + if (keyRecord.expiresAt && keyRecord.expiresAt < new Date()) { + // Key exists but is expired → treat as invalid + return null; + } + + // Key is valid and not expired + return keyRecord; + } + + // Nothing matched + return null; + }; + + const apiKeyScheme = () => ({ + authenticate: async (request, h) => { + + const apiKey = request.headers['x-api-key']; + + if (!apiKey) { + throw Boom.unauthorized('Missing API Key'); + } + + const keyRecord = await validateApiKey(apiKey); + + if (!keyRecord) { + throw Boom.unauthorized('Invalid API Key'); + } + + const { user_id, scope, expires } = keyRecord; + + // Check expiration + if (expires && new Date() > expires) { + throw Boom.unauthorized('API Key expired'); + } + + // Verify user still exists + const user = await db.collection(usersTable).findOne({ _id: new ObjectID(user_id) }); + if (!user || user.disabled) { + throw Boom.unauthorized('User disabled or missing'); + } + + // Log usage + await db.collection(apiKeysTable).updateOne( + { _id: keyRecord._id }, + { $set: { last_used: new Date() } } + ); + + return h.authenticated({ + credentials: { id: user_id, scope, type: 'api-key' } + }); + } + }); + + + server.auth.scheme('api-key', apiKeyScheme); + server.auth.strategy('api-key', 'api-key'); } }; diff --git a/plugins/db_api_keys.js b/plugins/db_api_keys.js new file mode 100644 index 0000000..de8058b --- /dev/null +++ b/plugins/db_api_keys.js @@ -0,0 +1,69 @@ +const { hashedApiKey } = require('../lib/utils'); + +const { + apiKeysTable +} = require('../config/db_constants'); + +exports.plugin = { + name: 'db_populate_apikeys', + dependencies: ['hapi-mongodb'], + register: async (server, options) => { + + const db = server.mongo.db; + const ObjectID = server.mongo.ObjectID; + + const expiresAt = new Date(); + expiresAt.setDate(expiresAt.getDate() + 30); + + const init_data = [ + { + _id: ObjectID('5981f167212b348ae32fa9f5'), + user_id: ObjectID('5981f167212b348aed7fa9f5'), // Reference to users collection + key_hash: await hashedApiKey('5981f167212b348ae32fa9f5'), // We store a hash, never raw key + label: 'Default Key', + scope: ['read_cruises'], // Optional: can match user scopes or add more granular scopes + created: new Date(), + last_used: null, + disabled: false, + expires: expiresAt + } + ]; + + console.log('Searching for API Keys Collection'); + const result = await db.listCollections({ name: apiKeysTable }).toArray(); + + if (result.length) { + if (process.env.NODE_ENV !== 'development') { + console.log('API Keys Collection already exists... we\'re done here.'); + return; + } + + console.log('API Keys Collection exists... dropping it!'); + try { + await db.dropCollection(apiKeysTable); + } + catch (err) { + console.log('DROP ERROR:', err.code); + throw (err); + } + } + + console.log('Creating API Keys Collection'); + try { + const collection = await db.createCollection(apiKeysTable); + + console.log('Creating API Key indexes'); + await collection.createIndex({ key_hash: 1 }, { unique: true }); + await collection.createIndex({ user_id: 1 }); + await collection.createIndex({ disabled: 1 }); + await collection.createIndex({ expires: 1 }); // for fast expiry checks + + console.log('Populating API Keys Collection'); + await collection.insertMany(init_data); + } + catch (err) { + console.log('CREATE ERROR:', err.code); + throw (err); + } + } +}; diff --git a/routes/api/v1/api_keys.js b/routes/api/v1/api_keys.js new file mode 100644 index 0000000..d67867f --- /dev/null +++ b/routes/api/v1/api_keys.js @@ -0,0 +1,289 @@ +// const Joi = require('joi'); +const Boom = require('@hapi/boom'); +const { hashApiKey, randomAsciiString } = require('../../../lib/utils'); + +const { + apiKeysTable +} = require('../../../config/db_constants'); + +const { + apiKeyCreatePayload, + apiKeyParam, + apiKeyQuery, + apiKeyUpdatePayload, + apiKeySuccessResponse, + authorizationHeader, + databaseInsertResponse +} = require('../../../lib/validations'); + +exports.plugin = { + name: 'apiKeys', + dependencies: ['hapi-auth-jwt2', 'hapi-mongodb'], + register: (server) => { + + const db = server.mongo.db; + const ObjectID = server.mongo.ObjectID; + + const _renameAndClearFields = (doc) => { + + //rename id + doc.id = doc._id; + delete doc._id; + + //remove fields entirely + delete doc.key_hash; + + return doc; + }; + + // ---------------- LIST API KEYS ---------------- + server.route({ + method: 'GET', + path: '/api-keys', + handler: async (request, h) => { + + // if the request includes a user_id that is not the current user's ID and the user id + if (request.params.user_id) { + if (!request.auth.credentials.roles.includes('admin') || request.auth.credentials.id !== request.params.user_id) { + return Boom.badRequest('The requesting user is unauthorized to make that request'); + } + } + + const userId = request.params.user_id || request.auth.credentials.id; + const result = await db.collection(apiKeysTable) + .find({ user_id: ObjectID(userId) }) + .toArray(); + + console.error(result); + + result.forEach(_renameAndClearFields); + + return h.response(result).code(200); + }, + config: { + auth: { + strategy: 'jwt', + scope: ['admin', 'read_apikeys'] + }, + validate: { + headers: authorizationHeader, + query: apiKeyQuery + }, + response: { + status: { + 200: apiKeySuccessResponse + } + }, + description: 'Return the API Keys based on query parameters', + notes: '

Requires authorization via: JWT token

\ +

Available to: admin

', + tags: ['api_keys','api'] + } + }); + + server.route({ + method: 'GET', + path: '/api-keys/{id}', + handler: async (request, h) => { + + const { id } = request.params; + + const result = await db.collection(apiKeysTable).findOne({ _id: new ObjectID(id) }); + + if (!result) { + throw Boom.notFound('API Key not found'); + } + + if (!request.auth.credentials.roles.includes('admin') && request.auth.credentials.id !== result.user_id) { + return Boom.badRequest('The requesting user is unauthorized to make that request'); + } + + const cleanedResult = _renameAndClearFields(result); + + return h.response(cleanedResult).code(200); + }, + config: { + auth: { + strategy: 'jwt', + scope: ['admin', 'read_apikeys'] + }, + validate: { + headers: authorizationHeader, + params: apiKeyQuery + }, + response: { + status: { + 200: apiKeySuccessResponse + } + }, + description: 'Return the API Keys based on query parameters', + notes: '

Requires authorization via: JWT token

\ +

Available to: admin

', + tags: ['api_keys','api'] + } + }); + + // ---------------- CREATE API KEY ---------------- + server.route({ + method: 'POST', + path: '/api-keys', + handler: async (request, h) => { + + const { label, roles = [], expires } = request.payload; + const apiKeyPlain = randomAsciiString(20); + const keyHash = await hashApiKey(apiKeyPlain); + + const result = await db.collection(apiKeysTable).insertOne({ + user_id: ObjectID(request.auth.credentials.id), + key_hash: keyHash, + label, + roles, + created: new Date(), + last_used: null, + disabled: false, + expires: expires ? new Date(expires) : null + }); + + return h.response({ + id: result.insertedId, + key: apiKeyPlain // show plain key ONCE + }).code(201); + }, + config: { + auth: { + strategy: 'jwt', + scope: ['admin', 'create_api_keys'] + }, + validate: { + headers: authorizationHeader, + payload: apiKeyCreatePayload, + failAction: (request, h, err) => { + + throw Boom.badRequest(err.message); + } + }, + response: { + status: { + 201: databaseInsertResponse + } + }, + + description: 'Create a new API key', + notes: '

Requires authorization via: JWT token

\ +

Available to: admin

', + tags: ['api_keys','api'] + } + }); + + + // ---------------- PATCH / UPDATE LABEL OR EXPIRATION ---------------- + server.route({ + method: 'PATCH', + path: '/api-keys/{id}', + handler: async (request, h) => { + + const { id } = request.params; + + const result = await db.collection(apiKeysTable).findOne({ _id: new ObjectID(id) }); + + if (!result) { + throw Boom.notFound('API Key not found'); + } + + if (!request.auth.credentials.roles.includes('admin') && request.auth.credentials.id !== result.user_id) { + return Boom.badRequest('The requesting user is unauthorized to make that request'); + } + + const updates = request.payload; + + const updateDoc = {}; + if (updates.label !== undefined) { + updateDoc.label = updates.label; + } + + if (updates.expires !== undefined) { + updateDoc.expires = updates.expires ? new Date(updates.expires) : null; + } + + if (updates.disabled !== undefined) { + updateDoc.disabled = updates.disabled; + } + + await db.collection(apiKeysTable).updateOne( + { _id: new ObjectID(id) }, + { $set: updateDoc } + ); + + return h.response({ message: 'API Key updated' }).code(200); + }, + config: { + auth: { + strategy: 'jwt', + scope: ['admin', 'write_api_keys'] + }, + validate: { + headers: authorizationHeader, + params: apiKeyParam, + payload: apiKeyUpdatePayload, + failAction: (request, h, err) => { + + throw Boom.badRequest(err.message); + } + }, + response: { + status: { } + }, + description: 'Update an API key', + notes: '

Requires authorization via: JWT token

\ +

Available to: admin

', + tags: ['api_keys','api'] + } + }); + + + // ---------------- DELETE / REVOKE API KEY ---------------- + server.route({ + method: 'DELETE', + path: '/api-keys/{id}', + handler: async (request, h) => { + + const { id } = request.params; + + const result = await db.collection(apiKeysTable).findOne({ _id: new ObjectID(id) }); + + if (!result) { + throw Boom.notFound('API Key not found'); + } + + if (!request.auth.credentials.roles.includes('admin') && request.auth.credentials.id !== result.user_id) { + return Boom.badRequest('The requesting user is unauthorized to make that request'); + } + + await db.collection(apiKeysTable).updateOne( + { _id: new ObjectID(id) }, + { $set: { disabled: true } } // soft delete + ); + + return h.response({ message: 'API Key disabled' }).code(200); + }, + config: { + auth: { + strategy: 'jwt', + scope: ['admin', 'create_api_keys'] + }, + validate: { + headers: authorizationHeader, + params: apiKeyParam + }, + response: { + status: {} + }, + description: 'Delete an API key', + notes: '

Requires authorization via: JWT token

\ +

Available to: admin

', + tags: ['api_keys','api'] + } + }); + } +}; + diff --git a/routes/api/v1/auth.js b/routes/api/v1/auth.js index e1fe41e..b9602b7 100644 --- a/routes/api/v1/auth.js +++ b/routes/api/v1/auth.js @@ -65,6 +65,9 @@ const _rolesToScope = (roles) => { else if (role === 'cruise_manager') { return scope_accumulator.concat(['read_events', 'write_events', 'read_event_templates', 'write_event_templates', 'read_cruises', 'write_cruises', 'read_lowerings', 'write_lowerings', 'read_users', 'write_users']); } + else if (role === 'apikey_manager') { + return scope_accumulator.concat(['read_events', 'write_events', 'read_event_templates', 'write_event_templates', 'read_cruises', 'write_cruises', 'read_lowerings', 'write_lowerings', 'read_users', 'write_users', 'read_api_keys', 'write_api_keys']); + } return scope_accumulator; }, []); diff --git a/routes/api/v1/cruises.js b/routes/api/v1/cruises.js index 3e2029e..36a7ffd 100644 --- a/routes/api/v1/cruises.js +++ b/routes/api/v1/cruises.js @@ -237,7 +237,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_cruises'] }, validate: { @@ -341,7 +341,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_cruises'] }, validate: { @@ -435,7 +435,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_cruises'] }, validate: { @@ -513,7 +513,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_cruises'] }, validate: { @@ -577,7 +577,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_cruises'] }, validate: { @@ -697,7 +697,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'create_cruises'] }, validate: { @@ -714,7 +714,7 @@ exports.plugin = { } }, - description: 'Create a new event template', + description: 'Create a new cruise', notes: '

Requires authorization via: JWT token

\

Available to: admin

', tags: ['cruises','api'] @@ -892,7 +892,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_cruises'] }, validate: { @@ -908,8 +908,12 @@ exports.plugin = { status: { } }, description: 'Update a cruise record', - notes: '

Requires authorization via: JWT token

\ -

Available to: admin

', + notes: '

Requires authorization using either:

\ +
    \ +
  • JWT Bearer token
    • \ +
  • API Key (x-api-key)
    • \ +
\ +

Available to roles: admin, cruise_manager

', tags: ['cruises','api'] } }); @@ -1016,7 +1020,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_cruises'] }, validate: { @@ -1081,7 +1085,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'create_cruises'] }, validate: { @@ -1131,7 +1135,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin'] }, validate: { diff --git a/routes/api/v1/custom_vars.js b/routes/api/v1/custom_vars.js index dc129da..d4d009d 100644 --- a/routes/api/v1/custom_vars.js +++ b/routes/api/v1/custom_vars.js @@ -64,7 +64,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -115,7 +115,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -181,7 +181,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_events'] }, validate: { diff --git a/routes/api/v1/event_aux_data.js b/routes/api/v1/event_aux_data.js index 705ef03..2ba0e51 100644 --- a/routes/api/v1/event_aux_data.js +++ b/routes/api/v1/event_aux_data.js @@ -136,7 +136,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -236,7 +236,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -372,7 +372,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -416,7 +416,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -553,7 +553,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_events'] }, validate: { @@ -647,7 +647,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_events'] }, validate: { @@ -703,7 +703,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_events'] }, validate: { diff --git a/routes/api/v1/event_exports.js b/routes/api/v1/event_exports.js index cb4d09d..1366d64 100644 --- a/routes/api/v1/event_exports.js +++ b/routes/api/v1/event_exports.js @@ -167,7 +167,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -308,7 +308,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -457,7 +457,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -531,7 +531,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { diff --git a/routes/api/v1/event_templates.js b/routes/api/v1/event_templates.js index c21ef43..87de0f7 100644 --- a/routes/api/v1/event_templates.js +++ b/routes/api/v1/event_templates.js @@ -78,7 +78,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_event_templates'] }, validate: { @@ -134,7 +134,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_event_templates'] }, validate: { @@ -211,7 +211,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_event_templates'] }, validate: { @@ -284,7 +284,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_event_templates'] }, validate: { @@ -351,7 +351,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_event_templates'] }, validate: { diff --git a/routes/api/v1/events.js b/routes/api/v1/events.js index ce14245..c81d31d 100644 --- a/routes/api/v1/events.js +++ b/routes/api/v1/events.js @@ -170,7 +170,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -281,7 +281,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -412,7 +412,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -518,7 +518,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -654,7 +654,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -748,7 +748,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -810,7 +810,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_events'] }, validate: { @@ -915,7 +915,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_events'] }, validate: { @@ -1035,7 +1035,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_events'] }, validate: { @@ -1192,7 +1192,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin'] }, validate: { @@ -1277,7 +1277,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_events'] }, validate: { @@ -1321,7 +1321,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin'] }, validate: { diff --git a/routes/api/v1/external_calls.js b/routes/api/v1/external_calls.js index 004ef05..0bfbe5f 100644 --- a/routes/api/v1/external_calls.js +++ b/routes/api/v1/external_calls.js @@ -146,7 +146,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin'] }, validate: { @@ -194,7 +194,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin'] }, validate: { diff --git a/routes/api/v1/lowerings.js b/routes/api/v1/lowerings.js index 296a9eb..39047a8 100644 --- a/routes/api/v1/lowerings.js +++ b/routes/api/v1/lowerings.js @@ -227,7 +227,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_lowerings'] }, validate: { @@ -377,7 +377,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_lowerings'] }, validate: { @@ -471,7 +471,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_lowerings'] }, validate: { @@ -549,7 +549,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_lowerings'] }, validate: { @@ -614,7 +614,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'read_lowerings'] }, validate: { @@ -728,7 +728,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'create_lowerings'] }, validate: { @@ -878,7 +878,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_lowerings'] }, validate: { @@ -1000,7 +1000,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'write_lowerings'] }, validate: { @@ -1062,7 +1062,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin', 'create_lowerings'] }, validate: { @@ -1112,7 +1112,7 @@ exports.plugin = { }, config: { auth: { - strategy: 'jwt', + strategies: ['jwt', 'api-key'], scope: ['admin'] }, validate: { diff --git a/routes/api/v1/users.js b/routes/api/v1/users.js index 724e41b..c2eebdd 100644 --- a/routes/api/v1/users.js +++ b/routes/api/v1/users.js @@ -13,6 +13,7 @@ const { } = require('../../../config/server_settings'); const { + apiKeysTable, cruisesTable, loweringsTable, usersTable @@ -406,6 +407,10 @@ exports.plugin = { try { await db.collection(usersTable).updateOne(query, { $set: user }); + if (typeof query.disabled === 'boolean' && !query.disabled) { + await db.collection(apiKeysTable).updateMany({ user_id: query._id }, { $set: { disabled: false } }); + } + return h.response().code(204); } catch (err) { @@ -473,6 +478,7 @@ exports.plugin = { try { await db.collection(usersTable).deleteOne(query); + await db.collection(apiKeysTable).deleteMany({ user_id: query._id }); } catch (err) { console.log('ERROR:', err); From 39e661f83502438073e7ac56ef84250b63f19f1e Mon Sep 17 00:00:00 2001 From: Webb Pinner Date: Tue, 24 Feb 2026 07:00:50 -0400 Subject: [PATCH 05/17] Updates API key route paths Updates the API key route paths to use snake_case instead of kebab-case for consistency. This change ensures that the API adheres to a consistent naming convention for route paths. --- routes/api/v1/api_keys.js | 10 +- test/api_keys.test.js | 311 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 316 insertions(+), 5 deletions(-) create mode 100644 test/api_keys.test.js diff --git a/routes/api/v1/api_keys.js b/routes/api/v1/api_keys.js index d67867f..3d04f18 100644 --- a/routes/api/v1/api_keys.js +++ b/routes/api/v1/api_keys.js @@ -39,7 +39,7 @@ exports.plugin = { // ---------------- LIST API KEYS ---------------- server.route({ method: 'GET', - path: '/api-keys', + path: '/api_keys', handler: async (request, h) => { // if the request includes a user_id that is not the current user's ID and the user id @@ -83,7 +83,7 @@ exports.plugin = { server.route({ method: 'GET', - path: '/api-keys/{id}', + path: '/api_keys/{id}', handler: async (request, h) => { const { id } = request.params; @@ -126,7 +126,7 @@ exports.plugin = { // ---------------- CREATE API KEY ---------------- server.route({ method: 'POST', - path: '/api-keys', + path: '/api_keys', handler: async (request, h) => { const { label, roles = [], expires } = request.payload; @@ -179,7 +179,7 @@ exports.plugin = { // ---------------- PATCH / UPDATE LABEL OR EXPIRATION ---------------- server.route({ method: 'PATCH', - path: '/api-keys/{id}', + path: '/api_keys/{id}', handler: async (request, h) => { const { id } = request.params; @@ -244,7 +244,7 @@ exports.plugin = { // ---------------- DELETE / REVOKE API KEY ---------------- server.route({ method: 'DELETE', - path: '/api-keys/{id}', + path: '/api_keys/{id}', handler: async (request, h) => { const { id } = request.params; diff --git a/test/api_keys.test.js b/test/api_keys.test.js new file mode 100644 index 0000000..661fe9c --- /dev/null +++ b/test/api_keys.test.js @@ -0,0 +1,311 @@ +'use strict'; + +const Lab = require('@hapi/lab'); +const { expect } = require('@hapi/code'); +const { beforeEach, afterEach, describe, it } = exports.lab = Lab.script(); +const { init } = require('../lib/server'); +const { randomAsciiString, hashedApiKey } = require('../lib/utils'); + +const { apikeysTable, usersTable } = require('../config/db_constants') + +const Jwt = require('jsonwebtoken'); +const { ObjectId } = require('mongodb'); + +const SECRET = require('../config/secret'); + +describe('Users API', () => { + let server; + let db; + let users; + + const adminLoginToken = randomAsciiString(20) + const normalLoginToken = randomAsciiString(20) + const apiKey = randomAsciiString(20) + const apiKeyHash = hashedApiKey(apiKey) + + const adminUser = { + _id: ObjectId('000000000000000000000001'), + username: 'admin', + fullname: 'test_admin', + email: 'admin@example.com', + password: 'hashed', + last_login: new Date(), + roles: ['admin'], + system_user: false, + disabled: false, + loginToken: adminLoginToken + }; + + const normalUser = { + _id: ObjectId('000000000000000000000002'), + username: 'bob', + fullname: 'test_bob', + email: 'bob@example.com', + password: 'hashedbob', + roles: ['cruise_manager'], + system_user: false, + disabled: false, + loginToken: normalLoginToken + }; + + const apiKeyRecord = [ + { + _id: ObjectId('000000000000000000000003'), + user_id: ObjectId('000000000000000000000002'), // Reference to users collection + key_hash: apiKeyHash, // We store a hash, never raw key + label: 'Default Key', + scope: ['read_cruises'], // Optional: can match user scopes or add more granular scopes + created: new Date(), + last_used: null, + disabled: false, + expires: null + } + ]; + + const adminJwt = Jwt.sign( + { id: adminUser._id, roles: adminUser.roles, scope: ['admin'] }, + SECRET + ); + + const normalJwt = Jwt.sign( + { id: normalUser._id, roles: normalUser.roles, scope: ['read_users'] }, + SECRET + ); + + beforeEach(async () => { + server = await init(); + db = server.mongo.db; + + users = db.collection(usersTable); + apikeys = db.collection(apikeysTable); + + await users.deleteMany({}); + await users.insertMany([adminUser, normalUser]); + await apikeys.deleteMany({}); + await apikeys.insertMany([apiKeyRecord]); + }); + + afterEach(async () => { + await server.stop(); + }); + + // ─────────────────────────────────────────────── + // GET /users + // ─────────────────────────────────────────────── + describe('GET /api_keys', () => { + it('returns all api_keys for admin', async () => { + const res = await server.inject({ + method: 'GET', + url: '/sealog-server/api/v1/api_keys', + headers: { Authorization: 'Bearer ' + adminJwt } + }); + + expect(res.statusCode).to.equal(200); + expect(res.result.length).to.equal(1); + expect(res.result[0]).to.not.include('key_hash'); + }); + + // it('returns only non-system api_keys for non-admin', async () => { + // // mark admin as system user + // await api_keys.updateOne( + // { _id: adminUser._id }, + // { $set: { system_user: true } } + // ); + + // const res = await server.inject({ + // method: 'GET', + // url: '/sealog-server/api/v1/api_keys', + // headers: { Authorization: 'Bearer ' + normalJwt } + // }); + + // expect(res.statusCode).to.equal(200); + // expect(res.result.length).to.equal(1); + // expect(res.result[0].username).to.equal('bob'); + // }); + }); + + // ─────────────────────────────────────────────── + // GET /api_keys/{id} + // ─────────────────────────────────────────────── +// describe('GET /api_keys/{id}', () => { +// it('returns the user for admin', async () => { +// const res = await server.inject({ +// method: 'GET', +// url: `/sealog-server/api/v1/api_keys/${normalUser._id}`, +// headers: { Authorization: 'Bearer ' + adminJwt } +// }); + +// expect(res.statusCode).to.equal(200); +// expect(res.result.username).to.equal('bob'); +// }); + +// it('blocks non-admin from accessing system_user', async () => { +// await api_keys.updateOne( +// { _id: ObjectId(adminUser._id) }, +// { $set: { system_user: true } } +// ); + +// const res = await server.inject({ +// method: 'GET', +// url: `/sealog-server/api/v1/api_keys/${adminUser._id}`, +// headers: { Authorization: 'Bearer ' + normalJwt } +// }); + +// expect(res.statusCode).to.equal(400); +// }); +// }); + +// // ─────────────────────────────────────────────── +// // POST /api_keys +// // ─────────────────────────────────────────────── +// describe('POST /api_keys', () => { +// it('creates a new user', async () => { +// const payload = { +// username: 'charlie', +// password: 'pass123', +// fullname: 'test_charlie', +// email: 'charlie@example.com', +// resetURL: 'https://example/reset/', +// roles: ['event_logger'] +// }; + +// const res = await server.inject({ +// method: 'POST', +// url: '/sealog-server/api/v1/api_keys', +// headers: { Authorization: 'Bearer ' + adminJwt }, +// payload +// }); + +// expect(res.statusCode).to.equal(201); + +// const newUser = await api_keys.findOne({ username: 'charlie' }); +// expect(newUser).to.exist(); +// expect(newUser.password).to.exist(); +// }); + +// it('rejects duplicate username', async () => { +// const payload = { +// username: 'bob', +// password: 'pass123', +// fullname: 'test_bob', +// email: 'x@example.com', +// resetURL: 'https://example/reset/', +// roles: ['event_logger'] +// }; + +// const res = await server.inject({ +// method: 'POST', +// url: '/sealog-server/api/v1/api_keys', +// headers: { Authorization: 'Bearer ' + adminJwt }, +// payload +// }); + +// expect(res.statusCode).to.equal(409); +// }); +// }); + +// // ─────────────────────────────────────────────── +// // PATCH /api_keys/{id} +// // ─────────────────────────────────────────────── +// describe('PATCH /api_keys/{id}', () => { +// it('updates a user when authorized', async () => { +// const res = await server.inject({ +// method: 'PATCH', +// url: `/sealog-server/api/v1/api_keys/${normalUser._id}`, +// headers: { Authorization: 'Bearer ' + normalJwt }, +// payload: { fullname: 'new_bob' } +// }); + +// expect(res.statusCode).to.equal(204); + +// const updated = await api_keys.findOne({ _id: normalUser._id }); +// expect(updated.fullname).to.equal('new_bob'); +// }); + +// it('disallows non-admin modifying system_user account', async () => { +// await api_keys.updateOne( +// { _id: adminUser._id }, +// { $set: { system_user: true } } +// ); + +// const res = await server.inject({ +// method: 'PATCH', +// url: `/sealog-server/api/v1/api_keys/${adminUser._id}`, +// headers: { Authorization: 'Bearer ' + normalJwt }, +// payload: { fullname: 'new_admin' } +// }); + +// expect(res.statusCode).to.equal(400); +// }); +// }); + +// // ─────────────────────────────────────────────── +// // DELETE /api_keys/{id} +// // ─────────────────────────────────────────────── +// describe('DELETE /api_keys/{id}', () => { +// it('deletes a user as admin', async () => { +// const res = await server.inject({ +// method: 'DELETE', +// url: `/sealog-server/api/v1/api_keys/${normalUser._id}`, +// headers: { Authorization: 'Bearer ' + adminJwt } +// }); + +// expect(res.statusCode).to.equal(204); + +// const gone = await api_keys.findOne({ _id: normalUser._id }); +// expect(gone).to.not.exist(); +// }); + +// it('prevents user from deleting self', async () => { +// const res = await server.inject({ +// method: 'DELETE', +// url: `/sealog-server/api/v1/api_keys/${adminUser._id}`, +// headers: { Authorization: 'Bearer ' + adminJwt } +// }); + +// expect(res.statusCode).to.equal(400); +// }); +// }); + +// // ─────────────────────────────────────────────── +// // GET /api_keys/{id}/token +// // ─────────────────────────────────────────────── +// describe('GET /api_keys/{id}/token', () => { +// it('returns JWT for owner', async () => { +// const res = await server.inject({ +// method: 'GET', +// url: `/sealog-server/api/v1/api_keys/${normalUser._id}/token`, +// headers: { Authorization: 'Bearer ' + normalJwt } +// }); + +// expect(res.statusCode).to.equal(200); +// expect(res.result.token).to.exist(); +// }); + +// it('blocks others unless admin', async () => { +// const res = await server.inject({ +// method: 'GET', +// url: `/sealog-server/api/v1/api_keys/${normalUser._id}/token`, +// headers: { Authorization: 'Bearer ' + adminJwt } +// }); + +// expect(res.statusCode).to.equal(200); +// }); +// }); + +// // ─────────────────────────────────────────────── +// // GET /api_keys/{id}/loginToken +// // ─────────────────────────────────────────────── +// describe('GET /api_keys/{id}/loginToken', () => { +// it('returns loginToken for user', async () => { +// const res = await server.inject({ +// method: 'GET', +// url: `/sealog-server/api/v1/api_keys/${normalUser._id}/loginToken`, +// headers: { Authorization: 'Bearer ' + normalJwt } +// }); + +// expect(res.statusCode).to.equal(200); +// expect(res.result.loginToken).to.equal(normalLoginToken); +// }); +// }); +}); From 1568f68a3138d91c534916c4280fecfbc4be6777 Mon Sep 17 00:00:00 2001 From: ljones Date: Tue, 25 Nov 2025 10:42:50 -0800 Subject: [PATCH 06/17] Make AuxDataRecordBuilder base class and inherit from it in the Influx and Coriolix implementations Note: initial refactor was done by CoPilot and then edited by me --- misc/base_aux_data_record_builder.py | 214 ++++++++++++++++++ .../aux_data_record_builder.py | 154 +------------ misc/influx_sealog/aux_data_record_builder.py | 162 +------------ 3 files changed, 235 insertions(+), 295 deletions(-) create mode 100644 misc/base_aux_data_record_builder.py diff --git a/misc/base_aux_data_record_builder.py b/misc/base_aux_data_record_builder.py new file mode 100644 index 0000000..37de82c --- /dev/null +++ b/misc/base_aux_data_record_builder.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +''' +FILE: base_aux_data_record_builder.py + +DESCRIPTION: Base class for building sealog aux_data records from various data sources. + +BUGS: +NOTES: +AUTHOR: Webb Pinner +COMPANY: OceanDataTools.org +VERSION: 1.0 +CREATED: 2025-02-25 +REVISION: + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' +import json +import logging +from abc import ABC, abstractmethod + +class AuxDataRecordBuilder(ABC): + ''' + Abstract base class for building sealog aux_data records from various data sources. + Handles common functionality for querying external data sources and building + aux_data records with transformations. + ''' + + def __init__(self, aux_data_config): + ''' + Initialize the base builder with configuration. + + Args: + aux_data_config (dict): Configuration dictionary containing: + - query_measurements: List of measurements to query + - aux_record_lookup: Mapping of fields to output configuration + - data_source: Name of the data source + ''' + self._query_measurements = aux_data_config['query_measurements'] + self._query_fields = list(aux_data_config['aux_record_lookup'].keys()) + self._aux_record_lookup = aux_data_config['aux_record_lookup'] + self._data_source = aux_data_config['data_source'] + self.logger = logging.getLogger(__name__) + + @staticmethod + @abstractmethod + def _build_query_range(ts): + ''' + Builds the temporal range for queries based on the provided timestamp (ts). + Format depends on the specific data source implementation. + + Args: + ts (str): Timestamp in ISO 8601 format (YYYY-MM-DDTHH:MM:SS.fffZ) + + Returns: + str or None: Query range string (format varies by implementation) + ''' + # I'm not sure why we need this to be a static method, but I'm including it in the base class since it's in both implementations + pass + + + @abstractmethod + def build_aux_data_record(self, event): + ''' + Build the aux_data record for the given event. + Must be implemented by subclasses. + + Args: + event (dict): Event dictionary containing 'id' and 'ts' keys + + Returns: + dict or None: Aux data record or None if no data available + ''' + pass + + def _build_aux_data_dict(self, event_id, query_data): # pylint:disable=R0915 + ''' + Internal method to build the sealog aux_data record using the event_id, + query_data and the class instance's datasource value. + + This method handles common transformations and modifications across all data sources. + + Args: + event_id (str): The ID of the event + query_data (dict): Dictionary of field names to values from the data source + + Returns: + dict or None: Aux data record or None if no data available + ''' + aux_data_record = { + 'event_id': event_id, + 'data_source': self._data_source, + 'data_array': [] + } + + logging.debug("raw values: %s", json.dumps(query_data, indent=2)) + + if not query_data: + return None + + for key, value in self._aux_record_lookup.items(): + try: + if "no_output" in value and value['no_output'] is True: + continue + + if key not in query_data: + continue + + output_value = query_data[key] + + if "modify" in value: + logging.debug("modify found in record") + for mod_op in value['modify']: + test_result = True + + if 'test' in mod_op: + logging.debug("test found in mod_op") + test_result = False + + for test in mod_op['test']: + logging.debug(json.dumps(test)) + + if 'field' in test: + + if test['field'] not in query_data: + logging.error("test field data not in query results") + return None + + if 'eq' in test and query_data[test['field']] == test['eq']: + test_result = True + break + + if 'gt' in test and query_data[test['field']] > test['gt']: + test_result = True + break + + if 'gte' in test and query_data[test['field']] >= test['gt']: + test_result = True + break + + if 'lt' in test and query_data[test['field']] < test['lt']: + test_result = True + break + + if 'lte' in test and query_data[test['field']] <= test['lt']: + test_result = True + break + + if 'ne' in test and query_data[test['field']] != test['ne']: + test_result = True + break + + if test_result and 'operation' in mod_op: + logging.debug("operation found in mod_op") + for operan in mod_op['operation']: + + if 'add' in operan: + output_value += operan['add'] + + if 'subtract' in operan: + output_value -= operan['subtract'] + + if 'multiply' in operan: + output_value *= operan['multiply'] + + if 'divide' in operan: + output_value /= operan['divide'] + + aux_data_record['data_array'].append({ + 'data_name': value['name'], + 'data_value': ( + str(round(output_value, value['round'])) + if 'round' in value + else str(output_value) + ), + 'data_uom': value['uom'] if 'uom' in value else '' + }) + except ValueError as exc: + logging.warning("Problem adding %s", key) + logging.debug(str(exc)) + continue + + if len(aux_data_record['data_array']) > 0: + return aux_data_record + + return None + + @property + def data_source(self): + ''' + Getter method for the data_source property + ''' + return self._data_source + + @property + def measurements(self): + ''' + Getter method for the _query_measurements property + ''' + return self._query_measurements + + @property + def fields(self): + ''' + Getter method for the _query_fields property + ''' + return self._query_fields + + @property + def record_lookup(self): + ''' + Getter method for the _aux_record_lookup property + ''' + return self._aux_record_lookup \ No newline at end of file diff --git a/misc/coriolix_sealog/aux_data_record_builder.py b/misc/coriolix_sealog/aux_data_record_builder.py index f3bfce2..aa61153 100644 --- a/misc/coriolix_sealog/aux_data_record_builder.py +++ b/misc/coriolix_sealog/aux_data_record_builder.py @@ -2,8 +2,8 @@ ''' FILE: aux_data_record_builder.py -DESCRIPTION: This script builds a sealog aux_data record with data pulled from an - influx database. +DESCRIPTION: This script builds a sealog aux_data record with data pulled from a + CORIOLIX API. BUGS: NOTES: @@ -28,27 +28,24 @@ from os.path import dirname, realpath sys.path.append(dirname(dirname(dirname(realpath(__file__))))) +from misc.base_aux_data_record_builder import AuxDataRecordBuilder from misc.coriolix_sealog.settings import CORIOLIX_URL -class SealogCORIOLIXAuxDataRecordBuilder(): +class SealogCORIOLIXAuxDataRecordBuilder(AuxDataRecordBuilder): ''' - Class that handles the construction of an influxDB query and using the + Class that handles the construction of CORIOLIX API queries and using the resulting data to build a sealog aux_data record. ''' def __init__(self, aux_data_config, url=None): + super().__init__(aux_data_config) self.url = url or CORIOLIX_URL - self._query_measurements = aux_data_config['query_measurements'] - self._query_fields = list(aux_data_config['aux_record_lookup'].keys()) - self._aux_record_lookup = aux_data_config['aux_record_lookup'] - self._data_source = aux_data_config['data_source'] - self.logger = logging.getLogger(__name__) @staticmethod def _build_query_range(ts): ''' - Builds the temporal range for the influxDB query based on the provided + Builds the temporal range for the CORIOLIX query based on the provided timestamp (ts). ''' try: @@ -63,7 +60,7 @@ def _build_query_range(ts): def _build_query_urls(self, ts): ''' - Builds the complete influxDB query using the provided timestamp (ts) + Builds the CORIOLIX API URLs using the provided timestamp (ts) and the class instance's query_measurements and query_fields values. ''' @@ -77,109 +74,6 @@ def _build_query_urls(self, ts): return query_urls - def _build_aux_data_dict(self, event_id, query_results): # pylint:disable=R0915 - ''' - Internal method to build the sealog aux_data record using the event_id, - query_results and the class instance's datasource value. - ''' - - aux_data_record = { - 'event_id': event_id, - 'data_source': self._data_source, - 'data_array': [] - } - - coriolix_data = query_results - - logging.debug("raw values: %s", json.dumps(coriolix_data, indent=2)) - - if not coriolix_data: - return None - - for key, value in self._aux_record_lookup.items(): - try: - if "no_output" in value and value['no_output'] is True: - continue - - if key not in coriolix_data: - continue - - output_value = coriolix_data[key] - - if "modify" in value: - logging.debug("modify found in record") - for mod_op in value['modify']: - test_result = True - - if 'test' in mod_op: - logging.debug("test found in mod_op") - test_result = False - - for test in mod_op['test']: - logging.debug(json.dumps(test)) - - if 'field' in test: - - if test['field'] not in coriolix_data: - logging.warning("test field data not in CORIOLIX query") - return None - - if 'eq' in test and coriolix_data[test['field']] == test['eq']: - test_result = True - break - - if 'gt' in test and coriolix_data[test['field']] > test['gt']: - test_result = True - break - - if 'gte' in test and coriolix_data[test['field']] >= test['gt']: - test_result = True - break - - if 'lt' in test and coriolix_data[test['field']] < test['lt']: - test_result = True - break - - if 'lte' in test and coriolix_data[test['field']] <= test['lt']: - test_result = True - break - - if 'ne' in test and coriolix_data[test['field']] != test['ne']: - test_result = True - break - - if test_result and 'operation' in mod_op: - logging.debug("operation found in mod_op") - for operan in mod_op['operation']: - - if 'add' in operan: - output_value += operan['add'] - - if 'subtract' in operan: - output_value -= operan['subtract'] - - if 'multiply' in operan: - output_value *= operan['multiply'] - - if 'divide' in operan: - output_value /= operan['divide'] - - aux_data_record['data_array'].append({ - 'data_name': value['name'], - 'data_value': str(round(output_value, value['round'])) if 'round' in value - else str(output_value), - 'data_uom': value['uom'] if 'uom' in value else '' - }) - except ValueError as exc: - logging.warning("Problem adding %s", key) - logging.debug(str(exc)) - continue - - if len(aux_data_record['data_array']) > 0: - return aux_data_record - - return None - def build_aux_data_record(self, event): ''' Build the aux_data record for the given event. @@ -194,7 +88,7 @@ def build_aux_data_record(self, event): logging.debug("Query URL: %s", url) measurement = os.path.basename(urlparse(url).path.strip('/')) - # run the query against the influxDB + # run the query against the CORIOLIX API try: response = requests.get(url, timeout=2) if response.status_code != 200: @@ -222,32 +116,4 @@ def build_aux_data_record(self, event): logging.error("Something went wrong processing the API response") aux_data_record = self._build_aux_data_dict(event['id'], query_results) - return aux_data_record - - @property - def data_source(self): - ''' - Getter method for the data_source property - ''' - return self._data_source - - @property - def measurements(self): - ''' - Getter method for the _query_measurements property - ''' - return self._query_measurements - - @property - def fields(self): - ''' - Getter method for the _query_fields property - ''' - return self._query_fields - - @property - def record_lookup(self): - ''' - Getter method for the _aux_record_lookup property - ''' - return self._aux_record_lookup + return aux_data_record \ No newline at end of file diff --git a/misc/influx_sealog/aux_data_record_builder.py b/misc/influx_sealog/aux_data_record_builder.py index 4bab6ab..8083a9e 100644 --- a/misc/influx_sealog/aux_data_record_builder.py +++ b/misc/influx_sealog/aux_data_record_builder.py @@ -17,7 +17,6 @@ Copyright (C) OceanDataTools.org 2025 ''' import sys -import json import logging from datetime import datetime, timedelta from urllib3.exceptions import NewConnectionError @@ -26,6 +25,7 @@ from os.path import dirname, realpath sys.path.append(dirname(dirname(dirname(realpath(__file__))))) +from misc.base_aux_data_record_builder import AuxDataRecordBuilder from misc.influx_sealog.settings import ( INFLUXDB_URL, INFLUXDB_AUTH_TOKEN, @@ -34,23 +34,19 @@ ) -class SealogInfluxAuxDataRecordBuilder(): +class SealogInfluxAuxDataRecordBuilder(AuxDataRecordBuilder): ''' Class that handles the construction of an influxDB query and using the resulting data to build a sealog aux_data record. ''' def __init__(self, influxdb_client, aux_data_config, influxdb_bucket=INFLUXDB_BUCKET): + super().__init__(aux_data_config) self._influxdb_client = influxdb_client.query_api() self._influxdb_bucket = ( aux_data_config['query_bucket'] if 'query_bucket' in aux_data_config else influxdb_bucket ) - self._query_measurements = aux_data_config['query_measurements'] - self._query_fields = list(aux_data_config['aux_record_lookup'].keys()) - self._aux_record_lookup = aux_data_config['aux_record_lookup'] - self._data_source = aux_data_config['data_source'] - self.logger = logging.getLogger(__name__) @staticmethod def _build_query_range(ts): @@ -94,120 +90,6 @@ def _build_query(self, ts): logging.debug("Query: %s", query) return query - def _build_aux_data_dict( - self, event_id, influx_query_result - ): # pylint:disable=R0915 - ''' - Internal method to build the sealog aux_data record using the event_id, - influx_query_result and the class instance's datasource value. - ''' - - aux_data_record = { - 'event_id': event_id, - 'data_source': self._data_source, - 'data_array': [] - } - - influx_data = { - } - - for table in influx_query_result: - for record in table.records: - - influx_data[record.get_field()] = record.get_value() - - logging.debug("raw values: %s", json.dumps(influx_data, indent=2)) - - if not influx_data: - return None - - for key, value in self._aux_record_lookup.items(): - try: - if "no_output" in value and value['no_output'] is True: - continue - - if key not in influx_data: - continue - - output_value = influx_data[key] - - if "modify" in value: - logging.debug("modify found in record") - for mod_op in value['modify']: - test_result = True - - if 'test' in mod_op: - logging.debug("test found in mod_op") - test_result = False - - for test in mod_op['test']: - logging.debug(json.dumps(test)) - - if 'field' in test: - - if test['field'] not in influx_data: - logging.error("test field data not in influx query") - return None - - if 'eq' in test and influx_data[test['field']] == test['eq']: - test_result = True - break - - if 'gt' in test and influx_data[test['field']] > test['gt']: - test_result = True - break - - if 'gte' in test and influx_data[test['field']] >= test['gt']: - test_result = True - break - - if 'lt' in test and influx_data[test['field']] < test['lt']: - test_result = True - break - - if 'lte' in test and influx_data[test['field']] <= test['lt']: - test_result = True - break - - if 'ne' in test and influx_data[test['field']] != test['ne']: - test_result = True - break - - if test_result and 'operation' in mod_op: - logging.debug("operation found in mod_op") - for operan in mod_op['operation']: - - if 'add' in operan: - output_value += operan['add'] - - if 'subtract' in operan: - output_value -= operan['subtract'] - - if 'multiply' in operan: - output_value *= operan['multiply'] - - if 'divide' in operan: - output_value /= operan['divide'] - - aux_data_record['data_array'].append({ - 'data_name': value['name'], - 'data_value': ( - str(round(output_value, value['round'])) - if 'round' in value - else str(output_value) - ), - 'data_uom': value['uom'] if 'uom' in value else '' - }) - except ValueError as exc: - logging.warning("Problem adding %s", key) - logging.debug(str(exc)) - continue - - if len(aux_data_record['data_array']) > 0: - return aux_data_record - - return None - def build_aux_data_record(self, event): ''' Build the aux_data record for the given event. @@ -239,36 +121,14 @@ def build_aux_data_record(self, event): logging.error(str(exc)) raise exc else: - aux_data_record = self._build_aux_data_dict(event['id'], query_result) - - return aux_data_record + # Parse InfluxDB result into a dictionary format + influx_data = {} + for table in query_result: + for record in table.records: + influx_data[record.get_field()] = record.get_value() - return None + aux_data_record = self._build_aux_data_dict(event['id'], influx_data) - @property - def data_source(self): - ''' - Getter method for the data_source property - ''' - return self._data_source - - @property - def measurements(self): - ''' - Getter method for the _query_measurements property - ''' - return self._query_measurements - - @property - def fields(self): - ''' - Getter method for the _query_fields property - ''' - return self._query_fields + return aux_data_record - @property - def record_lookup(self): - ''' - Getter method for the _aux_record_lookup property - ''' - return self._aux_record_lookup + return None \ No newline at end of file From 3c6f6ed0fd92fc7a860404541aa9ebbdf6e4ad47 Mon Sep 17 00:00:00 2001 From: ljones Date: Thu, 5 Mar 2026 13:39:50 -0800 Subject: [PATCH 07/17] Moved common aux data inserter code into a separate file --- misc/aux_data_inserter_runner.py | 268 ++++++++++++++++ .../sealog_aux_data_inserter_coriolix.py.dist | 294 +---------------- misc/sealog_aux_data_inserter_influx.py.dist | 300 +----------------- 3 files changed, 291 insertions(+), 571 deletions(-) create mode 100644 misc/aux_data_inserter_runner.py diff --git a/misc/aux_data_inserter_runner.py b/misc/aux_data_inserter_runner.py new file mode 100644 index 0000000..ff81787 --- /dev/null +++ b/misc/aux_data_inserter_runner.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +''' +FILE: sealog_aux_data_inserter_runner.py + +DESCRIPTION: Shared runner for aux-data inserter scripts. +''' +import re +import sys +import json +import time +import logging +import asyncio +import websockets +import yaml +from typing import Callable, Dict, Any + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(realpath(__file__)))) + +from misc.base_aux_data_record_builder import AuxDataRecordBuilder +from misc.python_sealog.events import get_event, get_events_by_cruise, get_events_by_lowering +from misc.python_sealog.event_aux_data import create_event_aux_data +from misc.python_sealog.lowerings import get_lowering_uid_by_id +from misc.python_sealog.cruises import get_cruise_uid_by_id + +EXCLUDE_SET = set() #TODO: not sure what you would want to put here. Should this on the individual inserters instead? + +def parse_event_ids(event_id_file): + ''' + Builds list of event uid from csv-formatted file. + ''' + event_ids_from_file = [] + with open(event_id_file, 'r', encoding='utf-8') as event_id_fp: + for line in event_id_fp: + line = line.rstrip('\n') + logging.debug(line) + event_ids_from_file += line.split(',') + + event_ids_from_file = [event_id.strip() for event_id in event_ids_from_file] + + for event_id in event_ids_from_file: + if re.match(r"^[a-f\d]{24}$", event_id) is None: + logging.error("\"%s\" is an invalid event_id... quiting", event_id) + raise ValueError(f'"{event_id}" is an invalid event_id... quiting') + + return event_ids_from_file + + +def insert_aux_data(aux_data_builders, event, dry_run=False): + ''' + Add aux_data records for only the specified event + ''' + for builder in aux_data_builders: + logging.debug("Building aux data record") + record = builder.build_aux_data_record(event) + if record: + try: + logging.debug("Submitting aux data record to Sealog Server") + logging.debug(json.dumps(record)) + if not dry_run: + create_event_aux_data(record) + + except Exception as exc: # pylint:disable=W0718 + logging.warning("Error submitting aux data record") + logging.debug(str(exc)) + else: + logging.debug("No aux data for data_source: %s", builder.data_source) + + +def insert_aux_data_from_list(aux_data_builders, event_ids_from_list, dry_run=False): + ''' + Add aux_data records for only the events in the specified list + ''' + for event_id in event_ids_from_list: + try: + logging.debug("Retrieving event record from Sealog Server") + event = get_event(event_id) + logging.debug("Event: %s", event) + + except Exception as exc: + logging.warning("Error submitting aux data record") + logging.debug(str(exc)) + raise exc + + insert_aux_data(aux_data_builders, event, dry_run) + + +def insert_aux_data_for_cruise(aux_data_builders, cruise_id, dry_run=False): + ''' + Add aux_data records for only the events in the specified cruise + ''' + cruise_uid = get_cruise_uid_by_id(cruise_id) + + # exit if no cruise found + if not cruise_uid: + logging.error("cruise not found") + return + + # retrieve events for cruise + cruise_events = get_events_by_cruise(cruise_uid) + + # exit if no cruise found + if not cruise_events: + logging.error("no events found for cruise") + return + + for event in cruise_events: + insert_aux_data(aux_data_builders, event, dry_run) + + +def insert_aux_data_for_lowering(aux_data_builders, lowering_id, dry_run=False): + ''' + Add aux_data records for only the events in the specified lowering + ''' + lowering_uid = get_lowering_uid_by_id(lowering_id) + + # exit if no lowering found + if not lowering_uid: + logging.error("lowering not found") + return + + # retrieve events for lowering + lowering_events = get_events_by_lowering(lowering_uid) + + # exit if no lowering found + if not lowering_events: + logging.error("no events found for lowering") + return + + for event in lowering_events: + insert_aux_data(aux_data_builders, event, dry_run) + +async def insert_aux_data_from_ws(aux_data_builders, ws_server_url, headers, client_wsid, dry_run=False): + ''' + Use the aux_data_builder and to submit aux_data + records built from external data to the sealog-server API + ''' + try: + async with websockets.connect(ws_server_url) as websocket: + + HELLO = { + 'type': 'hello', + 'id': client_wsid, + 'auth': {'headers': headers}, + 'version': '2', + 'subs': ['/ws/status/newEvents'] + } + await websocket.send(json.dumps(HELLO)) + + while True: + + event = await websocket.recv() + event_obj = json.loads(event) + + if event_obj['type'] and event_obj['type'] == 'ping': + PING = { + 'type': 'ping', + 'id': client_wsid + } + await websocket.send(json.dumps(PING)) + + elif event_obj['type'] and event_obj['type'] == 'pub': + + if event_obj['message']['event_value'] in EXCLUDE_SET: + logging.debug("Skipping because event value is in the exclude set") + continue + + logging.debug("Event: %s", event_obj['message']) + + insert_aux_data(aux_data_builders, event_obj['message'], dry_run) + + except Exception as exc: + logging.error(str(exc)) + raise exc + +def run_aux_data_inserter( + builder_factory: Callable[[Dict[str, Any]], AuxDataRecordBuilder], + inline_config: str, + ws_server_url: str, + headers: Dict[str, Any], + client_wsid: str, +): + """ + Shared CLI and main loop. + + builder_factory: callable(config_dict) -> builder instance + inline_config: YAML string used if -f not provided + ws_server_url, headers, client_wsid: for websocket connection. + """ + import argparse + import os + + parser = argparse.ArgumentParser(description='Aux Data Inserter Service (shared)') + parser.add_argument('-v', '--verbosity', dest='verbosity', default=0, action='count', help='Increase output verbosity') + parser.add_argument('-f', '--config_file', help='use the specifed configuration file') + parser.add_argument('-n', '--dry_run', action='store_true', help='compile the aux_data records but do not submit to server API') + parser.add_argument('-e', '--events', help='list of event_ids to apply aux data') + parser.add_argument('-c', '--cruise_id', help='cruise_id to fix aux_data for') + parser.add_argument('-l', '--lowering_id', help='lowering_id to fix aux_data for') + + parsed_args = parser.parse_args() + + # Logging setup + LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s' + logging.basicConfig(format=LOGGING_FORMAT) + LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} + parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS)) + logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity]) + + if parsed_args.config_file: + config_file, data_source = ( + parsed_args.config_file.split(":") + if ":" in parsed_args.config_file + else [parsed_args.config_file, None] + ) + + try: + with open(config_file, 'r', encoding='utf-8') as config_fp: + aux_data_configs = yaml.safe_load(config_fp) + if data_source: + aux_data_configs = [c for c in aux_data_configs if c['data_source'] == data_source] + except yaml.parser.ParserError: + logging.error("Invalid YAML syntax") + sys.exit(1) + else: + try: + aux_data_configs = yaml.safe_load(inline_config) + except yaml.parser.ParserError: + logging.error("Invalid YAML syntax") + sys.exit(1) + + logging.debug(json.dumps(aux_data_configs, indent=2)) + + # Build builders + aux_data_builder_list = [builder_factory(cfg) for cfg in aux_data_configs] + + # Dispatch actions requested on CLI + if parsed_args.events: + event_ids = parse_event_ids(parsed_args.events) + logging.info("Event IDs:\n%s", json.dumps(event_ids, indent=2)) + insert_aux_data_from_list(aux_data_builder_list, event_ids, parsed_args.dry_run) + sys.exit(0) + + if parsed_args.cruise_id: + insert_aux_data_for_cruise(aux_data_builder_list, parsed_args.cruise_id, parsed_args.dry_run) + sys.exit(0) + + if parsed_args.lowering_id: + insert_aux_data_for_lowering(aux_data_builder_list, parsed_args.lowering_id, parsed_args.dry_run) + sys.exit(0) + + # Wait then start WS loop forever + while True: + time.sleep(5) + try: + logging.debug("Connecting to event websocket feed...") + asyncio.get_event_loop().run_until_complete( + insert_aux_data_from_ws(aux_data_builder_list, ws_server_url, headers, client_wsid, parsed_args.dry_run) + ) + except KeyboardInterrupt: + logging.error('Keyboard Interrupted') + try: + sys.exit(0) + except SystemExit: + os._exit(0) + except Exception as err: # pylint:disable=W0718 + logging.debug(str(err)) + logging.error("Lost connection to server, trying again in 5 seconds") \ No newline at end of file diff --git a/misc/sealog_aux_data_inserter_coriolix.py.dist b/misc/sealog_aux_data_inserter_coriolix.py.dist index ce28f06..3cef8fa 100644 --- a/misc/sealog_aux_data_inserter_coriolix.py.dist +++ b/misc/sealog_aux_data_inserter_coriolix.py.dist @@ -29,25 +29,13 @@ LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for det Copyright (C) OceanDataTools.org 2025 ''' -import re import sys -import json -import time -import logging -import asyncio -import websockets -import yaml - from os.path import dirname, realpath sys.path.append(dirname(dirname(realpath(__file__)))) -from misc.python_sealog.events import get_event, get_events_by_cruise, get_events_by_lowering -from misc.python_sealog.event_aux_data import create_event_aux_data -from misc.python_sealog.lowerings import get_lowering_uid_by_id -from misc.python_sealog.cruises import get_cruise_uid_by_id - from misc.python_sealog.settings import WS_SERVER_URL, HEADERS from misc.coriolix_sealog.aux_data_record_builder import SealogCORIOLIXAuxDataRecordBuilder +from misc.aux_data_inserter_runner import run_aux_data_inserter # ----------------------------------------------------------------------------- # @@ -105,284 +93,22 @@ INLINE_CONFIG = ''' - multiply: 0.539957 ''' -# set of events to ignore -EXCLUDE_SET = set() - # needs to be unique for all currently active dataInserter scripts. CLIENT_WSID = 'auxData-dataInserter-coriolix' -HELLO = { - 'type': 'hello', - 'id': CLIENT_WSID, - 'auth': { - 'headers': HEADERS - }, - 'version': '2', - 'subs': ['/ws/status/newEvents'] -} - -PING = { - 'type': 'ping', - 'id': CLIENT_WSID -} - - -def parse_event_ids(event_id_file): - ''' - Builds list of event uid from csv-formatted file. - ''' - event_ids_from_file = [] - with open(event_id_file, 'r', encoding='utf-8') as event_id_fp: - for line in event_id_fp: - line = line.rstrip('\n') - logging.debug(line) - event_ids_from_file += line.split(',') - - event_ids_from_file = [event_id.strip() for event_id in event_ids_from_file] - - for event_id in event_ids_from_file: - if re.match(r"^[a-f\d]{24}$", event_id) is None: - logging.error("\"%s\" is an invalid event_id... quiting", event_id) - raise ValueError(f'"{event_id}" is an invalid event_id... quiting') - - return event_ids_from_file - - -def insert_aux_data(aux_data_builders, event, dry_run=False): - ''' - Add aux_data records for only the specified event - ''' - for builder in aux_data_builders: - logging.debug("Building aux data record") - record = builder.build_aux_data_record(event) - if record: - try: - logging.debug("Submitting aux data record to Sealog Server") - logging.debug(json.dumps(record)) - if not dry_run: - create_event_aux_data(record) - - except Exception as exc: # pylint:disable=W0718 - logging.warning("Error submitting aux data record") - logging.debug(str(exc)) - else: - logging.debug("No aux data for data_source: %s", builder.data_source) - - -def insert_aux_data_from_list(aux_data_builders, event_ids_from_list, dry_run=False): - ''' - Add aux_data records for only the events in the specified list - ''' - for event_id in event_ids_from_list: - try: - logging.debug("Retrieving event record from Sealog Server") - event = get_event(event_id) - logging.debug("Event: %s", event) - - except Exception as exc: - logging.warning("Error submitting aux data record") - logging.debug(str(exc)) - raise exc - - insert_aux_data(aux_data_builders, event, dry_run) - - -def insert_aux_data_for_cruise(aux_data_builders, cruise_id, dry_run=False): - ''' - Add aux_data records for only the events in the specified cruise - ''' - cruise_uid = get_cruise_uid_by_id(cruise_id) - - # exit if no cruise found - if not cruise_uid: - logging.error("cruise not found") - return - - # retrieve events for cruise - cruise_events = get_events_by_cruise(cruise_uid) - - # exit if no cruise found - if not cruise_events: - logging.error("no events found for cruise") - return - - for event in cruise_events: - insert_aux_data(aux_data_builders, event, dry_run) - - -def insert_aux_data_for_lowering(aux_data_builders, lowering_id, dry_run=False): - ''' - Add aux_data records for only the events in the specified lowering - ''' - lowering_uid = get_lowering_uid_by_id(lowering_id) - - # exit if no lowering found - if not lowering_uid: - logging.error("lowering not found") - return - - # retrieve events for lowering - lowering_events = get_events_by_lowering(lowering_uid) - - # exit if no lowering found - if not lowering_events: - logging.error("no events found for lowering") - return - - for event in lowering_events: - insert_aux_data(aux_data_builders, event, dry_run) - - -async def insert_aux_data_from_ws(aux_data_builders, dry_run=False): - ''' - Use the aux_data_builder and the coriolix_sealog wrapper to submit aux_data - records built from CORIOLIX data to the sealog-server API - ''' - try: - async with websockets.connect(WS_SERVER_URL) as websocket: - - await websocket.send(json.dumps(HELLO)) - - while True: - - event = await websocket.recv() - event_obj = json.loads(event) - - if event_obj['type'] and event_obj['type'] == 'ping': - await websocket.send(json.dumps(PING)) - elif event_obj['type'] and event_obj['type'] == 'pub': - - if event_obj['message']['event_value'] in EXCLUDE_SET: - logging.debug("Skipping because event value is in the exclude set") - continue - - logging.debug("Event: %s", event_obj['message']) - - insert_aux_data(aux_data_builders, event_obj['message'], dry_run) - - except Exception as exc: - logging.error(str(exc)) - raise exc - # ------------------------------------------------------------------------------------- # The main loop of the utility # ------------------------------------------------------------------------------------- if __name__ == '__main__': - import argparse - import os + def builder_factory(aux_data_config): + '''Build an AuxDataRecordBuilder''' + return SealogCORIOLIXAuxDataRecordBuilder(aux_data_config) - parser = argparse.ArgumentParser(description='Aux Data Inserter Service - CORIOLIX') - parser.add_argument('-v', '--verbosity', dest='verbosity', - default=0, action='count', - help='Increase output verbosity') - parser.add_argument('-f', '--config_file', help='use the specifed configuration file') - parser.add_argument( - '-n', '--dry_run', action='store_true', - help='compile the aux_data records but do not submit to server API' + run_aux_data_inserter( + builder_factory=builder_factory, + inline_config=INLINE_CONFIG, + ws_server_url=WS_SERVER_URL, + headers=HEADERS, + client_wsid=CLIENT_WSID ) - parser.add_argument('-e', '--events', help='list of event_ids to apply the CORIOLIX data') - parser.add_argument('-c', '--cruise_id', help='cruise_id to fix aux_data for') - parser.add_argument('-l', '--lowering_id', help='lowering_id to fix aux_data for') - - parsed_args = parser.parse_args() - - ############################ - # Set up logging before we do any other argument parsing (so that we - # can log problems with argument parsing). - - LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s' - logging.basicConfig(format=LOGGING_FORMAT) - - LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} - parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS)) - logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity]) - - AUX_DATA_CONFIGS = None - - if parsed_args.config_file: - - config_file, data_source = ( - parsed_args.config_file.split(":") - if ":" in parsed_args.config_file - else [parsed_args.config_file, None] - ) - - try: - with open(config_file, 'r', encoding='utf-8') as config_fp: - AUX_DATA_CONFIGS = yaml.safe_load(config_fp) - - if data_source: - AUX_DATA_CONFIGS = [ - config for config in AUX_DATA_CONFIGS if config['data_source'] == data_source - ] - - except yaml.parser.ParserError: - logging.error("Invalid YAML syntax") - sys.exit(1) - else: - try: - AUX_DATA_CONFIGS = yaml.safe_load(INLINE_CONFIG) - except yaml.parser.ParserError: - logging.error("Invalid YAML syntax") - sys.exit(1) - - logging.debug(json.dumps(AUX_DATA_CONFIGS, indent=2)) - - # Create the Aux Data Record Builders - aux_data_builder_list = [ - SealogCORIOLIXAuxDataRecordBuilder(config) for config in AUX_DATA_CONFIGS - ] - - if parsed_args.events: - logging.debug("Processing list of event ids") - - event_ids = parse_event_ids(parsed_args.events) - logging.info("Event IDs:\n%s", json.dumps(event_ids, indent=2)) - - insert_aux_data_from_list(aux_data_builder_list, event_ids, parsed_args.dry_run) - - sys.exit(0) - - if parsed_args.cruise_id: - logging.debug("Processing events for an entire cruise") - - insert_aux_data_for_cruise( - aux_data_builder_list, - parsed_args.cruise_id, - parsed_args.dry_run - ) - - sys.exit(0) - - if parsed_args.lowering_id: - logging.debug("Processing events for an entire lowering") - - insert_aux_data_for_lowering( - aux_data_builder_list, - parsed_args.lowering_id, - parsed_args.dry_run - ) - - sys.exit(0) - - # Run the main loop - while True: - - # Wait 5 seconds for the server to complete startup - time.sleep(5) - - try: - logging.debug("Connecting to event websocket feed...") - asyncio.get_event_loop().run_until_complete( - insert_aux_data_from_ws(aux_data_builder_list, parsed_args.dry_run) - ) - except KeyboardInterrupt: - logging.error('Keyboard Interrupted') - try: - sys.exit(0) - except SystemExit: - os._exit(0) - except Exception as err: # pylint:disable=W0718 - logging.debug(str(err)) - logging.error("Lost connection to server, trying again in 5 seconds") diff --git a/misc/sealog_aux_data_inserter_influx.py.dist b/misc/sealog_aux_data_inserter_influx.py.dist index 56455e5..5f825d3 100644 --- a/misc/sealog_aux_data_inserter_influx.py.dist +++ b/misc/sealog_aux_data_inserter_influx.py.dist @@ -29,24 +29,12 @@ LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for det Copyright (C) OceanDataTools.org 2025 ''' -import re import sys -import json -import time -import logging -import asyncio -import websockets -import yaml from influxdb_client import InfluxDBClient from os.path import dirname, realpath sys.path.append(dirname(dirname(realpath(__file__)))) -from misc.python_sealog.events import get_event, get_events_by_cruise, get_events_by_lowering -from misc.python_sealog.event_aux_data import create_event_aux_data -from misc.python_sealog.lowerings import get_lowering_uid_by_id -from misc.python_sealog.cruises import get_cruise_uid_by_id - from misc.python_sealog.settings import WS_SERVER_URL, HEADERS from misc.influx_sealog.settings import ( INFLUXDB_URL, @@ -55,6 +43,7 @@ from misc.influx_sealog.settings import ( INFLUXDB_VERIFY_SSL ) from misc.influx_sealog.aux_data_record_builder import SealogInfluxAuxDataRecordBuilder +from misc.aux_data_inserter_runner import run_aux_data_inserter # ----------------------------------------------------------------------------- # @@ -100,225 +89,10 @@ EXCLUDE_SET = set() # needs to be unique for all currently active dataInserter scripts. CLIENT_WSID = 'auxData-dataInserter-influx' -HELLO = { - 'type': 'hello', - 'id': CLIENT_WSID, - 'auth': { - 'headers': HEADERS - }, - 'version': '2', - 'subs': ['/ws/status/newEvents'] -} - -PING = { - 'type': 'ping', - 'id': CLIENT_WSID -} - - -def parse_event_ids(event_id_file): - ''' - Builds list of event uid from csv-formatted file. - ''' - event_ids_from_file = [] - with open(event_id_file, 'r', encoding='utf-8') as event_id_fp: - for line in event_id_fp: - line = line.rstrip('\n') - logging.debug(line) - event_ids_from_file += line.split(',') - - event_ids_from_file = [event_id.strip() for event_id in event_ids_from_file] - - for event_id in event_ids_from_file: - if re.match(r"^[a-f\d]{24}$", event_id) is None: - logging.error("\"%s\" is an invalid event_id... quiting", event_id) - raise ValueError(f'"{event_id}" is an invalid event_id... quiting') - - return event_ids_from_file - - -def insert_aux_data(aux_data_builders, event, dry_run=False): - ''' - Add aux_data records for only the specified event - ''' - for builder in aux_data_builders: - logging.debug("Building aux data record") - record = builder.build_aux_data_record(event) - if record: - try: - logging.debug("Submitting aux data record to Sealog Server") - logging.debug(json.dumps(record)) - if not dry_run: - create_event_aux_data(record) - - except Exception as exc: # pylint:disable=W0718 - logging.warning("Error submitting aux data record") - logging.debug(str(exc)) - else: - logging.debug("No aux data for data_source: %s", builder.data_source) - - -def insert_aux_data_from_list(aux_data_builders, event_ids_from_list, dry_run=False): - ''' - Add aux_data records for only the events in the specified list - ''' - for event_id in event_ids_from_list: - try: - logging.debug("Retrieving event record from Sealog Server") - event = get_event(event_id) - logging.debug("Event: %s", event) - - except Exception as exc: - logging.warning("Error submitting aux data record") - logging.debug(str(exc)) - raise exc - - insert_aux_data(aux_data_builders, event, dry_run) - - -def insert_aux_data_for_cruise(aux_data_builders, cruise_id, dry_run=False): - ''' - Add aux_data records for only the events in the specified cruise - ''' - cruise_uid = get_cruise_uid_by_id(cruise_id) - - # exit if no cruise found - if not cruise_uid: - logging.error("cruise not found") - return - - # retrieve events for cruise - cruise_events = get_events_by_cruise(cruise_uid) - - # exit if no cruise found - if not cruise_events: - logging.error("no events found for cruise") - return - - for event in cruise_events: - insert_aux_data(aux_data_builders, event, dry_run) - - -def insert_aux_data_for_lowering(aux_data_builders, lowering_id, dry_run=False): - ''' - Add aux_data records for only the events in the specified lowering - ''' - lowering_uid = get_lowering_uid_by_id(lowering_id) - - # exit if no lowering found - if not lowering_uid: - logging.error("lowering not found") - return - - # retrieve events for lowering - lowering_events = get_events_by_lowering(lowering_uid) - - # exit if no lowering found - if not lowering_events: - logging.error("no events found for lowering") - return - - for event in lowering_events: - insert_aux_data(aux_data_builders, event, dry_run) - - -async def insert_aux_data_from_ws(aux_data_builders, dry_run=False): - ''' - Use the aux_data_builder and the influx_sealog wrapper to submit aux_data - records built from influxDB data to the sealog-server API - ''' - try: - async with websockets.connect(WS_SERVER_URL) as websocket: - - await websocket.send(json.dumps(HELLO)) - - while True: - - event = await websocket.recv() - event_obj = json.loads(event) - - if event_obj['type'] and event_obj['type'] == 'ping': - await websocket.send(json.dumps(PING)) - elif event_obj['type'] and event_obj['type'] == 'pub': - - if event_obj['message']['event_value'] in EXCLUDE_SET: - logging.debug("Skipping because event value is in the exclude set") - continue - - logging.debug("Event: %s", event_obj['message']) - - insert_aux_data(aux_data_builders, event_obj['message'], dry_run) - - except Exception as exc: - logging.error(str(exc)) - raise exc - # ------------------------------------------------------------------------------------- # The main loop of the utility # ------------------------------------------------------------------------------------- if __name__ == '__main__': - - import argparse - import os - - parser = argparse.ArgumentParser(description='Aux Data Inserter Service - InfluxDB') - parser.add_argument( - '-v', '--verbosity', dest='verbosity', default=0, - action='count', help='Increase output verbosity' - ) - parser.add_argument('-f', '--config_file', help='use the specifed configuration file') - parser.add_argument( - '-n', '--dry_run', action='store_true', - help='compile the aux_data records but do not submit to server API' - ) - parser.add_argument('-e', '--events', help='list of event_ids to apply the influx data') - parser.add_argument('-c', '--cruise_id', help='cruise_id to fix aux_data for') - parser.add_argument('-l', '--lowering_id', help='lowering_id to fix aux_data for') - - parsed_args = parser.parse_args() - - ############################ - # Set up logging before we do any other argument parsing (so that we - # can log problems with argument parsing). - - LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s' - logging.basicConfig(format=LOGGING_FORMAT) - - LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} - parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS)) - logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity]) - - AUX_DATA_CONFIGS = None - - if parsed_args.config_file: - - config_file, data_source = ( - parsed_args.config_file.split(":") - if ":" in parsed_args.config_file - else [parsed_args.config_file, None] - ) - - try: - with open(config_file, 'r', encoding='utf-8') as config_fp: - AUX_DATA_CONFIGS = yaml.safe_load(config_fp) - - if data_source: - AUX_DATA_CONFIGS = [ - config for config in AUX_DATA_CONFIGS if config['data_source'] == data_source - ] - - except yaml.parser.ParserError: - logging.error("Invalid YAML syntax") - sys.exit(1) - else: - try: - AUX_DATA_CONFIGS = yaml.safe_load(INLINE_CONFIG) - except yaml.parser.ParserError: - logging.error("Invalid YAML syntax") - sys.exit(1) - - logging.debug(json.dumps(AUX_DATA_CONFIGS, indent=2)) - # create an influxDB Client use_ssl = INFLUXDB_URL.find('https:') == 0 client = InfluxDBClient(url=INFLUXDB_URL, @@ -327,63 +101,15 @@ if __name__ == '__main__': ssl=use_ssl, verify_ssl=INFLUXDB_VERIFY_SSL) - # Create the Aux Data Record Builders - aux_data_builder_list = list( - map( - lambda config: SealogInfluxAuxDataRecordBuilder(client, config), - AUX_DATA_CONFIGS - ) - ) - - if parsed_args.events: - logging.debug("Processing list of event ids") - - event_ids = parse_event_ids(parsed_args.events) - logging.info("Event IDs:\n%s", json.dumps(event_ids, indent=2)) - - insert_aux_data_from_list(aux_data_builder_list, event_ids, parsed_args.dry_run) - - sys.exit(0) - - if parsed_args.cruise_id: - logging.debug("Processing events for an entire cruise") - - insert_aux_data_for_cruise( - aux_data_builder_list, - parsed_args.cruise_id, - parsed_args.dry_run - ) - - sys.exit(0) - - if parsed_args.lowering_id: - logging.debug("Processing events for an entire lowering") - - insert_aux_data_for_lowering( - aux_data_builder_list, - parsed_args.lowering_id, - parsed_args.dry_run - ) - - sys.exit(0) - - # Run the main loop - while True: - - # Wait 5 seconds for the server to complete startup - time.sleep(5) - - try: - logging.debug("Connecting to event websocket feed...") - asyncio.get_event_loop().run_until_complete( - insert_aux_data_from_ws(aux_data_builder_list, parsed_args.dry_run) - ) - except KeyboardInterrupt: - logging.error('Keyboard Interrupted') - try: - sys.exit(0) - except SystemExit: - os._exit(0) - except Exception as err: # pylint:disable=W0718 - logging.debug(str(err)) - logging.error("Lost connection to server, trying again in 5 seconds") + # builder factory that injects the client + def builder_factory(aux_data_config): + '''Build an AuxDataRecordBuilder''' + return SealogInfluxAuxDataRecordBuilder(client, aux_data_config) + + run_aux_data_inserter( + builder_factory=builder_factory, + inline_config=INLINE_CONFIG, + ws_server_url=WS_SERVER_URL, + headers=HEADERS, + client_wsid=CLIENT_WSID + ) \ No newline at end of file From 90e2dbe371631a35bae4aeefaa7366cd4a6aadca Mon Sep 17 00:00:00 2001 From: ljones Date: Tue, 2 Dec 2025 13:29:47 -0800 Subject: [PATCH 08/17] slight loop optimization insert_aux_data_from_ws now runs one connection session and raises exceptions cleanly on disconnect The outer loop in run_aux_data_inserter is now the sole responsibility for retry logic. inner loop = "listen for events", outer loop = "reconnect on failure" --- misc/aux_data_inserter_runner.py | 73 +++++++++++++++----------------- 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/misc/aux_data_inserter_runner.py b/misc/aux_data_inserter_runner.py index ff81787..c46774b 100644 --- a/misc/aux_data_inserter_runner.py +++ b/misc/aux_data_inserter_runner.py @@ -135,43 +135,38 @@ async def insert_aux_data_from_ws(aux_data_builders, ws_server_url, headers, cli Use the aux_data_builder and to submit aux_data records built from external data to the sealog-server API ''' - try: - async with websockets.connect(ws_server_url) as websocket: - - HELLO = { - 'type': 'hello', - 'id': client_wsid, - 'auth': {'headers': headers}, - 'version': '2', - 'subs': ['/ws/status/newEvents'] - } - await websocket.send(json.dumps(HELLO)) - - while True: - - event = await websocket.recv() - event_obj = json.loads(event) - - if event_obj['type'] and event_obj['type'] == 'ping': - PING = { - 'type': 'ping', - 'id': client_wsid - } - await websocket.send(json.dumps(PING)) - - elif event_obj['type'] and event_obj['type'] == 'pub': - - if event_obj['message']['event_value'] in EXCLUDE_SET: + async with websockets.connect(ws_server_url) as websocket: + + HELLO = { + 'type': 'hello', + 'id': client_wsid, + 'auth': {'headers': headers}, + 'version': '2', + 'subs': ['/ws/status/newEvents'] + } + await websocket.send(json.dumps(HELLO)) + + # Precompute PING JSON to avoid reconstructing on every ping + PING = json.dumps({'type': 'ping', 'id': client_wsid}) + + while True: + event = await websocket.recv() + event_obj = json.loads(event) + event_type = event_obj.get('type') + + if event_type: + if event_type == 'ping': + await websocket.send(PING) + + elif event_type == 'pub': + message = event_obj.get('message') + if message.get('event_value') in EXCLUDE_SET: logging.debug("Skipping because event value is in the exclude set") - continue - - logging.debug("Event: %s", event_obj['message']) - - insert_aux_data(aux_data_builders, event_obj['message'], dry_run) - - except Exception as exc: - logging.error(str(exc)) - raise exc + else: + logging.debug("Event: %s", message) + insert_aux_data(aux_data_builders, message, dry_run) + else: + logging.warning("Malformed event received") def run_aux_data_inserter( builder_factory: Callable[[Dict[str, Any]], AuxDataRecordBuilder], @@ -249,7 +244,7 @@ def run_aux_data_inserter( insert_aux_data_for_lowering(aux_data_builder_list, parsed_args.lowering_id, parsed_args.dry_run) sys.exit(0) - # Wait then start WS loop forever + # Wait then start WS loop forever. Sleep and retry on any disconnect/error while True: time.sleep(5) try: @@ -262,7 +257,7 @@ def run_aux_data_inserter( try: sys.exit(0) except SystemExit: - os._exit(0) - except Exception as err: # pylint:disable=W0718 + os._exit(0) # pylint: disable=protected-access + except Exception as err: # pylint: disable=W0718 logging.debug(str(err)) logging.error("Lost connection to server, trying again in 5 seconds") \ No newline at end of file From ba63651a6127ff0728b83c88675fb038324949f3 Mon Sep 17 00:00:00 2001 From: ljones Date: Wed, 3 Dec 2025 07:55:14 -0800 Subject: [PATCH 09/17] Don't fail if inserter config is missing query-related properties--not all inserters need a query --- misc/base_aux_data_record_builder.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/misc/base_aux_data_record_builder.py b/misc/base_aux_data_record_builder.py index 37de82c..26ebe6d 100644 --- a/misc/base_aux_data_record_builder.py +++ b/misc/base_aux_data_record_builder.py @@ -36,10 +36,13 @@ def __init__(self, aux_data_config): - aux_record_lookup: Mapping of fields to output configuration - data_source: Name of the data source ''' - self._query_measurements = aux_data_config['query_measurements'] - self._query_fields = list(aux_data_config['aux_record_lookup'].keys()) - self._aux_record_lookup = aux_data_config['aux_record_lookup'] - self._data_source = aux_data_config['data_source'] + self._data_source = aux_data_config['data_source'] # don't use get--should throw an error if not specified + self._query_measurements = aux_data_config.get('query_measurements') + self._aux_record_lookup = aux_data_config.get('aux_record_lookup') + if self._aux_record_lookup is None: + self._query_fields = [] + else: + self._query_fields = list(self._aux_record_lookup.keys()) self.logger = logging.getLogger(__name__) @staticmethod From 04bec35d1be244ae6cc698d4c5db8f5f08905983 Mon Sep 17 00:00:00 2001 From: ljones Date: Tue, 17 Feb 2026 16:16:46 -0800 Subject: [PATCH 10/17] Add functionality for opening/closing connections with AuxDataRecordBuilders --- misc/aux_data_inserter_runner.py | 12 +++++++++++- misc/base_aux_data_record_builder.py | 15 +++++++++++++++ misc/coriolix_sealog/aux_data_record_builder.py | 14 ++++++++++++++ misc/influx_sealog/aux_data_record_builder.py | 16 +++++++++++++++- 4 files changed, 55 insertions(+), 2 deletions(-) diff --git a/misc/aux_data_inserter_runner.py b/misc/aux_data_inserter_runner.py index c46774b..573e545 100644 --- a/misc/aux_data_inserter_runner.py +++ b/misc/aux_data_inserter_runner.py @@ -248,6 +248,8 @@ def run_aux_data_inserter( while True: time.sleep(5) try: + for aux_data_builder in aux_data_builder_list: + aux_data_builder.open_connections() logging.debug("Connecting to event websocket feed...") asyncio.get_event_loop().run_until_complete( insert_aux_data_from_ws(aux_data_builder_list, ws_server_url, headers, client_wsid, parsed_args.dry_run) @@ -257,7 +259,15 @@ def run_aux_data_inserter( try: sys.exit(0) except SystemExit: + try: + for aux_data_builder in aux_data_builder_list: + aux_data_builder.close_connections() + except Exception as err: # pylint: disable=W0718 + logging.debug(str(err)) os._exit(0) # pylint: disable=protected-access except Exception as err: # pylint: disable=W0718 logging.debug(str(err)) - logging.error("Lost connection to server, trying again in 5 seconds") \ No newline at end of file + logging.error("Lost connection to server, trying again in 5 seconds") + finally: + for aux_data_builder in aux_data_builder_list: + aux_data_builder.close_connections() \ No newline at end of file diff --git a/misc/base_aux_data_record_builder.py b/misc/base_aux_data_record_builder.py index 26ebe6d..8d5b6b9 100644 --- a/misc/base_aux_data_record_builder.py +++ b/misc/base_aux_data_record_builder.py @@ -61,6 +61,21 @@ def _build_query_range(ts): # I'm not sure why we need this to be a static method, but I'm including it in the base class since it's in both implementations pass + @abstractmethod + def open_connections(self): + ''' + Open any necessary connections to external data sources. + Must be implemented by subclasses. + ''' + pass + + @abstractmethod + def close_connections(self): + ''' + Close any open connections to external data sources. + Must be implemented by subclasses. + ''' + pass @abstractmethod def build_aux_data_record(self, event): diff --git a/misc/coriolix_sealog/aux_data_record_builder.py b/misc/coriolix_sealog/aux_data_record_builder.py index aa61153..1a6af35 100644 --- a/misc/coriolix_sealog/aux_data_record_builder.py +++ b/misc/coriolix_sealog/aux_data_record_builder.py @@ -74,6 +74,20 @@ def _build_query_urls(self, ts): return query_urls + def open_connections(self): + ''' + Open any necessary connections to external data sources. + For CORIOLIX, no persistent connection is needed. + ''' + pass + + def close_connections(self): + ''' + Close any open connections to external data sources. + For CORIOLIX, no persistent connection is needed. + ''' + pass + def build_aux_data_record(self, event): ''' Build the aux_data record for the given event. diff --git a/misc/influx_sealog/aux_data_record_builder.py b/misc/influx_sealog/aux_data_record_builder.py index 8083a9e..c1fdeb9 100644 --- a/misc/influx_sealog/aux_data_record_builder.py +++ b/misc/influx_sealog/aux_data_record_builder.py @@ -3,7 +3,7 @@ FILE: aux_data_record_builder.py DESCRIPTION: This script builds a sealog aux_data record with data pulled from an - influx database. + influx v2 database. BUGS: NOTES: @@ -89,6 +89,20 @@ def _build_query(self, ts): logging.debug("Query: %s", query) return query + + def open_connections(self): + ''' + Open any necessary connections to external data sources. + For Influx, no persistent connection is needed. + ''' + pass + + def close_connections(self): + ''' + Close any open connections to external data sources. + For Influx, no persistent connection is needed. + ''' + pass def build_aux_data_record(self, event): ''' From 3459ef3fac4b65ee73d6d670689abc2906b03f5a Mon Sep 17 00:00:00 2001 From: ljones Date: Tue, 17 Feb 2026 16:19:27 -0800 Subject: [PATCH 11/17] Make the set of event types to exclude individual to inserters --- misc/aux_data_inserter_runner.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/misc/aux_data_inserter_runner.py b/misc/aux_data_inserter_runner.py index 573e545..7e7ba5f 100644 --- a/misc/aux_data_inserter_runner.py +++ b/misc/aux_data_inserter_runner.py @@ -23,8 +23,6 @@ from misc.python_sealog.lowerings import get_lowering_uid_by_id from misc.python_sealog.cruises import get_cruise_uid_by_id -EXCLUDE_SET = set() #TODO: not sure what you would want to put here. Should this on the individual inserters instead? - def parse_event_ids(event_id_file): ''' Builds list of event uid from csv-formatted file. @@ -130,7 +128,7 @@ def insert_aux_data_for_lowering(aux_data_builders, lowering_id, dry_run=False): for event in lowering_events: insert_aux_data(aux_data_builders, event, dry_run) -async def insert_aux_data_from_ws(aux_data_builders, ws_server_url, headers, client_wsid, dry_run=False): +async def insert_aux_data_from_ws(aux_data_builders, ws_server_url, headers, client_wsid, exclude_set, dry_run): ''' Use the aux_data_builder and to submit aux_data records built from external data to the sealog-server API @@ -160,7 +158,7 @@ async def insert_aux_data_from_ws(aux_data_builders, ws_server_url, headers, cli elif event_type == 'pub': message = event_obj.get('message') - if message.get('event_value') in EXCLUDE_SET: + if message.get('event_value') in exclude_set: logging.debug("Skipping because event value is in the exclude set") else: logging.debug("Event: %s", message) @@ -174,6 +172,7 @@ def run_aux_data_inserter( ws_server_url: str, headers: Dict[str, Any], client_wsid: str, + exclude_set: set = () ): """ Shared CLI and main loop. @@ -196,7 +195,7 @@ def run_aux_data_inserter( parsed_args = parser.parse_args() # Logging setup - LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s' + LOGGING_FORMAT = '%(asctime)-15s %(levelname)s %(lineno)s - %(message)s' logging.basicConfig(format=LOGGING_FORMAT) LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS)) @@ -252,7 +251,7 @@ def run_aux_data_inserter( aux_data_builder.open_connections() logging.debug("Connecting to event websocket feed...") asyncio.get_event_loop().run_until_complete( - insert_aux_data_from_ws(aux_data_builder_list, ws_server_url, headers, client_wsid, parsed_args.dry_run) + insert_aux_data_from_ws(aux_data_builder_list, ws_server_url, headers, client_wsid, exclude_set, parsed_args.dry_run) ) except KeyboardInterrupt: logging.error('Keyboard Interrupted') From bd01194cda03e52b6b6d249541e397c67553905f Mon Sep 17 00:00:00 2001 From: ljones Date: Tue, 17 Feb 2026 16:43:04 -0800 Subject: [PATCH 12/17] Rename "aux data inserter"s to "aux data manager"s --- ...data_inserter_runner.py => aux_data_manager_runner.py} | 7 ++++--- ...x.py.dist => sealog_aux_data_manager_coriolix.py.dist} | 8 ++++---- ...lux.py.dist => sealog_aux_data_manager_influx.py.dist} | 8 ++++---- 3 files changed, 12 insertions(+), 11 deletions(-) rename misc/{aux_data_inserter_runner.py => aux_data_manager_runner.py} (97%) rename misc/{sealog_aux_data_inserter_coriolix.py.dist => sealog_aux_data_manager_coriolix.py.dist} (93%) rename misc/{sealog_aux_data_inserter_influx.py.dist => sealog_aux_data_manager_influx.py.dist} (94%) diff --git a/misc/aux_data_inserter_runner.py b/misc/aux_data_manager_runner.py similarity index 97% rename from misc/aux_data_inserter_runner.py rename to misc/aux_data_manager_runner.py index 7e7ba5f..9ae1acf 100644 --- a/misc/aux_data_inserter_runner.py +++ b/misc/aux_data_manager_runner.py @@ -128,7 +128,7 @@ def insert_aux_data_for_lowering(aux_data_builders, lowering_id, dry_run=False): for event in lowering_events: insert_aux_data(aux_data_builders, event, dry_run) -async def insert_aux_data_from_ws(aux_data_builders, ws_server_url, headers, client_wsid, exclude_set, dry_run): +async def manage_aux_data_from_ws(aux_data_builders, ws_server_url, headers, client_wsid, exclude_set, dry_run): ''' Use the aux_data_builder and to submit aux_data records built from external data to the sealog-server API @@ -162,11 +162,12 @@ async def insert_aux_data_from_ws(aux_data_builders, ws_server_url, headers, cli logging.debug("Skipping because event value is in the exclude set") else: logging.debug("Event: %s", message) + # TODO: Check which websocket this came from and then insert, update, or delete accordingly insert_aux_data(aux_data_builders, message, dry_run) else: logging.warning("Malformed event received") -def run_aux_data_inserter( +def run_aux_data_manager( builder_factory: Callable[[Dict[str, Any]], AuxDataRecordBuilder], inline_config: str, ws_server_url: str, @@ -251,7 +252,7 @@ def run_aux_data_inserter( aux_data_builder.open_connections() logging.debug("Connecting to event websocket feed...") asyncio.get_event_loop().run_until_complete( - insert_aux_data_from_ws(aux_data_builder_list, ws_server_url, headers, client_wsid, exclude_set, parsed_args.dry_run) + manage_aux_data_from_ws(aux_data_builder_list, ws_server_url, headers, client_wsid, exclude_set, parsed_args.dry_run) ) except KeyboardInterrupt: logging.error('Keyboard Interrupted') diff --git a/misc/sealog_aux_data_inserter_coriolix.py.dist b/misc/sealog_aux_data_manager_coriolix.py.dist similarity index 93% rename from misc/sealog_aux_data_inserter_coriolix.py.dist rename to misc/sealog_aux_data_manager_coriolix.py.dist index 3cef8fa..960e1d5 100644 --- a/misc/sealog_aux_data_inserter_coriolix.py.dist +++ b/misc/sealog_aux_data_manager_coriolix.py.dist @@ -35,7 +35,7 @@ sys.path.append(dirname(dirname(realpath(__file__)))) from misc.python_sealog.settings import WS_SERVER_URL, HEADERS from misc.coriolix_sealog.aux_data_record_builder import SealogCORIOLIXAuxDataRecordBuilder -from misc.aux_data_inserter_runner import run_aux_data_inserter +from misc.aux_data_manager_runner import run_aux_data_manager # ----------------------------------------------------------------------------- # @@ -93,8 +93,8 @@ INLINE_CONFIG = ''' - multiply: 0.539957 ''' -# needs to be unique for all currently active dataInserter scripts. -CLIENT_WSID = 'auxData-dataInserter-coriolix' +# needs to be unique for all currently active dataManager scripts. +CLIENT_WSID = 'auxData-dataManager-coriolix' # ------------------------------------------------------------------------------------- # The main loop of the utility @@ -105,7 +105,7 @@ if __name__ == '__main__': '''Build an AuxDataRecordBuilder''' return SealogCORIOLIXAuxDataRecordBuilder(aux_data_config) - run_aux_data_inserter( + run_aux_data_manager( builder_factory=builder_factory, inline_config=INLINE_CONFIG, ws_server_url=WS_SERVER_URL, diff --git a/misc/sealog_aux_data_inserter_influx.py.dist b/misc/sealog_aux_data_manager_influx.py.dist similarity index 94% rename from misc/sealog_aux_data_inserter_influx.py.dist rename to misc/sealog_aux_data_manager_influx.py.dist index 5f825d3..a8e4dca 100644 --- a/misc/sealog_aux_data_inserter_influx.py.dist +++ b/misc/sealog_aux_data_manager_influx.py.dist @@ -43,7 +43,7 @@ from misc.influx_sealog.settings import ( INFLUXDB_VERIFY_SSL ) from misc.influx_sealog.aux_data_record_builder import SealogInfluxAuxDataRecordBuilder -from misc.aux_data_inserter_runner import run_aux_data_inserter +from misc.aux_data_manager_runner import run_aux_data_manager # ----------------------------------------------------------------------------- # @@ -86,8 +86,8 @@ INLINE_CONFIG = ''' # set of events to ignore EXCLUDE_SET = set() -# needs to be unique for all currently active dataInserter scripts. -CLIENT_WSID = 'auxData-dataInserter-influx' +# needs to be unique for all currently active dataManager scripts. +CLIENT_WSID = 'auxData-dataManager-influx' # ------------------------------------------------------------------------------------- # The main loop of the utility @@ -106,7 +106,7 @@ if __name__ == '__main__': '''Build an AuxDataRecordBuilder''' return SealogInfluxAuxDataRecordBuilder(client, aux_data_config) - run_aux_data_inserter( + run_aux_data_manager( builder_factory=builder_factory, inline_config=INLINE_CONFIG, ws_server_url=WS_SERVER_URL, From 759fed0eb0e7c90dfe262d8a2abdb8e8f1f0fc7a Mon Sep 17 00:00:00 2001 From: ljones Date: Fri, 20 Feb 2026 15:40:41 -0800 Subject: [PATCH 13/17] Move aux_data deletion/clean up to the aux_data_managers Instead of directly interacting with the eventAuxDataTable, the events API endpoints now publish deleteEvents messages, which the aux_data_managers are listening for. When an aux_data_manager receives a deleteEvents messages, it sends the contents to its aux_data_file_cleaner(s) which do any external clean up required and then send a delete request to the API event_aux_data endpoint # Conflicts: # routes/api/v1/events.js --- .../base_aux_data_file_cleaner.py | 94 +++++++++ .../do_nothing_aux_data_file_cleaner.py | 61 ++++++ .../stillcapffmpeg_aux_data_file_cleaner.py | 82 ++++++++ misc/aux_data_manager_runner.py | 178 +++++++++++++----- misc/delete_aux_data_records.py | 1 + misc/python_sealog/event_aux_data.py | 15 ++ misc/sealog_aux_data_manager_coriolix.py.dist | 6 + misc/sealog_aux_data_manager_influx.py.dist | 6 + routes/api/v1/events.js | 142 ++++++-------- 9 files changed, 453 insertions(+), 132 deletions(-) create mode 100644 misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py create mode 100644 misc/aux_data_file_cleaners/do_nothing_aux_data_file_cleaner.py create mode 100644 misc/aux_data_file_cleaners/stillcapffmpeg_aux_data_file_cleaner.py diff --git a/misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py b/misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py new file mode 100644 index 0000000..5c122de --- /dev/null +++ b/misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +''' +FILE: base_aux_data_file_cleaner.py + +DESCRIPTION: Base class for cleaning up any changes made by aux_data_inserters outside of the aux data mongo collection. + +BUGS: +NOTES: +AUTHOR: Lindsey Jones +COMPANY: OET +VERSION: 1.0 +CREATED: 2026-02-20 +REVISION: + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' +import json +import logging +from abc import ABC, abstractmethod + +class AuxDataFileCleaner(ABC): + ''' + Abstract base class for building sealog aux_data records from various data sources. + Handles common functionality for querying external data sources and building + aux_data records with transformations. + ''' + + def __init__(self, aux_data_config): + ''' + Initialize the base builder with configuration. + ''' + self._data_source = aux_data_config['data_source'] + self.logger = logging.getLogger(__name__) + + def _get_aux_data_for_source(self, event): + ''' + Get the aux data record for the given event based on the data source. + + Args: + event (dict): Event dictionary containing 'id' and 'ts' keys + + Returns: + str or None: ID of cleaned aux data record, if there is one + ''' + all_aux_data = event.get('aux_data', {}) + if not all_aux_data: + self.logger.debug(f"No aux data found for event {event['id']}") + return None + + # Find the first aux_data record that matches the data source + # There should only ever be one record per data source + aux_data_for_source = next( + (aux_data for aux_data in all_aux_data + if aux_data["data_source"] == self._data_source), + None # default if not found + ) + + if not aux_data_for_source: + self.logger.debug(f"No {self._data_source} aux data found for event {event['id']}") + + return aux_data_for_source + + + @abstractmethod + def open_connections(self): + ''' + Open any necessary connections to external data sources. + Must be implemented by subclasses. + ''' + pass + + @abstractmethod + def close_connections(self): + ''' + Close any open connections to external data sources. + Must be implemented by subclasses. + ''' + pass + + @abstractmethod + def clean_aux_data_record(self, event, dry_run): + ''' + Do any clean up required for the given event + Must be implemented by subclasses. + + Args: + event (dict): Event dictionary containing 'id' and 'ts' keys + dry_run (bool): If True, do not actually delete any aux data records + + Returns: + dict or None: Aux data record or None if no data available + ''' + pass \ No newline at end of file diff --git a/misc/aux_data_file_cleaners/do_nothing_aux_data_file_cleaner.py b/misc/aux_data_file_cleaners/do_nothing_aux_data_file_cleaner.py new file mode 100644 index 0000000..ac53934 --- /dev/null +++ b/misc/aux_data_file_cleaners/do_nothing_aux_data_file_cleaner.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +''' +FILE: do_nothing_aux_data_file_cleaner.py + +DESCRIPTION: Aux data file cleaner for aux data inserters that don't make any external changes. + +BUGS: +NOTES: +AUTHOR: Lindsey Jones +COMPANY: OET +VERSION: 1.0 +CREATED: 2026-02-20 +REVISION: + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' +import sys + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(dirname(realpath(__file__))))) + +from misc.aux_data_file_cleaners.base_aux_data_file_cleaner import AuxDataFileCleaner + + +class DoNothingAuxDataFileCleaner(AuxDataFileCleaner): + ''' + Aux data file cleaner for aux data inserters that don't make any external changes. + ''' + + def open_connections(self): + ''' + No connections to open. + ''' + + def close_connections(self): + ''' + No connections to close. + ''' + + def clean_aux_data_record(self, event, dry_run): # pylint: disable=W0613 + ''' + Do any clean up required for the given event + + Args: + event (dict): Event dictionary containing 'id' and 'ts' keys + dry_run (bool): If True, do not actually delete any aux data records + + Returns: + str or None: ID of cleaned aux data record, if there is one + ''' + aux_data = self._get_aux_data_for_source(event) + + if aux_data: + self.logger.debug( + f"No additional clean up required for event {event['id']} {self._data_source} " + "aux data records" + ) + return aux_data["_id"] + + return None diff --git a/misc/aux_data_file_cleaners/stillcapffmpeg_aux_data_file_cleaner.py b/misc/aux_data_file_cleaners/stillcapffmpeg_aux_data_file_cleaner.py new file mode 100644 index 0000000..90efa46 --- /dev/null +++ b/misc/aux_data_file_cleaners/stillcapffmpeg_aux_data_file_cleaner.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +''' +FILE: stillcapffmpeg_aux_data_file_cleaner.py + +DESCRIPTION: Cleans up the files created by the stillcap_ffmpeg aux data creation process. + +BUGS: +NOTES: +AUTHOR: Lindsey Jones +COMPANY: OET +VERSION: 1.0 +CREATED: 2026-02-20 +REVISION: + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' +import sys +import os + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(dirname(realpath(__file__))))) + +from misc.aux_data_file_cleaners.base_aux_data_file_cleaner import AuxDataFileCleaner + + +class StillcapFFMPEGAuxDataFileCleaner(AuxDataFileCleaner): + ''' + Cleans up the files created by the stillcap_ffmpeg aux data creation process + ''' + + def open_connections(self): + ''' + No connections to open. + ''' + + def close_connections(self): + ''' + No connections to close. + ''' + + def clean_aux_data_record(self, event, dry_run): + ''' + Do any clean up required for the given event + + Args: + event (dict): Event dictionary containing 'id' and 'ts' keys + dry_run (bool): If True, do not actually delete any aux data records + + Returns: + str or None: ID of cleaned aux data record, if there is one + ''' + aux_data = self._get_aux_data_for_source(event) + + if aux_data: + file_paths = [ + data["data_value"] + for data in aux_data["data_array"] + if data["data_name"] == "filename" + ] + for file_path in file_paths: + try: + self.logger.debug(f"Deleting file {file_path} for event {event['id']}") + if not dry_run: + os.remove(file_path) + except Exception as e: # pylint: disable=W0718 + self.logger.error( + f"Error deleting file {file_path} for event {event['id']}: {e}" + ) + + try: + sym_path = file_path.replace('png', 'symlink') + self.logger.debug(f"Deleting symlink {sym_path} for event {event['id']}") + if not dry_run: + os.remove(sym_path) + except Exception as e: # pylint: disable=W0718 + self.logger.error( + f"Error deleting symlink {sym_path} for event {event['id']}: {e}" + ) + return aux_data["_id"] + + return None diff --git a/misc/aux_data_manager_runner.py b/misc/aux_data_manager_runner.py index 9ae1acf..c8c6f90 100644 --- a/misc/aux_data_manager_runner.py +++ b/misc/aux_data_manager_runner.py @@ -12,17 +12,25 @@ import asyncio import websockets import yaml +import os +import argparse from typing import Callable, Dict, Any from os.path import dirname, realpath sys.path.append(dirname(dirname(realpath(__file__)))) from misc.base_aux_data_record_builder import AuxDataRecordBuilder +from misc.aux_data_file_cleaners.base_aux_data_file_cleaner import AuxDataFileCleaner from misc.python_sealog.events import get_event, get_events_by_cruise, get_events_by_lowering -from misc.python_sealog.event_aux_data import create_event_aux_data +from misc.python_sealog.event_aux_data import create_event_aux_data, delete_event_aux_data from misc.python_sealog.lowerings import get_lowering_uid_by_id from misc.python_sealog.cruises import get_cruise_uid_by_id + +LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} +LOGGING_FORMAT = '%(asctime)-15s %(levelname)s %(lineno)s - %(message)s' + + def parse_event_ids(event_id_file): ''' Builds list of event uid from csv-formatted file. @@ -44,6 +52,29 @@ def parse_event_ids(event_id_file): return event_ids_from_file +def delete_aux_data(aux_data_cleaners, event, dry_run=False): + ''' + Delete aux_data records for the specified event and any impacts they may have had + ''' + for cleaner in aux_data_cleaners: + logging.debug("Cleaning aux data record") + # NOTE: I think this code would be more single-responsibility if we got the aux data + # and then passed it into the cleaner, but I wanted to match the pattern of the builders + aux_data_id = cleaner.clean_aux_data_record(event, dry_run) + if aux_data_id: + try: + logging.debug("Deleting aux data files from Sealog Server") + logging.debug(json.dumps(aux_data_id)) + if not dry_run: + delete_event_aux_data(aux_data_id) + + except Exception as exc: # pylint:disable=W0718 + logging.warning("Error deleting aux data record") + logging.debug(str(exc)) + else: + logging.debug("No aux data for data_source: %s", cleaner.data_source) + + def insert_aux_data(aux_data_builders, event, dry_run=False): ''' Add aux_data records for only the specified event @@ -128,24 +159,35 @@ def insert_aux_data_for_lowering(aux_data_builders, lowering_id, dry_run=False): for event in lowering_events: insert_aux_data(aux_data_builders, event, dry_run) -async def manage_aux_data_from_ws(aux_data_builders, ws_server_url, headers, client_wsid, exclude_set, dry_run): + +async def manage_aux_data_from_ws(aux_data_builders, + aux_data_cleaners, + ws_server_url, + headers, + client_wsid, + exclude_set, + dry_run): # pylint: disable=R0914 ''' Use the aux_data_builder and to submit aux_data records built from external data to the sealog-server API ''' async with websockets.connect(ws_server_url) as websocket: - HELLO = { + subscription_message = { 'type': 'hello', 'id': client_wsid, 'auth': {'headers': headers}, 'version': '2', - 'subs': ['/ws/status/newEvents'] + 'subs': ['/ws/status/newEvents', + '/ws/status/deleteEvents'] } - await websocket.send(json.dumps(HELLO)) + # updateEvents is only published when the timestamp is unchanged + # no time change-->no need to get aux data for a different time-->no need to subscribe + # if you wanted to, say, add a new type of aux data, you wouldn't be using the ws pathway + await websocket.send(json.dumps(subscription_message)) # Precompute PING JSON to avoid reconstructing on every ping - PING = json.dumps({'type': 'ping', 'id': client_wsid}) + ping_message = json.dumps({'type': 'ping', 'id': client_wsid}) while True: event = await websocket.recv() @@ -154,7 +196,7 @@ async def manage_aux_data_from_ws(aux_data_builders, ws_server_url, headers, cli if event_type: if event_type == 'ping': - await websocket.send(PING) + await websocket.send(ping_message) elif event_type == 'pub': message = event_obj.get('message') @@ -162,58 +204,56 @@ async def manage_aux_data_from_ws(aux_data_builders, ws_server_url, headers, cli logging.debug("Skipping because event value is in the exclude set") else: logging.debug("Event: %s", message) - # TODO: Check which websocket this came from and then insert, update, or delete accordingly - insert_aux_data(aux_data_builders, message, dry_run) + pub_type = event_obj.get('topic', '').split('/')[-1] + match pub_type: + case 'newEvents': + insert_aux_data(aux_data_builders, message, dry_run) + case 'deleteEvents': + delete_aux_data(aux_data_cleaners, message, dry_run) + case _: + logging.warning("Unhandled pub type: %s", pub_type) else: - logging.warning("Malformed event received") - -def run_aux_data_manager( - builder_factory: Callable[[Dict[str, Any]], AuxDataRecordBuilder], - inline_config: str, - ws_server_url: str, - headers: Dict[str, Any], - client_wsid: str, - exclude_set: set = () -): - """ - Shared CLI and main loop. + logging.warning("Malformed event received: %s", event_obj) - builder_factory: callable(config_dict) -> builder instance - inline_config: YAML string used if -f not provided - ws_server_url, headers, client_wsid: for websocket connection. - """ - import argparse - import os - parser = argparse.ArgumentParser(description='Aux Data Inserter Service (shared)') - parser.add_argument('-v', '--verbosity', dest='verbosity', default=0, action='count', help='Increase output verbosity') +def parse_aux_data_args(): + ''' + Parse command line arguments for aux data manager scripts. + ''' + parser = argparse.ArgumentParser(description='Aux Data Manager Service (shared)') + parser.add_argument('-v', '--verbosity', dest='verbosity', default=0, action='count', + help='Increase output verbosity') parser.add_argument('-f', '--config_file', help='use the specifed configuration file') - parser.add_argument('-n', '--dry_run', action='store_true', help='compile the aux_data records but do not submit to server API') + parser.add_argument('-n', '--dry_run', action='store_true', + help='compile the aux_data records but do not submit to server API') parser.add_argument('-e', '--events', help='list of event_ids to apply aux data') parser.add_argument('-c', '--cruise_id', help='cruise_id to fix aux_data for') parser.add_argument('-l', '--lowering_id', help='lowering_id to fix aux_data for') parsed_args = parser.parse_args() - - # Logging setup - LOGGING_FORMAT = '%(asctime)-15s %(levelname)s %(lineno)s - %(message)s' - logging.basicConfig(format=LOGGING_FORMAT) - LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS)) - logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity]) + return parsed_args - if parsed_args.config_file: + +def get_aux_data_configs(inline_config, config_file): + ''' + Get aux data configuration from file or inline config. + ''' + if config_file: config_file, data_source = ( - parsed_args.config_file.split(":") - if ":" in parsed_args.config_file - else [parsed_args.config_file, None] + config_file.split(":") + if ":" in config_file + else [config_file, None] ) try: with open(config_file, 'r', encoding='utf-8') as config_fp: aux_data_configs = yaml.safe_load(config_fp) if data_source: - aux_data_configs = [c for c in aux_data_configs if c['data_source'] == data_source] + aux_data_configs = [ + c + for c in aux_data_configs + if c['data_source'] == data_source] except yaml.parser.ParserError: logging.error("Invalid YAML syntax") sys.exit(1) @@ -225,9 +265,36 @@ def run_aux_data_manager( sys.exit(1) logging.debug(json.dumps(aux_data_configs, indent=2)) + return aux_data_configs + + +def run_aux_data_manager( + builder_factory: Callable[[Dict[str, Any]], AuxDataRecordBuilder], + cleaner_factory: Callable[[Dict[str, Any]], AuxDataFileCleaner], + inline_config: str, + ws_server_url: str, + headers: Dict[str, Any], + client_wsid: str, + exclude_set: set = () +): + """ + Shared CLI and main loop. + + builder_factory: callable(config_dict) -> builder instance + inline_config: YAML string used if -f not provided + ws_server_url, headers, client_wsid: for websocket connection. + """ + parsed_args = parse_aux_data_args() + + # Logging setup + logging.basicConfig(format=LOGGING_FORMAT) + logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity]) + + aux_data_configs = get_aux_data_configs(inline_config, parsed_args.config_file) - # Build builders + # Build builders and cleaners aux_data_builder_list = [builder_factory(cfg) for cfg in aux_data_configs] + aux_data_cleaner_list = [cleaner_factory(cfg) for cfg in aux_data_configs] # Dispatch actions requested on CLI if parsed_args.events: @@ -237,11 +304,17 @@ def run_aux_data_manager( sys.exit(0) if parsed_args.cruise_id: - insert_aux_data_for_cruise(aux_data_builder_list, parsed_args.cruise_id, parsed_args.dry_run) + insert_aux_data_for_cruise( + aux_data_builder_list, + parsed_args.cruise_id, + parsed_args.dry_run) sys.exit(0) if parsed_args.lowering_id: - insert_aux_data_for_lowering(aux_data_builder_list, parsed_args.lowering_id, parsed_args.dry_run) + insert_aux_data_for_lowering( + aux_data_builder_list, + parsed_args.lowering_id, + parsed_args.dry_run) sys.exit(0) # Wait then start WS loop forever. Sleep and retry on any disconnect/error @@ -250,9 +323,18 @@ def run_aux_data_manager( try: for aux_data_builder in aux_data_builder_list: aux_data_builder.open_connections() + for aux_data_cleaner in aux_data_cleaner_list: + aux_data_cleaner.open_connections() logging.debug("Connecting to event websocket feed...") asyncio.get_event_loop().run_until_complete( - manage_aux_data_from_ws(aux_data_builder_list, ws_server_url, headers, client_wsid, exclude_set, parsed_args.dry_run) + manage_aux_data_from_ws( + aux_data_builder_list, + aux_data_cleaner_list, + ws_server_url, + headers, + client_wsid, + exclude_set, + parsed_args.dry_run) ) except KeyboardInterrupt: logging.error('Keyboard Interrupted') @@ -262,6 +344,8 @@ def run_aux_data_manager( try: for aux_data_builder in aux_data_builder_list: aux_data_builder.close_connections() + for aux_data_cleaner in aux_data_cleaner_list: + aux_data_cleaner.close_connections() except Exception as err: # pylint: disable=W0718 logging.debug(str(err)) os._exit(0) # pylint: disable=protected-access @@ -270,4 +354,6 @@ def run_aux_data_manager( logging.error("Lost connection to server, trying again in 5 seconds") finally: for aux_data_builder in aux_data_builder_list: - aux_data_builder.close_connections() \ No newline at end of file + aux_data_builder.close_connections() + for aux_data_cleaner in aux_data_cleaner_list: + aux_data_cleaner.close_connections() diff --git a/misc/delete_aux_data_records.py b/misc/delete_aux_data_records.py index d856ec6..41528df 100644 --- a/misc/delete_aux_data_records.py +++ b/misc/delete_aux_data_records.py @@ -28,6 +28,7 @@ def main(uid_file, dry_run): ''' Main function of script, read the file containing aux_data record ids and delete them. + This does not use the aux data managers, so it does not do any additional clean up ''' logging.info("Starting main function.") logging.info(dry_run) diff --git a/misc/python_sealog/event_aux_data.py b/misc/python_sealog/event_aux_data.py index 1b62fa6..be3782e 100644 --- a/misc/python_sealog/event_aux_data.py +++ b/misc/python_sealog/event_aux_data.py @@ -104,6 +104,21 @@ def get_event_aux_data_by_lowering(lowering_uid, datasource=None, limit=0, logging.error(str(exc)) raise exc +def get_event_aux_data_by_event(aux_data_uid, + api_server_url=API_SERVER_URL, + headers=HEADERS): + ''' + Get the aux_data records for an event + ''' + try: + url = f'{api_server_url}{EVENT_AUX_DATA_API_PATH}' + req = requests.post(url, headers=headers, data=json.dumps(payload), timeout=(2, None)) + logging.debug(req.text) + + except requests.exceptions.RequestException as exc: + logging.error(str(exc)) + raise exc + def create_event_aux_data(payload, api_server_url=API_SERVER_URL, headers=HEADERS): ''' diff --git a/misc/sealog_aux_data_manager_coriolix.py.dist b/misc/sealog_aux_data_manager_coriolix.py.dist index 960e1d5..557dabc 100644 --- a/misc/sealog_aux_data_manager_coriolix.py.dist +++ b/misc/sealog_aux_data_manager_coriolix.py.dist @@ -35,6 +35,7 @@ sys.path.append(dirname(dirname(realpath(__file__)))) from misc.python_sealog.settings import WS_SERVER_URL, HEADERS from misc.coriolix_sealog.aux_data_record_builder import SealogCORIOLIXAuxDataRecordBuilder +from misc.aux_data_file_cleaners.do_nothing_aux_data_file_cleaner import DoNothingAuxDataFileCleaner from misc.aux_data_manager_runner import run_aux_data_manager # ----------------------------------------------------------------------------- # @@ -104,9 +105,14 @@ if __name__ == '__main__': def builder_factory(aux_data_config): '''Build an AuxDataRecordBuilder''' return SealogCORIOLIXAuxDataRecordBuilder(aux_data_config) + + def cleaner_factory(aux_data_config): + '''Build an AuxDataFileCleaner''' + return DoNothingAuxDataFileCleaner(aux_data_config) run_aux_data_manager( builder_factory=builder_factory, + cleaner_factory=cleaner_factory, inline_config=INLINE_CONFIG, ws_server_url=WS_SERVER_URL, headers=HEADERS, diff --git a/misc/sealog_aux_data_manager_influx.py.dist b/misc/sealog_aux_data_manager_influx.py.dist index a8e4dca..c8aa849 100644 --- a/misc/sealog_aux_data_manager_influx.py.dist +++ b/misc/sealog_aux_data_manager_influx.py.dist @@ -43,6 +43,7 @@ from misc.influx_sealog.settings import ( INFLUXDB_VERIFY_SSL ) from misc.influx_sealog.aux_data_record_builder import SealogInfluxAuxDataRecordBuilder +from misc.aux_data_file_cleaners.do_nothing_aux_data_file_cleaner import DoNothingAuxDataFileCleaner from misc.aux_data_manager_runner import run_aux_data_manager # ----------------------------------------------------------------------------- # @@ -105,9 +106,14 @@ if __name__ == '__main__': def builder_factory(aux_data_config): '''Build an AuxDataRecordBuilder''' return SealogInfluxAuxDataRecordBuilder(client, aux_data_config) + + def cleaner_factory(aux_data_config): + '''Build an AuxDataFileCleaner''' + return DoNothingAuxDataFileCleaner(aux_data_config) run_aux_data_manager( builder_factory=builder_factory, + cleaner_factory=cleaner_factory, inline_config=INLINE_CONFIG, ws_server_url=WS_SERVER_URL, headers=HEADERS, diff --git a/routes/api/v1/events.js b/routes/api/v1/events.js index c81d31d..9a18bdc 100644 --- a/routes/api/v1/events.js +++ b/routes/api/v1/events.js @@ -49,6 +49,47 @@ const _renameAndClearFields = (doc) => { return doc; }; +const _deleteEventsWithAuxData = async (db, server, query = {}, limit = 0, offset = 0, sort = { ts: 1 }) => { + // Find the events + const eventsToDelete = await db.collection(eventsTable) + .find(query) + .sort(sort) + .skip(offset) + .limit(limit) + .toArray(); + + if (eventsToDelete.length === 0) { + return { deletedCount: 0 }; + } + + const eventIDs = eventsToDelete.map((x) => x._id); + + // Fetch and group aux_data + const allAuxData = await db.collection(eventAuxDataTable) + .find({ event_id: { $in: eventIDs } }) + .toArray(); + + const eventIDToAuxData = allAuxData.reduce((dict, auxData) => { + const eventId = auxData.event_id.toString(); + if (!dict[eventId]) { + dict[eventId] = []; + } + dict[eventId].push(auxData); + return dict; + }, {}); + + // Publish deleteEvents message for each event with its aux data + // Let the aux data managers handle aux data deletion + for (const event of eventsToDelete) { + event.aux_data = eventIDToAuxData[event._id.toString()] || []; + server.publish('/ws/status/deleteEvents', _renameAndClearFields(event)); + } + + // Delete event records + const results = await db.collection(eventsTable).deleteMany({ _id: { $in: eventIDs } }); + + return { deletedCount: results.deletedCount }; +}; exports.plugin = { name: 'routes-api-events', @@ -1011,12 +1052,14 @@ exports.plugin = { const updatedEvent = _renameAndClearFields(result.value); if (time_change) { - server.publish('/ws/status/deleteEvents', updatedEvent); - // delete any aux_data const aux_data_query = { event_id: updatedEvent.id }; - await db.collection(eventAuxDataTable).deleteMany(aux_data_query); + const aux_data_result = await db.collection(eventAuxDataTable).find(aux_data_query).toArray(); + updatedEvent.aux_data = aux_data_result; + server.publish('/ws/status/deleteEvents', _renameAndClearFields(updatedEvent)); + + // console.log(del_results); server.publish('/ws/status/newEvents', updatedEvent); @@ -1106,42 +1149,13 @@ exports.plugin = { const offset = (request.query.offset) ? request.query.offset : 0; const sort = (request.query.sort === 'newest') ? { ts: -1 } : { ts: 1 }; - let eventIDs = []; - - // find the events try { - const results = await db.collection(eventsTable).find(query).sort(sort).project({ _id: 1 }).skip(offset).limit(limit).toArray(); // should return just the ids - // console.log("results:",results); - - if (results.length === 0) { - return h.response({ deletedCount: 0 }).code(200); - } - - eventIDs = results.map((x) => x._id); - // console.log("eventIDs:",eventIDs); + const result = await _deleteEventsWithAuxData(db, server, query, limit, offset, sort); + return h.response(result).code(200); } catch (err) { console.log(err); - return Boom.serverUnavailable('database error'); - } - - // delete the aux_data records - try { - await db.collection(eventAuxDataTable).deleteMany({ event_id: { $in: eventIDs } }); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } - - // delete the event records - try { - const results = await db.collection(eventsTable).deleteMany({ _id: { $in: eventIDs } }); - return h.response({ deletedCount: results.deletedCount }).code(200); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); + return Boom.serverUnavailable('database error', err); } } else { @@ -1151,43 +1165,15 @@ exports.plugin = { const offset = (request.query.offset) ? request.query.offset : 0; const sort = (request.query.sort === 'newest') ? { ts: -1 } : { ts: 1 }; - let eventIDs = []; - - // find the events try { - const results = await db.collection(eventsTable).find(query).sort(sort).project({ _id: 1 }).skip(offset).limit(limit).toArray(); - // console.log("results:", results); - - if (results.length === 0) { - return h.response({ deletedCount: 0 }).code(200); - } - - eventIDs = results.map((x) => x._id); - // console.log("eventIDs:",eventIDs); + const result = await _deleteEventsWithAuxData(db, server, query, limit, offset, sort); + return h.response(result).code(200); } catch (err) { console.log(err); - return Boom.serverUnavailable('database error'); - } - - // delete the aux_data records - try { - await db.collection(eventAuxDataTable).deleteMany({ event_id: { $in: eventIDs } }); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); + return Boom.serverUnavailable('database error', err); } - // delete the event records - try { - const results = await db.collection(eventsTable).deleteMany({ _id: { $in: eventIDs } }); - return h.response({ deletedCount: results.deletedCount }).code(200); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } } }, config: { @@ -1263,14 +1249,6 @@ exports.plugin = { return Boom.serverUnavailable('database error'); } - try { - await db.collection(eventAuxDataTable).deleteMany({ event_id: new ObjectID(request.params.id) }); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } - server.publish('/ws/status/deleteEvents', _renameAndClearFields(event)); return h.response().code(204); @@ -1303,20 +1281,12 @@ exports.plugin = { // const ObjectID = request.mongo.ObjectID; try { - await db.collection(eventsTable).deleteMany(); - } - catch (err) { - console.log(err); - return Boom.serverUnavailable('database error'); - } - - try { - await db.collection(eventAuxDataTable).deleteMany(); - return h.response().code(204); + const result = await _deleteEventsWithAuxData(db, server); + return h.response(result).code(200); } catch (err) { console.log(err); - return Boom.serverUnavailable('database error'); + return Boom.serverUnavailable('database error', err); } }, config: { @@ -1337,4 +1307,4 @@ exports.plugin = { } }); } -}; +}; \ No newline at end of file From 1a3734da14548a1931852937e97ab7f7a13f71d8 Mon Sep 17 00:00:00 2001 From: ljones Date: Wed, 4 Mar 2026 15:48:22 -0800 Subject: [PATCH 14/17] Update framegrab aux data to use new framework as an example. Now, when you delete framegrab aux data, it will also delete the images from the sealog files Note: we do not use framegrab, so I didn't have a great way to test this, but it is very similar to our stillcap_ffmpeg aux data mgr, which I have tested extensively --- .../delete_files_aux_data_file_cleaner.py | 73 +++++ .../aux_data_record_builder_framegrab_http.py | 129 ++++++++ ...aux_data_record_builder_framegrab_local.py | 111 +++++++ .../aux_data_record_builder_framegrab_scp.py | 141 +++++++++ misc/framegrab_aux/settings.py | 44 +++ ...sealog_aux_data_inserter_framegrab.py.dist | 284 ------------------ .../sealog_aux_data_manager_framegrab.py.dist | 72 +++++ 7 files changed, 570 insertions(+), 284 deletions(-) create mode 100644 misc/aux_data_file_cleaners/delete_files_aux_data_file_cleaner.py create mode 100644 misc/framegrab_aux/aux_data_record_builder_framegrab_http.py create mode 100644 misc/framegrab_aux/aux_data_record_builder_framegrab_local.py create mode 100644 misc/framegrab_aux/aux_data_record_builder_framegrab_scp.py create mode 100644 misc/framegrab_aux/settings.py delete mode 100644 misc/sealog_aux_data_inserter_framegrab.py.dist create mode 100644 misc/sealog_aux_data_manager_framegrab.py.dist diff --git a/misc/aux_data_file_cleaners/delete_files_aux_data_file_cleaner.py b/misc/aux_data_file_cleaners/delete_files_aux_data_file_cleaner.py new file mode 100644 index 0000000..59ce441 --- /dev/null +++ b/misc/aux_data_file_cleaners/delete_files_aux_data_file_cleaner.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +''' +FILE: delete_files_aux_data_file_cleaner.py + +DESCRIPTION: Cleans up the files created by the aux data creation process. + +BUGS: +NOTES: +AUTHOR: Lindsey Jones +COMPANY: OET +VERSION: 1.0 +CREATED: 2026-02-20 +REVISION: + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' +import sys +import os + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(dirname(realpath(__file__))))) + +from misc.aux_data_file_cleaners.base_aux_data_file_cleaner import AuxDataFileCleaner + + +class DeleteFilesAuxDataFileCleaner(AuxDataFileCleaner): + ''' + Cleans up the files created by the stillcap_ffmpeg aux data creation process + ''' + + def open_connections(self): + ''' + No connections to open. + ''' + + def close_connections(self): + ''' + No connections to close. + ''' + + def clean_aux_data_record(self, event, dry_run): + ''' + Do any clean up required for the given event + + Args: + event (dict): Event dictionary containing 'id' and 'ts' keys + dry_run (bool): If True, do not actually delete any aux data records + + Returns: + str or None: ID of cleaned aux data record, if there is one + ''' + aux_data = self._get_aux_data_for_source(event) + + if not aux_data: + return None + + file_paths = [ + data["data_value"] + for data in aux_data["data_array"] + if data["data_name"] == "filename" + ] + for file_path in file_paths: + try: + self.logger.debug(f"Deleting file {file_path} for event {event['id']}") + if not dry_run: + os.remove(file_path) + except Exception as e: # pylint: disable=W0718 + self.logger.error( + f"Error deleting file {file_path} for event {event['id']}: {e}" + ) + + return aux_data["_id"] diff --git a/misc/framegrab_aux/aux_data_record_builder_framegrab_http.py b/misc/framegrab_aux/aux_data_record_builder_framegrab_http.py new file mode 100644 index 0000000..9e79736 --- /dev/null +++ b/misc/framegrab_aux/aux_data_record_builder_framegrab_http.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +''' +FILE: aux_data_record_builder_test_images.py + +DESCRIPTION: This script generates test images and uses them to build a sealog aux_data record. +''' +import os +import sys +import logging +from datetime import datetime, timedelta +import requests +import shutil + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(realpath(__file__)))) + +from misc.base_aux_data_record_builder import AuxDataRecordBuilder +from misc.framegrab_aux.settings import DEST_DIR, SOURCES, THRESHOLD + +class FramegrabHTTPAuxDataRecordBuilder(AuxDataRecordBuilder): # pylint: disable=too-few-public-methods + ''' + Class that handles generating test images and using the + resulting data to build a sealog aux_data record. + ''' + + def __init__(self, + aux_data_config): + super().__init__(aux_data_config) + + @staticmethod + def _build_query_range(ts): + # see, this is making me feel like this shouldn't be in the base class + raise NotImplementedError("No query for framegrab") + + def open_connections(self): + ''' + Open any necessary connections to external data sources. + Must be implemented by subclasses. + ''' + pass + + def close_connections(self): + ''' + Close any open connections to external data sources. + Must be implemented by subclasses. + ''' + pass + + def _build_destination_filepath(self, str_timestamp, filename_prefix, filename_suffix): + timestamp = datetime.strptime( + str_timestamp, + '%Y-%m-%dT%H:%M:%S.%fZ' + ) + filename_date = datetime.date(timestamp) + filename_time = datetime.time(timestamp) + filename_middle = datetime.combine( + filename_date, filename_time + ).strftime("%Y%m%d_%H%M%S%f")[:-3] + + destination_filepath = os.path.join( + DEST_DIR, + filename_prefix + filename_middle + filename_suffix + ) + + return destination_filepath + + def build_aux_data_record(self, event): + ''' + Build the aux_data record for the given event. + ''' + if datetime.strptime( + event['ts'], + '%Y-%m-%dT%H:%M:%S.%fZ' + ) < datetime.utcnow()-timedelta(seconds=THRESHOLD): + logging.debug("Skipping because event ts is older than thresold") + return None + + aux_data_record = { + 'event_id': event['id'], + 'data_source': self._data_source, + 'data_array': [] + } + + for source in SOURCES: + + dst = self._build_destination_filepath( + event['ts'], + source['filename_prefix'], + source['filename_suffix'] + ) + + logging.debug("dst: %s", dst) + + try: + res = requests.get( + source['source_url'] + source['source_filename'], + stream=True, + timeout=(2, None) + ) + + if res.status_code != 200: + logging.error( + "Unable to retrieve image from: %s", + source['source_url'] + source['source_filename'] + ) + continue + + except requests.exceptions.RequestException as exc: + logging.error("Unable to retrieve image from remote server") + logging.error(exc) + + try: + with open(dst, 'wb') as f: + shutil.copyfileobj(res.raw, f) + + except shutil.Error as exc: + logging.error("Unable to save image to server") + logging.error(exc) + + aux_data_record['data_array'].append( + {'data_name': "camera_name", 'data_value': source['source_name']} + ) + aux_data_record['data_array'].append( + {'data_name': "filename", 'data_value': dst} + ) + + if len(aux_data_record['data_array']) > 0: + return aux_data_record + return None diff --git a/misc/framegrab_aux/aux_data_record_builder_framegrab_local.py b/misc/framegrab_aux/aux_data_record_builder_framegrab_local.py new file mode 100644 index 0000000..2590df5 --- /dev/null +++ b/misc/framegrab_aux/aux_data_record_builder_framegrab_local.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +''' +FILE: aux_data_record_builder_test_images.py + +DESCRIPTION: This script generates test images and uses them to build a sealog aux_data record. +''' +import os +import sys +import logging +from datetime import datetime, timedelta +import shutil + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(realpath(__file__)))) + +from misc.base_aux_data_record_builder import AuxDataRecordBuilder +from misc.framegrab_aux.settings import SOURCE_DIR, DEST_DIR, SOURCES, THRESHOLD + +class FramegrabSCPAuxDataRecordBuilder(AuxDataRecordBuilder): # pylint: disable=too-few-public-methods + ''' + Class that handles generating test images and using the + resulting data to build a sealog aux_data record. + ''' + + def __init__(self, + aux_data_config): + super().__init__(aux_data_config) + + @staticmethod + def _build_query_range(ts): + # see, this is making me feel like this shouldn't be in the base class + raise NotImplementedError("No query for framegrab") + + def open_connections(self): + ''' + Open any necessary connections to external data sources. + Must be implemented by subclasses. + ''' + pass + + def close_connections(self): + ''' + Close any open connections to external data sources. + Must be implemented by subclasses. + ''' + pass + + def _build_destination_filepath(self, str_timestamp, filename_prefix, filename_suffix): + timestamp = datetime.strptime( + str_timestamp, + '%Y-%m-%dT%H:%M:%S.%fZ' + ) + filename_date = datetime.date(timestamp) + filename_time = datetime.time(timestamp) + filename_middle = datetime.combine( + filename_date, filename_time + ).strftime("%Y%m%d_%H%M%S%f")[:-3] + + destination_filepath = os.path.join( + DEST_DIR, + filename_prefix + filename_middle + filename_suffix + ) + + return destination_filepath + + def build_aux_data_record(self, event): + ''' + Build the aux_data record for the given event. + ''' + if datetime.strptime( + event['ts'], + '%Y-%m-%dT%H:%M:%S.%fZ' + ) < datetime.utcnow()-timedelta(seconds=THRESHOLD): + logging.debug("Skipping because event ts is older than thresold") + return None + + aux_data_record = { + 'event_id': event['id'], + 'data_source': self._data_source, + 'data_array': [] + } + + for source in SOURCES: + + dst = self._build_destination_filepath( + event['ts'], + source['filename_prefix'], + source['filename_suffix'] + ) + + logging.debug("dst: %s", dst) + + latest_file = os.path.join(SOURCE_DIR, source['source_filename']) + src = os.path.join(SOURCE_DIR, latest_file) + try: + shutil.copyfile(src,dst) + + except shutil.Error as exc: + logging.error("Unable to save image to server") + logging.error(exc) + + aux_data_record['data_array'].append( + {'data_name': "camera_name", 'data_value': source['source_name']} + ) + aux_data_record['data_array'].append( + {'data_name': "filename", 'data_value': dst} + ) + + if len(aux_data_record['data_array']) > 0: + return aux_data_record + return None diff --git a/misc/framegrab_aux/aux_data_record_builder_framegrab_scp.py b/misc/framegrab_aux/aux_data_record_builder_framegrab_scp.py new file mode 100644 index 0000000..98cbc4e --- /dev/null +++ b/misc/framegrab_aux/aux_data_record_builder_framegrab_scp.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +''' +FILE: aux_data_record_builder_test_images.py + +DESCRIPTION: This script generates test images and uses them to build a sealog aux_data record. +''' +import os +import sys +import logging +from datetime import datetime, timedelta + +try: + from paramiko import RSAKey, SFTPClient, Transport # noqa:F401 pylint:disable=W0611 + from paramiko.sftp import SFTPError # noqa:F401 pylint:disable=W0611 + PARAMIKO_ENABLED = True +except ImportError: + PARAMIKO_ENABLED = False + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(realpath(__file__)))) + +from misc.base_aux_data_record_builder import AuxDataRecordBuilder +from misc.framegrab_aux.settings import SOURCE_DIR, DEST_DIR, SOURCES, THRESHOLD, \ + HOST, USER, KEY_FILE, PORT + +class FramegrabSCPAuxDataRecordBuilder(AuxDataRecordBuilder): # pylint: disable=too-few-public-methods + ''' + Class that handles generating test images and using the + resulting data to build a sealog aux_data record. + ''' + + def __init__(self, + aux_data_config): + super().__init__(aux_data_config) + if not PARAMIKO_ENABLED: + raise ModuleNotFoundError( + 'paramiko module is not installed. Try "pip3 install paramiko" prior to use.' + ) + self._host = HOST + self._user = USER + self._key = RSAKey.from_private_key_file(KEY_FILE) + self._port = PORT + + self._scp_transport = None + self._sftp_client = None + + @staticmethod + def _build_query_range(ts): + # see, this is making me feel like this shouldn't be in the base class + raise NotImplementedError("No query for framegrab") + + def open_connections(self): + ''' + Open any necessary connections to external data sources. + Must be implemented by subclasses. + ''' + self._scp_transport = Transport(self._host, self._port) + self._scp_transport.connect(username=self._user, pkey=self._key) + self._scp_transport.set_keepalive(30) + # since opening connection higher up, set keepalive to prevent going stale + logging.info("Opening SFTP connection") + self._sftp_client = SFTPClient.from_transport(self._scp_transport) + pass + + def close_connections(self): + ''' + Close any open connections to external data sources. + Must be implemented by subclasses. + ''' + try: + self._scp_transport.close() + except Exception as err: # pylint: disable=W0718 + logging.error("Error closing SCP Transport: %s", str(err)) + pass + + def _build_destination_filepath(self, str_timestamp, filename_prefix, filename_suffix): + timestamp = datetime.strptime( + str_timestamp, + '%Y-%m-%dT%H:%M:%S.%fZ' + ) + filename_date = datetime.date(timestamp) + filename_time = datetime.time(timestamp) + filename_middle = datetime.combine( + filename_date, filename_time + ).strftime("%Y%m%d_%H%M%S%f")[:-3] + + destination_filepath = os.path.join( + DEST_DIR, + filename_prefix + filename_middle + filename_suffix + ) + + return destination_filepath + + def build_aux_data_record(self, event): + ''' + Build the aux_data record for the given event. + ''' + if datetime.strptime( + event['ts'], + '%Y-%m-%dT%H:%M:%S.%fZ' + ) < datetime.utcnow()-timedelta(seconds=THRESHOLD): + logging.debug("Skipping because event ts is older than thresold") + return None + + aux_data_record = { + 'event_id': event['id'], + 'data_source': self._data_source, + 'data_array': [] + } + + for source in SOURCES: + + dst = self._build_destination_filepath( + event['ts'], + source['filename_prefix'], + source['filename_suffix'] + ) + + logging.debug("dst: %s", dst) + + try: + latest_file = os.path.join(SOURCE_DIR, source['source_filename']) + src = os.path.join(SOURCE_DIR, latest_file) + self._sftp_client.put(src, dst) + except SFTPError as exc: + logging.error("Unable to copy image to server") + logging.error(exc) + except OSError as exc: + logging.error("Unable to copy image to server") + logging.error(exc) + + aux_data_record['data_array'].append( + {'data_name': "camera_name", 'data_value': source['source_name']} + ) + aux_data_record['data_array'].append( + {'data_name': "filename", 'data_value': dst} + ) + + if len(aux_data_record['data_array']) > 0: + return aux_data_record + return None diff --git a/misc/framegrab_aux/settings.py b/misc/framegrab_aux/settings.py new file mode 100644 index 0000000..2a71f8b --- /dev/null +++ b/misc/framegrab_aux/settings.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +''' +FILE: settings.py + +DESCRIPTION: This file contains the settings used by the influx_sealog + functions to communicate with the influxDB API + +BUGS: +NOTES: +AUTHOR: Webb Pinner +COMPANY: OceanDataTools.org +VERSION: 2.0 +CREATED: 2025-02-08 +REVISION: + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' + +THRESHOLD = 20 # seconds + +# ------------ only needed for scp transfers -------------- +USER = 'survey' +HOST = '192.168.1.42' +PORT = 22 +KEY_FILE = '/home/sealog/.ssh/id_rsa' + +# ------------ only needed for local transfers ------------ +SOURCE_DIR = '/mnt/ramdisk' + +# --------------------------------------------------------- + +# This needs to match the FILEPATH_ROOT variable in ../config/server_setting.js +DEST_DIR = '/opt/sealog-server/sealog-files/images' + +SOURCES = [ + { + 'source_url': 'http://192.168.1.42/images/', + 'source_filename': 'camera1.jpg', + 'source_name': 'CAMERA_1', + 'filename_prefix': '', + 'filename_suffix': '.jpg' + } +] \ No newline at end of file diff --git a/misc/sealog_aux_data_inserter_framegrab.py.dist b/misc/sealog_aux_data_inserter_framegrab.py.dist deleted file mode 100644 index d4bc8e8..0000000 --- a/misc/sealog_aux_data_inserter_framegrab.py.dist +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env python3 -''' -FILE: sealog_aux_data_inserter_framegrab.py - -DESCRIPTION: This service listens for new events submitted to Sealog, copies - frame grab file(s) from a source dir, renames/copies the file - to the sealog-files/images directory and creates an aux_data - record containing the specified real-time data and associates - the aux data record with the newly created event. However if - the realtime data is older than 20 seconds this service will - consider the data stale and will not associate it with the - newly created event. - -BUGS: -NOTES: -AUTHOR: Webb Pinner -COMPANY: OceanDataTools.org -VERSION: 1.1 -CREATED: 2020-01-27 -REVISION: 2023-02-10 - -LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) - Copyright (C) OceanDataTools.org 2025 -''' - -import os -import sys -import asyncio -import json -import time -import shutil -import logging -from datetime import datetime, timedelta -import requests -import websockets - -from os.path import dirname, realpath -sys.path.append(dirname(dirname(realpath(__file__)))) - -try: - from paramiko import RSAKey, SSHClient, SFTPClient, Transport # noqa:F401 pylint:disable=W0611 - from paramiko.sftp import SFTPError # noqa:F401 pylint:disable=W0611 - PARAMIKO_ENABLED = True -except ImportError: - PARAMIKO_ENABLED = False - -from misc.python_sealog.settings import WS_SERVER_URL, HEADERS -from misc.python_sealog.event_aux_data import create_event_aux_data - -# The data_source to use for the auxData records -AUX_DATA_DATASOURCE = 'vehicleRealtimeFramegrabberData' - -# set of events to ignore -EXCLUDE_SET = () - -# needs to be unique for all currently active dataInserter scripts. -CLIENT_WSID = f'aux_data_inserter_{AUX_DATA_DATASOURCE}' - -THRESHOLD = 20 # seconds - -# ------------ only needed for scp transfers -------------- -# if not PARAMIKO_ENABLED: -# raise ModuleNotFoundError('paramiko module is not installed. Please try -# "pip3 install paramiko" prior to use.') -# user = 'survey' -# host = '192.168.1.42' -# port = 22 -# key_file = '/home/sealog/.ssh/id_rsa' -# my_key = RSAKey.from_private_key_file(key_file) -# t = Transport(host, port) - -# ------------ only needed for local transfers ------------ -SOURCE_DIR = '/mnt/ramdisk' - -# --------------------------------------------------------- - -# This needs to match the FILEPATH_ROOT variable in ../config/server_setting.js -DEST_DIR = '/opt/sealog-server/sealog-files/images' - -sources = [ - { - 'source_url': 'http://192.168.1.42/images/', - 'source_filename': 'camera1.jpg', - 'source_name': 'CAMERA_1', - 'filename_prefix': '', - 'filename_suffix': '.jpg' - } -] - -HELLO = { - 'type': 'hello', - 'id': CLIENT_WSID, - 'auth': { - 'headers': HEADERS - }, - 'version': '2', - 'subs': ['/ws/status/newEvents'] -} - -PING = { - 'type': 'ping', - 'id': CLIENT_WSID -} - - -async def aux_data_inserter(): - ''' - Connect to the websocket feed for new events. When new events arrive, - build aux_data records and submit them to the sealog-server. - ''' - - logging.debug("Connecting to event websocket feed...") - try: - async with websockets.connect(WS_SERVER_URL) as websocket: - - await websocket.send(json.dumps(HELLO)) - - while True: - - event = await websocket.recv() - event_obj = json.loads(event) - - if event_obj['type'] and event_obj['type'] == 'ping': - await websocket.send(json.dumps(PING)) - - elif event_obj['type'] and event_obj['type'] == 'pub': - - if event_obj['message']['event_value'] in EXCLUDE_SET: - logging.debug( - "Skipping because event value is in the exclude set" - ) - continue - - if datetime.strptime( - event_obj['message']['ts'], - '%Y-%m-%dT%H:%M:%S.%fZ' - ) < datetime.utcnow()-timedelta(seconds=THRESHOLD): - logging.debug("Skipping because event ts is older than thresold") - continue - - aux_data_record = { - 'event_id': event_obj['message']['id'], - 'data_source': AUX_DATA_DATASOURCE, - 'data_array': [] - } - - for source in sources: - - filename_date = datetime.date(datetime.strptime( - event_obj['message']['ts'], - '%Y-%m-%dT%H:%M:%S.%fZ') - ) - filename_time = datetime.time( - datetime.strptime( - event_obj['message']['ts'], - '%Y-%m-%dT%H:%M:%S.%fZ' - ) - ) - filename_middle = datetime.combine( - filename_date, filename_time - ).strftime("%Y%m%d_%H%M%S%f")[:-3] - - dst = os.path.join( - DEST_DIR, - source['filename_prefix'] + filename_middle + source['filename_suffix'] - ) - - logging.debug("dst: %s", dst) - - # ------------ only needed for scp transfers ------------- - # try: - # latest_file = os.path.join(SOURCE_DIR, source['source_filename']) - # src = os.path.join(SOURCE_DIR, latest_file) - # sftp = SFTPClient.from_transport(t) - # sftp.put(src, dst) - # sftp.close() - - # except SFTPError as exc: - # logging.error("Unable to copy image to server") - # logging.error(exc) - - # except OSError as exc: - # logging.error("Unable to copy image to server") - # logging.error(exc) - # ------------ only needed for scp transfers ------------- - - # ------------ only needed for http transfers ------------- - try: - res = requests.get( - source['source_url'] + source['source_filename'], - stream=True, - timeout=(2, None) - ) - - if res.status_code != 200: - logging.error( - "Unable to retrieve image from: %s", - source['source_url'] + source['source_filename'] - ) - continue - - except requests.exceptions.RequestException as exc: - logging.error("Unable to retrieve image from remote server") - logging.error(exc) - - try: - with open(dst, 'wb') as f: - shutil.copyfileobj(res.raw, f) - - except shutil.Error as exc: - logging.error("Unable to save image to server") - logging.error(exc) - # ------------ only needed for http transfers ------------- - - # ----------- only needed for local transfers ------------ - # latest_file = os.path.join(SOURCE_DIR, source['source_filename']) - # src = os.path.join(SOURCE_DIR, latest_file) - # try: - # shutil.copyfile(src,dst) - - # except shutil.Error as exc: - # logging.error("Unable to save image to server") - # logging.error(exc) - # ----------- only needed for local transfers ------------ - - aux_data_record['data_array'].append( - {'data_name': "camera_name", 'data_value': source['source_name']} - ) - aux_data_record['data_array'].append( - {'data_name': "filename", 'data_value': dst} - ) - - if len(aux_data_record['data_array']) > 0: - create_event_aux_data(aux_data_record) - - except Exception as exc: - logging.error(str(exc)) - raise exc - -# ------------------------------------------------------------------------------------- -# Required python code for running the script as a stand-alone utility -# ------------------------------------------------------------------------------------- -if __name__ == '__main__': - - import argparse - - parser = argparse.ArgumentParser( - description='Aux Data Inserter Service - ' + AUX_DATA_DATASOURCE - ) - parser.add_argument( - '-v', '--verbosity', dest='verbosity', default=0, action='count', - help='Increase output verbosity') - - parsed_args = parser.parse_args() - - ############################ - # Set up logging before we do any other argument parsing (so that we - # can log problems with argument parsing). - - LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s' - logging.basicConfig(format=LOGGING_FORMAT) - - LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} - parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS)) - logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity]) - - # Run the main loop - while True: - - # Wait 5 seconds for the server to complete startup - time.sleep(5) - - try: - # t.connect(username=user, pkey=my_key) # only needed for scp transfers - asyncio.get_event_loop().run_until_complete(aux_data_inserter()) - except KeyboardInterrupt: - logging.error('Keyboard Interrupted') - try: - sys.exit(0) - except SystemExit: - os._exit(0) - except Exception as exc: # pylint:disable=W0718 - logging.error("Lost connection to server, trying again in 5 seconds") - logging.debug(str(exc)) diff --git a/misc/sealog_aux_data_manager_framegrab.py.dist b/misc/sealog_aux_data_manager_framegrab.py.dist new file mode 100644 index 0000000..6df56de --- /dev/null +++ b/misc/sealog_aux_data_manager_framegrab.py.dist @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +''' +FILE: sealog_aux_data_manager_framegrab.py + +DESCRIPTION: This service listens for new events submitted to Sealog, copies + frame grab file(s) from a source dir, renames/copies the file + to the sealog-files/images directory and creates an aux_data + record containing the specified real-time data and associates + the aux data record with the newly created event. However if + the realtime data is older than 20 seconds this service will + consider the data stale and will not associate it with the + newly created event. + +BUGS: +NOTES: +AUTHOR: Webb Pinner +COMPANY: OceanDataTools.org +VERSION: 1.1 +CREATED: 2020-01-27 +REVISION: 2023-02-10 + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' + +import sys +from os.path import dirname, realpath +sys.path.append(dirname(dirname(realpath(__file__)))) + +from misc.python_sealog.settings import WS_SERVER_URL, HEADERS +from misc.framegrab_aux.aux_data_record_builder_framegrab_http import ( + FramegrabHTTPAuxDataRecordBuilder + ) +from misc.aux_data_file_cleaners.delete_files_aux_data_file_cleaner import ( + DeleteFilesAuxDataFileCleaner + ) +from misc.aux_data_manager_runner import run_aux_data_manager + +# The data_source to use for the auxData records +INLINE_CONFIG = ''' +- data_source: 'vehicleRealtimeFramegrabberData' +''' + +# set of events to ignore +EXCLUDE_SET = () + +# needs to be unique for all currently active dataInserter scripts. +CLIENT_WSID = f'aux_data_inserter_vehicleRealtimeFramegrabberData' + + +# ------------------------------------------------------------------------------------- +# Required python code for running the script as a stand-alone utility +# ------------------------------------------------------------------------------------- +if __name__ == '__main__': + + def builder_factory(aux_data_config): + '''Build an AuxDataRecordBuilder''' + return FramegrabHTTPAuxDataRecordBuilder(aux_data_config) + + def cleaner_factory(aux_data_config): + '''Build an AuxDataFileCleaner''' + return DeleteFilesAuxDataFileCleaner(aux_data_config) + + run_aux_data_manager( + builder_factory=builder_factory, + cleaner_factory=cleaner_factory, + inline_config=INLINE_CONFIG, + ws_server_url=WS_SERVER_URL, + headers=HEADERS, + client_wsid=CLIENT_WSID, + exclude_set=EXCLUDE_SET + ) From dcb011e6c15a722fd336900e4ac07ce94d721b7d Mon Sep 17 00:00:00 2001 From: ljones Date: Wed, 4 Mar 2026 16:16:29 -0800 Subject: [PATCH 15/17] Add OET's old InfluxDB aux data manager as another example --- .../aux_data_record_builder_v1.py | 148 ++++++++++++++++++ misc/influx_sealog/settings.py.dist | 5 + misc/sealog_aux_data_manager_influx.py.dist | 4 +- .../sealog_aux_data_manager_influx_v1.py.dist | 108 +++++++++++++ 4 files changed, 263 insertions(+), 2 deletions(-) create mode 100644 misc/influx_sealog/aux_data_record_builder_v1.py create mode 100644 misc/sealog_aux_data_manager_influx_v1.py.dist diff --git a/misc/influx_sealog/aux_data_record_builder_v1.py b/misc/influx_sealog/aux_data_record_builder_v1.py new file mode 100644 index 0000000..c1f8271 --- /dev/null +++ b/misc/influx_sealog/aux_data_record_builder_v1.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +''' +FILE: aux_data_record_builder.py + +DESCRIPTION: This script builds a sealog aux_data record with data pulled from an + influx database. + +BUGS: +NOTES: +AUTHOR: Webb Pinner +COMPANY: OceanDataTools.org +VERSION: 1.0 +CREATED: 2021-01-01 +REVISION: 2022-02-13 + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' +import sys +import logging +from datetime import datetime, timedelta +from urllib3.exceptions import NewConnectionError +from influxdb.exceptions import InfluxDBClientError, InfluxDBServerError + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(dirname(realpath(__file__))))) + +from misc.base_aux_data_record_builder import AuxDataRecordBuilder +from misc.influx_sealog.settings import ( + INFLUXDB_URL, + INFLUXDB_AUTH_TOKEN, + INFLUXDB_ORG, + INFLUXDB_BUCKET +) + + +class SealogInfluxV1AuxDataRecordBuilder(AuxDataRecordBuilder): # pylint: disable=too-few-public-methods # noqa: E501 + ''' + Class that handles the construction of an influxDB query and using the + resulting data to build a sealog aux_data record. + ''' + + def __init__(self, influxdb_client, aux_data_config, influxdb_bucket=INFLUXDB_BUCKET): + super().__init__(aux_data_config) + self._query_filters = aux_data_config.get('query_filters', []) + self._influxdb_client = influxdb_client + self._influxdb_bucket = ( + aux_data_config['query_bucket'] + if 'query_bucket' in aux_data_config else influxdb_bucket + ) + + @staticmethod + def _build_query_range(ts): + ''' + Builds the temporal range for the influxDB query based on the provided + timestamp (ts). + ''' + str_start_ts = datetime.strftime( + datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ") - timedelta(seconds=20), + "%Y-%m-%dT%H:%M:%SZ" + ) + # Note: in the new code, you subtract a whole minute. Did we choose 20 s on purpose? + + ts_filter = f"time <= '{ts}' AND time > '{str_start_ts}'" + return ts_filter + + def _build_query(self, ts): + ''' + Builds the complete influxDB query using the provided timestamp (ts) + and the class instance's query_measurements and query_fields values. + ''' + + str_field_names = ", ".join([ + f'"{q_field}"' + for q_field in self._query_fields + ]) + + str_time_range = self._build_query_range(ts) + + str_filters = " AND ".join(self._query_filters + [str_time_range]) + + # not sure what to do when there's more than one query measurement. + # Is that even possible in influx v1? + query = f'''SELECT {str_field_names} + FROM "{self._influxdb_bucket}"."one_month"."{self._query_measurements[0]}" + WHERE {str_filters} + ORDER BY DESC LIMIT 1''' + + logging.debug("Query: %s", query) + + return query + + def open_connections(self): + ''' + Open any necessary connections to external data sources. + For Influx, no persistent connection is needed. + ''' + pass + + def close_connections(self): + ''' + Close any open connections to external data sources. + For Influx, no persistent connection is needed. + ''' + pass + + def build_aux_data_record(self, event): + ''' + Build the aux_data record for the given event. + ''' + + logging.debug("building query") + query = self._build_query(event['ts']) + + logging.debug("Query: %s", query) + # run the query against the influxDB + try: + query_result = self._influxdb_client.query(query=query) + + except NewConnectionError: + logging.error("InfluxDB connection error, verify URL: %s", INFLUXDB_URL) + + except (InfluxDBClientError, InfluxDBServerError) as exc: + _, value, _ = sys.exc_info() + + if str(value).startswith("(400)"): + logging.error("InfluxDB API error, verify org: %s", INFLUXDB_ORG) + elif str(value).startswith("(401)"): + logging.error("InfluxDB API error, verify token: %s", INFLUXDB_AUTH_TOKEN) + elif str(value).startswith("(404)"): + logging.error("InfluxDB API error, verify bucket: %s", self._influxdb_bucket) + else: + logging.error("Error with query:") + logging.error(query.replace("|>", '\n')) + logging.error(str(exc)) + raise exc + else: + # Parse InfluxDB result into a dictionary format + influx_data = {} + for table in query_result: + for record in table.records: + influx_data[record.get_field()] = record.get_value() + + aux_data_record = self._build_aux_data_dict(event['id'], influx_data) + + return aux_data_record + + return None diff --git a/misc/influx_sealog/settings.py.dist b/misc/influx_sealog/settings.py.dist index bc94b5c..a629e05 100644 --- a/misc/influx_sealog/settings.py.dist +++ b/misc/influx_sealog/settings.py.dist @@ -23,3 +23,8 @@ INFLUXDB_ORG = 'openrvdas' INFLUXDB_BUCKET = 'openrvdas' INFLUXDB_AUTH_TOKEN = 'DEFAULT_INFLUXDB_AUTH_TOKEN' # noqa:E501 INFLUXDB_VERIFY_SSL = False + +INFLUX_HOST = '10.1.90.40' +INFLUX_PORT = 8086 +INFLUX_USER = '' +INFLUX_PASS = '' \ No newline at end of file diff --git a/misc/sealog_aux_data_manager_influx.py.dist b/misc/sealog_aux_data_manager_influx.py.dist index c8aa849..cf8661a 100644 --- a/misc/sealog_aux_data_manager_influx.py.dist +++ b/misc/sealog_aux_data_manager_influx.py.dist @@ -1,6 +1,6 @@ #!/usr/bin/env python3 ''' -FILE: sealog_aux_data_inserter_influx.py +FILE: sealog_aux_data_manager_influx.py DESCRIPTION: This service listens for new events submitted to Sealog, create aux_data records containing the specified real-time data and @@ -118,4 +118,4 @@ if __name__ == '__main__': ws_server_url=WS_SERVER_URL, headers=HEADERS, client_wsid=CLIENT_WSID - ) \ No newline at end of file + ) diff --git a/misc/sealog_aux_data_manager_influx_v1.py.dist b/misc/sealog_aux_data_manager_influx_v1.py.dist new file mode 100644 index 0000000..c0f1c8b --- /dev/null +++ b/misc/sealog_aux_data_manager_influx_v1.py.dist @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +''' +FILE: sealog_aux_data_manager_influx.py + +DESCRIPTION: This service listens for new events submitted to Sealog, create + aux_data records containing the specified real-time data and + associates the aux data records with the newly created event. + + This script leverages the OpenRVDAS/InfluxDB integration so that + if can add ancillary data from any point in time so long as the + data is availble from the InfluxDB. In the event the data is not + available the script will NOT add the corresponding aux_data + records. + + This script can also add influx data to a list of event ids + (comma-separated) using the -e flag, or all the events for a given + lowering using the -l flag, or all the events for a given + cruise using the -c flag + +BUGS: +NOTES: +AUTHOR: Webb Pinner +COMPANY: OceanDataTools.org +VERSION: 1.1 +CREATED: 2021-04-21 +REVISION: 2023-02-10 + +LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) + Copyright (C) OceanDataTools.org 2025 +''' + +import sys +from influxdb import InfluxDBClient + +from os.path import dirname, realpath +sys.path.append(dirname(dirname(realpath(__file__)))) + +from misc.python_sealog.settings import WS_SERVER_URL, HEADERS +from misc.influx_sealog.settings import ( + INFLUX_HOST, + INFLUX_PORT, + INFLUX_USER, + INFLUX_PASS +) +from misc.influx_sealog.aux_data_record_builder_v1 import SealogInfluxV1AuxDataRecordBuilder +from misc.aux_data_file_cleaners.do_nothing_aux_data_file_cleaner import DoNothingAuxDataFileCleaner +from misc.aux_data_manager_runner import run_aux_data_manager + +# ----------------------------------------------------------------------------- # + +INLINE_CONFIG = ''' +- data_source: hercRealtimeNavData + query_measurements: + - sonardyne_nav + query_filters: + - |- + "name" = 'Herc 2412' + aux_record_lookup: + latitude: + name: latitude + uom: ddeg + round: 6 + longitude: + name: longitude + uom: ddeg + round: 6 + depth: + name: depth + uom: m + round: 2 + name: + no_output: true +''' + +# set of events to ignore +EXCLUDE_SET = set() + +# needs to be unique for all currently active dataManager scripts. +CLIENT_WSID = 'auxData-dataManager-influx-v1' + +# ------------------------------------------------------------------------------------- +# The main loop of the utility +# ------------------------------------------------------------------------------------- +if __name__ == '__main__': + # create an influxDB Client + client = InfluxDBClient(host=INFLUX_HOST, + port=INFLUX_PORT, + username=INFLUX_USER, + password=INFLUX_PASS + ) + + # builder factory that injects the client + def builder_factory(aux_data_config): + '''Build an AuxDataRecordBuilder''' + return SealogInfluxV1AuxDataRecordBuilder(client, aux_data_config) + + def cleaner_factory(aux_data_config): + '''Build an AuxDataFileCleaner''' + return DoNothingAuxDataFileCleaner(aux_data_config) + + run_aux_data_manager( + builder_factory=builder_factory, + cleaner_factory=cleaner_factory, + inline_config=INLINE_CONFIG, + ws_server_url=WS_SERVER_URL, + headers=HEADERS, + client_wsid=CLIENT_WSID + ) From 88b7e97d85924e3ee57072513e63f426ee68db8b Mon Sep 17 00:00:00 2001 From: ljones Date: Thu, 5 Mar 2026 13:51:41 -0800 Subject: [PATCH 16/17] Clean up Lint all files, alphabetize imports, remove unused functions, use built in logger --- .../base_aux_data_file_cleaner.py | 36 +++++++-------- .../delete_files_aux_data_file_cleaner.py | 21 +++++---- .../do_nothing_aux_data_file_cleaner.py | 7 +-- .../stillcapffmpeg_aux_data_file_cleaner.py | 28 +++++++----- misc/aux_data_manager_runner.py | 28 ++++++------ misc/base_aux_data_record_builder.py | 39 +++++----------- .../aux_data_record_builder.py | 44 ++++++++++-------- .../aux_data_record_builder_framegrab_http.py | 43 +++++++----------- ...aux_data_record_builder_framegrab_local.py | 39 ++++++---------- .../aux_data_record_builder_framegrab_scp.py | 40 +++++++---------- .../{settings.py => settings.py.dist} | 2 +- misc/influx_sealog/aux_data_record_builder.py | 43 +++++++++--------- .../aux_data_record_builder_v1.py | 45 ++++++++++--------- misc/influx_sealog/settings.py.dist | 2 +- misc/python_sealog/event_aux_data.py | 15 ------- misc/sealog_aux_data_manager_coriolix.py.dist | 6 +-- .../sealog_aux_data_manager_framegrab.py.dist | 2 +- misc/sealog_aux_data_manager_influx.py.dist | 2 +- .../sealog_aux_data_manager_influx_v1.py.dist | 2 +- routes/api/v1/events.js | 7 ++- 20 files changed, 207 insertions(+), 244 deletions(-) rename misc/framegrab_aux/{settings.py => settings.py.dist} (99%) diff --git a/misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py b/misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py index 5c122de..f44a173 100644 --- a/misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py +++ b/misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py @@ -2,7 +2,8 @@ ''' FILE: base_aux_data_file_cleaner.py -DESCRIPTION: Base class for cleaning up any changes made by aux_data_inserters outside of the aux data mongo collection. +DESCRIPTION: Base class for cleaning up any changes made by aux_data_inserters outside + of the aux data mongo collection. BUGS: NOTES: @@ -15,10 +16,11 @@ LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) Copyright (C) OceanDataTools.org 2025 ''' -import json import logging + from abc import ABC, abstractmethod + class AuxDataFileCleaner(ABC): ''' Abstract base class for building sealog aux_data records from various data sources. @@ -32,63 +34,59 @@ def __init__(self, aux_data_config): ''' self._data_source = aux_data_config['data_source'] self.logger = logging.getLogger(__name__) - + def _get_aux_data_for_source(self, event): ''' Get the aux data record for the given event based on the data source. - + Args: event (dict): Event dictionary containing 'id' and 'ts' keys - + Returns: str or None: ID of cleaned aux data record, if there is one ''' all_aux_data = event.get('aux_data', {}) if not all_aux_data: - self.logger.debug(f"No aux data found for event {event['id']}") + self.logger.debug("No aux data found for event %s", event['id']) return None - + # Find the first aux_data record that matches the data source # There should only ever be one record per data source aux_data_for_source = next( (aux_data for aux_data in all_aux_data - if aux_data["data_source"] == self._data_source), + if aux_data["data_source"] == self._data_source), None # default if not found ) - + if not aux_data_for_source: - self.logger.debug(f"No {self._data_source} aux data found for event {event['id']}") - + self.logger.debug("No %s aux data found for event %s", self._data_source, event['id']) + return aux_data_for_source - - + @abstractmethod def open_connections(self): ''' Open any necessary connections to external data sources. Must be implemented by subclasses. ''' - pass - + @abstractmethod def close_connections(self): ''' Close any open connections to external data sources. Must be implemented by subclasses. ''' - pass @abstractmethod def clean_aux_data_record(self, event, dry_run): ''' Do any clean up required for the given event Must be implemented by subclasses. - + Args: event (dict): Event dictionary containing 'id' and 'ts' keys dry_run (bool): If True, do not actually delete any aux data records - + Returns: dict or None: Aux data record or None if no data available ''' - pass \ No newline at end of file diff --git a/misc/aux_data_file_cleaners/delete_files_aux_data_file_cleaner.py b/misc/aux_data_file_cleaners/delete_files_aux_data_file_cleaner.py index 59ce441..e4a2279 100644 --- a/misc/aux_data_file_cleaners/delete_files_aux_data_file_cleaner.py +++ b/misc/aux_data_file_cleaners/delete_files_aux_data_file_cleaner.py @@ -15,8 +15,8 @@ LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) Copyright (C) OceanDataTools.org 2025 ''' -import sys import os +import sys from os.path import dirname, realpath sys.path.append(dirname(dirname(dirname(realpath(__file__))))) @@ -54,20 +54,23 @@ def clean_aux_data_record(self, event, dry_run): if not aux_data: return None - + file_paths = [ - data["data_value"] - for data in aux_data["data_array"] - if data["data_name"] == "filename" - ] + data["data_value"] + for data in aux_data["data_array"] + if data["data_name"] == "filename" + ] for file_path in file_paths: try: - self.logger.debug(f"Deleting file {file_path} for event {event['id']}") + self.logger.debug("Deleting file %s for event %s", file_path, event['id']) if not dry_run: os.remove(file_path) except Exception as e: # pylint: disable=W0718 self.logger.error( - f"Error deleting file {file_path} for event {event['id']}: {e}" - ) + "Error deleting file %s for event %s: %s", + file_path, + event['id'], + e + ) return aux_data["_id"] diff --git a/misc/aux_data_file_cleaners/do_nothing_aux_data_file_cleaner.py b/misc/aux_data_file_cleaners/do_nothing_aux_data_file_cleaner.py index ac53934..cc36aa5 100644 --- a/misc/aux_data_file_cleaners/do_nothing_aux_data_file_cleaner.py +++ b/misc/aux_data_file_cleaners/do_nothing_aux_data_file_cleaner.py @@ -53,9 +53,10 @@ def clean_aux_data_record(self, event, dry_run): # pylint: disable=W0613 if aux_data: self.logger.debug( - f"No additional clean up required for event {event['id']} {self._data_source} " - "aux data records" - ) + "No additional clean up required for event %s %s aux data records", + event['id'], + self._data_source + ) return aux_data["_id"] return None diff --git a/misc/aux_data_file_cleaners/stillcapffmpeg_aux_data_file_cleaner.py b/misc/aux_data_file_cleaners/stillcapffmpeg_aux_data_file_cleaner.py index 90efa46..d545ba5 100644 --- a/misc/aux_data_file_cleaners/stillcapffmpeg_aux_data_file_cleaner.py +++ b/misc/aux_data_file_cleaners/stillcapffmpeg_aux_data_file_cleaner.py @@ -15,8 +15,8 @@ LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) Copyright (C) OceanDataTools.org 2025 ''' -import sys import os +import sys from os.path import dirname, realpath sys.path.append(dirname(dirname(dirname(realpath(__file__))))) @@ -54,29 +54,35 @@ def clean_aux_data_record(self, event, dry_run): if aux_data: file_paths = [ - data["data_value"] - for data in aux_data["data_array"] - if data["data_name"] == "filename" - ] + data["data_value"] + for data in aux_data["data_array"] + if data["data_name"] == "filename" + ] for file_path in file_paths: try: - self.logger.debug(f"Deleting file {file_path} for event {event['id']}") + self.logger.debug("Deleting file %s for event %s", file_path, event['id']) if not dry_run: os.remove(file_path) except Exception as e: # pylint: disable=W0718 self.logger.error( - f"Error deleting file {file_path} for event {event['id']}: {e}" - ) + "Error deleting file %s for event %s: %s", + file_path, + event['id'], + e + ) try: sym_path = file_path.replace('png', 'symlink') - self.logger.debug(f"Deleting symlink {sym_path} for event {event['id']}") + self.logger.debug("Deleting symlink %s for event %s", sym_path, event['id']) if not dry_run: os.remove(sym_path) except Exception as e: # pylint: disable=W0718 self.logger.error( - f"Error deleting symlink {sym_path} for event {event['id']}: {e}" - ) + "Error deleting symlink %s for event %s: %s", + sym_path, + event['id'], + e + ) return aux_data["_id"] return None diff --git a/misc/aux_data_manager_runner.py b/misc/aux_data_manager_runner.py index c8c6f90..191e814 100644 --- a/misc/aux_data_manager_runner.py +++ b/misc/aux_data_manager_runner.py @@ -4,28 +4,28 @@ DESCRIPTION: Shared runner for aux-data inserter scripts. ''' +import argparse +import asyncio +import json +import logging +import os import re import sys -import json import time -import logging -import asyncio import websockets import yaml -import os -import argparse + from typing import Callable, Dict, Any from os.path import dirname, realpath sys.path.append(dirname(dirname(realpath(__file__)))) -from misc.base_aux_data_record_builder import AuxDataRecordBuilder from misc.aux_data_file_cleaners.base_aux_data_file_cleaner import AuxDataFileCleaner +from misc.base_aux_data_record_builder import AuxDataRecordBuilder +from misc.python_sealog.cruises import get_cruise_uid_by_id from misc.python_sealog.events import get_event, get_events_by_cruise, get_events_by_lowering from misc.python_sealog.event_aux_data import create_event_aux_data, delete_event_aux_data from misc.python_sealog.lowerings import get_lowering_uid_by_id -from misc.python_sealog.cruises import get_cruise_uid_by_id - LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} LOGGING_FORMAT = '%(asctime)-15s %(levelname)s %(lineno)s - %(message)s' @@ -305,16 +305,16 @@ def run_aux_data_manager( if parsed_args.cruise_id: insert_aux_data_for_cruise( - aux_data_builder_list, - parsed_args.cruise_id, - parsed_args.dry_run) + aux_data_builder_list, + parsed_args.cruise_id, + parsed_args.dry_run) sys.exit(0) if parsed_args.lowering_id: insert_aux_data_for_lowering( - aux_data_builder_list, - parsed_args.lowering_id, - parsed_args.dry_run) + aux_data_builder_list, + parsed_args.lowering_id, + parsed_args.dry_run) sys.exit(0) # Wait then start WS loop forever. Sleep and retry on any disconnect/error diff --git a/misc/base_aux_data_record_builder.py b/misc/base_aux_data_record_builder.py index 8d5b6b9..29cef96 100644 --- a/misc/base_aux_data_record_builder.py +++ b/misc/base_aux_data_record_builder.py @@ -19,6 +19,7 @@ import logging from abc import ABC, abstractmethod + class AuxDataRecordBuilder(ABC): ''' Abstract base class for building sealog aux_data records from various data sources. @@ -29,14 +30,15 @@ class AuxDataRecordBuilder(ABC): def __init__(self, aux_data_config): ''' Initialize the base builder with configuration. - + Args: aux_data_config (dict): Configuration dictionary containing: - query_measurements: List of measurements to query - aux_record_lookup: Mapping of fields to output configuration - data_source: Name of the data source ''' - self._data_source = aux_data_config['data_source'] # don't use get--should throw an error if not specified + self._data_source = aux_data_config['data_source'] + # don't use get for data source--want to throw an error if not specified self._query_measurements = aux_data_config.get('query_measurements') self._aux_record_lookup = aux_data_config.get('aux_record_lookup') if self._aux_record_lookup is None: @@ -45,63 +47,44 @@ def __init__(self, aux_data_config): self._query_fields = list(self._aux_record_lookup.keys()) self.logger = logging.getLogger(__name__) - @staticmethod - @abstractmethod - def _build_query_range(ts): - ''' - Builds the temporal range for queries based on the provided timestamp (ts). - Format depends on the specific data source implementation. - - Args: - ts (str): Timestamp in ISO 8601 format (YYYY-MM-DDTHH:MM:SS.fffZ) - - Returns: - str or None: Query range string (format varies by implementation) - ''' - # I'm not sure why we need this to be a static method, but I'm including it in the base class since it's in both implementations - pass - @abstractmethod def open_connections(self): ''' Open any necessary connections to external data sources. Must be implemented by subclasses. ''' - pass - + @abstractmethod def close_connections(self): ''' Close any open connections to external data sources. Must be implemented by subclasses. ''' - pass @abstractmethod def build_aux_data_record(self, event): ''' Build the aux_data record for the given event. Must be implemented by subclasses. - + Args: event (dict): Event dictionary containing 'id' and 'ts' keys - + Returns: dict or None: Aux data record or None if no data available ''' - pass def _build_aux_data_dict(self, event_id, query_data): # pylint:disable=R0915 ''' Internal method to build the sealog aux_data record using the event_id, query_data and the class instance's datasource value. - + This method handles common transformations and modifications across all data sources. - + Args: event_id (str): The ID of the event query_data (dict): Dictionary of field names to values from the data source - + Returns: dict or None: Aux data record or None if no data available ''' @@ -229,4 +212,4 @@ def record_lookup(self): ''' Getter method for the _aux_record_lookup property ''' - return self._aux_record_lookup \ No newline at end of file + return self._aux_record_lookup diff --git a/misc/coriolix_sealog/aux_data_record_builder.py b/misc/coriolix_sealog/aux_data_record_builder.py index 1a6af35..815f14c 100644 --- a/misc/coriolix_sealog/aux_data_record_builder.py +++ b/misc/coriolix_sealog/aux_data_record_builder.py @@ -16,11 +16,11 @@ LICENSE INFO: This code is licensed under MIT license (see LICENSE.txt for details) Copyright (C) OceanDataTools.org 2025 ''' -import os -import sys import json -import logging +import os import requests +import sys + from datetime import datetime, timedelta from urllib.parse import quote, urlparse from urllib3.exceptions import NewConnectionError @@ -42,11 +42,16 @@ def __init__(self, aux_data_config, url=None): super().__init__(aux_data_config) self.url = url or CORIOLIX_URL - @staticmethod - def _build_query_range(ts): + def _build_query_range(self, ts): ''' Builds the temporal range for the CORIOLIX query based on the provided timestamp (ts). + + Args: + ts (str): Timestamp in ISO 8601 format (YYYY-MM-DDTHH:MM:SS.fffZ) + + Returns: + str or None: Query range string ''' try: start_ts = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ") - timedelta(minutes=1) @@ -55,7 +60,7 @@ def _build_query_range(ts): f'&date_before={quote(ts)}') except ValueError as exc: - logging.debug(str(exc)) + self.logger.debug(str(exc)) return None def _build_query_urls(self, ts): @@ -70,7 +75,7 @@ def _build_query_urls(self, ts): for measurement in self._query_measurements: query_urls.append(f'{self.url}/api/{measurement}/?format=json&{query_range}') - logging.debug("Query: %s", query_urls[-1]) + self.logger.debug("Query: %s", query_urls[-1]) return query_urls @@ -79,34 +84,35 @@ def open_connections(self): Open any necessary connections to external data sources. For CORIOLIX, no persistent connection is needed. ''' - pass - + def close_connections(self): ''' Close any open connections to external data sources. For CORIOLIX, no persistent connection is needed. ''' - pass - + def build_aux_data_record(self, event): ''' Build the aux_data record for the given event. ''' - logging.debug("building query") + self.logger.debug("building query") query_urls = self._build_query_urls(event['ts']) query_results = {} for url in query_urls: - logging.debug("Query URL: %s", url) + self.logger.debug("Query URL: %s", url) measurement = os.path.basename(urlparse(url).path.strip('/')) # run the query against the CORIOLIX API try: response = requests.get(url, timeout=2) if response.status_code != 200: - logging.error("Failed to retrieve data. Status code: %s", response.status_code) + self.logger.error( + "Failed to retrieve data. Status code: %s", + response.status_code + ) response_obj = json.loads(response.text) if isinstance(response_obj, dict): @@ -121,13 +127,13 @@ def build_aux_data_record(self, event): } except NewConnectionError: - logging.error("CORIOLIX connection error, verify URL: %s", self.url) + self.logger.error("CORIOLIX connection error, verify URL: %s", self.url) except json.decoder.JSONDecodeError: - logging.error("Unable to decode response from URL: %s", url) - logging.debug(response) + self.logger.error("Unable to decode response from URL: %s", url) + self.logger.debug(response) except KeyError: - logging.error("Something went wrong processing the API response") + self.logger.error("Something went wrong processing the API response") aux_data_record = self._build_aux_data_dict(event['id'], query_results) - return aux_data_record \ No newline at end of file + return aux_data_record diff --git a/misc/framegrab_aux/aux_data_record_builder_framegrab_http.py b/misc/framegrab_aux/aux_data_record_builder_framegrab_http.py index 9e79736..ef7b075 100644 --- a/misc/framegrab_aux/aux_data_record_builder_framegrab_http.py +++ b/misc/framegrab_aux/aux_data_record_builder_framegrab_http.py @@ -1,51 +1,40 @@ #!/usr/bin/env python3 ''' -FILE: aux_data_record_builder_test_images.py +FILE: aux_data_record_builder_framegrab_http.py -DESCRIPTION: This script generates test images and uses them to build a sealog aux_data record. +DESCRIPTION: This script builds a sealog aux_data record by fetching frame grab images from HTTP source. ''' import os -import sys -import logging -from datetime import datetime, timedelta import requests import shutil +import sys +from datetime import datetime, timedelta from os.path import dirname, realpath sys.path.append(dirname(dirname(realpath(__file__)))) from misc.base_aux_data_record_builder import AuxDataRecordBuilder from misc.framegrab_aux.settings import DEST_DIR, SOURCES, THRESHOLD -class FramegrabHTTPAuxDataRecordBuilder(AuxDataRecordBuilder): # pylint: disable=too-few-public-methods + +class FramegrabHTTPAuxDataRecordBuilder(AuxDataRecordBuilder): ''' Class that handles generating test images and using the resulting data to build a sealog aux_data record. ''' - def __init__(self, - aux_data_config): - super().__init__(aux_data_config) - - @staticmethod - def _build_query_range(ts): - # see, this is making me feel like this shouldn't be in the base class - raise NotImplementedError("No query for framegrab") - def open_connections(self): ''' Open any necessary connections to external data sources. Must be implemented by subclasses. ''' - pass - + def close_connections(self): ''' Close any open connections to external data sources. Must be implemented by subclasses. ''' - pass - + def _build_destination_filepath(self, str_timestamp, filename_prefix, filename_suffix): timestamp = datetime.strptime( str_timestamp, @@ -61,7 +50,7 @@ def _build_destination_filepath(self, str_timestamp, filename_prefix, filename_s DEST_DIR, filename_prefix + filename_middle + filename_suffix ) - + return destination_filepath def build_aux_data_record(self, event): @@ -72,7 +61,7 @@ def build_aux_data_record(self, event): event['ts'], '%Y-%m-%dT%H:%M:%S.%fZ' ) < datetime.utcnow()-timedelta(seconds=THRESHOLD): - logging.debug("Skipping because event ts is older than thresold") + self.logger.debug("Skipping because event ts is older than thresold") return None aux_data_record = { @@ -89,7 +78,7 @@ def build_aux_data_record(self, event): source['filename_suffix'] ) - logging.debug("dst: %s", dst) + self.logger.debug("dst: %s", dst) try: res = requests.get( @@ -99,23 +88,23 @@ def build_aux_data_record(self, event): ) if res.status_code != 200: - logging.error( + self.logger.error( "Unable to retrieve image from: %s", source['source_url'] + source['source_filename'] ) continue except requests.exceptions.RequestException as exc: - logging.error("Unable to retrieve image from remote server") - logging.error(exc) + self.logger.error("Unable to retrieve image from remote server") + self.logger.error(exc) try: with open(dst, 'wb') as f: shutil.copyfileobj(res.raw, f) except shutil.Error as exc: - logging.error("Unable to save image to server") - logging.error(exc) + self.logger.error("Unable to save image to server") + self.logger.error(exc) aux_data_record['data_array'].append( {'data_name': "camera_name", 'data_value': source['source_name']} diff --git a/misc/framegrab_aux/aux_data_record_builder_framegrab_local.py b/misc/framegrab_aux/aux_data_record_builder_framegrab_local.py index 2590df5..f3a0bcc 100644 --- a/misc/framegrab_aux/aux_data_record_builder_framegrab_local.py +++ b/misc/framegrab_aux/aux_data_record_builder_framegrab_local.py @@ -1,50 +1,39 @@ #!/usr/bin/env python3 ''' -FILE: aux_data_record_builder_test_images.py +FILE: aux_data_record_builder_framegrab_local.py -DESCRIPTION: This script generates test images and uses them to build a sealog aux_data record. +DESCRIPTION: This script builds a sealog aux_data record by copying frame grab images from local directory. ''' import os -import sys -import logging -from datetime import datetime, timedelta import shutil +import sys +from datetime import datetime, timedelta from os.path import dirname, realpath sys.path.append(dirname(dirname(realpath(__file__)))) from misc.base_aux_data_record_builder import AuxDataRecordBuilder from misc.framegrab_aux.settings import SOURCE_DIR, DEST_DIR, SOURCES, THRESHOLD -class FramegrabSCPAuxDataRecordBuilder(AuxDataRecordBuilder): # pylint: disable=too-few-public-methods + +class FramegrabLocalAuxDataRecordBuilder(AuxDataRecordBuilder): ''' Class that handles generating test images and using the resulting data to build a sealog aux_data record. ''' - def __init__(self, - aux_data_config): - super().__init__(aux_data_config) - - @staticmethod - def _build_query_range(ts): - # see, this is making me feel like this shouldn't be in the base class - raise NotImplementedError("No query for framegrab") - def open_connections(self): ''' Open any necessary connections to external data sources. Must be implemented by subclasses. ''' - pass - + def close_connections(self): ''' Close any open connections to external data sources. Must be implemented by subclasses. ''' - pass - + def _build_destination_filepath(self, str_timestamp, filename_prefix, filename_suffix): timestamp = datetime.strptime( str_timestamp, @@ -60,7 +49,7 @@ def _build_destination_filepath(self, str_timestamp, filename_prefix, filename_s DEST_DIR, filename_prefix + filename_middle + filename_suffix ) - + return destination_filepath def build_aux_data_record(self, event): @@ -71,7 +60,7 @@ def build_aux_data_record(self, event): event['ts'], '%Y-%m-%dT%H:%M:%S.%fZ' ) < datetime.utcnow()-timedelta(seconds=THRESHOLD): - logging.debug("Skipping because event ts is older than thresold") + self.logger.debug("Skipping because event ts is older than thresold") return None aux_data_record = { @@ -88,16 +77,16 @@ def build_aux_data_record(self, event): source['filename_suffix'] ) - logging.debug("dst: %s", dst) + self.logger.debug("dst: %s", dst) latest_file = os.path.join(SOURCE_DIR, source['source_filename']) src = os.path.join(SOURCE_DIR, latest_file) try: - shutil.copyfile(src,dst) + shutil.copyfile(src, dst) except shutil.Error as exc: - logging.error("Unable to save image to server") - logging.error(exc) + self.logger.error("Unable to save image to server") + self.logger.error(exc) aux_data_record['data_array'].append( {'data_name': "camera_name", 'data_value': source['source_name']} diff --git a/misc/framegrab_aux/aux_data_record_builder_framegrab_scp.py b/misc/framegrab_aux/aux_data_record_builder_framegrab_scp.py index 98cbc4e..e357437 100644 --- a/misc/framegrab_aux/aux_data_record_builder_framegrab_scp.py +++ b/misc/framegrab_aux/aux_data_record_builder_framegrab_scp.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 ''' -FILE: aux_data_record_builder_test_images.py +FILE: aux_data_record_builder_framegrab_scp.py -DESCRIPTION: This script generates test images and uses them to build a sealog aux_data record. +DESCRIPTION: This script builds a sealog aux_data record by transferring frame grab images via SCP. ''' import os import sys -import logging + from datetime import datetime, timedelta try: @@ -23,7 +23,8 @@ from misc.framegrab_aux.settings import SOURCE_DIR, DEST_DIR, SOURCES, THRESHOLD, \ HOST, USER, KEY_FILE, PORT -class FramegrabSCPAuxDataRecordBuilder(AuxDataRecordBuilder): # pylint: disable=too-few-public-methods + +class FramegrabSCPAuxDataRecordBuilder(AuxDataRecordBuilder): ''' Class that handles generating test images and using the resulting data to build a sealog aux_data record. @@ -40,15 +41,10 @@ def __init__(self, self._user = USER self._key = RSAKey.from_private_key_file(KEY_FILE) self._port = PORT - + self._scp_transport = None self._sftp_client = None - @staticmethod - def _build_query_range(ts): - # see, this is making me feel like this shouldn't be in the base class - raise NotImplementedError("No query for framegrab") - def open_connections(self): ''' Open any necessary connections to external data sources. @@ -58,10 +54,9 @@ def open_connections(self): self._scp_transport.connect(username=self._user, pkey=self._key) self._scp_transport.set_keepalive(30) # since opening connection higher up, set keepalive to prevent going stale - logging.info("Opening SFTP connection") + self.logger.info("Opening SFTP connection") self._sftp_client = SFTPClient.from_transport(self._scp_transport) - pass - + def close_connections(self): ''' Close any open connections to external data sources. @@ -70,9 +65,8 @@ def close_connections(self): try: self._scp_transport.close() except Exception as err: # pylint: disable=W0718 - logging.error("Error closing SCP Transport: %s", str(err)) - pass - + self.logger.error("Error closing SCP Transport: %s", str(err)) + def _build_destination_filepath(self, str_timestamp, filename_prefix, filename_suffix): timestamp = datetime.strptime( str_timestamp, @@ -88,7 +82,7 @@ def _build_destination_filepath(self, str_timestamp, filename_prefix, filename_s DEST_DIR, filename_prefix + filename_middle + filename_suffix ) - + return destination_filepath def build_aux_data_record(self, event): @@ -99,7 +93,7 @@ def build_aux_data_record(self, event): event['ts'], '%Y-%m-%dT%H:%M:%S.%fZ' ) < datetime.utcnow()-timedelta(seconds=THRESHOLD): - logging.debug("Skipping because event ts is older than thresold") + self.logger.debug("Skipping because event ts is older than thresold") return None aux_data_record = { @@ -116,18 +110,18 @@ def build_aux_data_record(self, event): source['filename_suffix'] ) - logging.debug("dst: %s", dst) + self.logger.debug("dst: %s", dst) try: latest_file = os.path.join(SOURCE_DIR, source['source_filename']) src = os.path.join(SOURCE_DIR, latest_file) self._sftp_client.put(src, dst) except SFTPError as exc: - logging.error("Unable to copy image to server") - logging.error(exc) + self.logger.error("Unable to copy image to server") + self.logger.error(exc) except OSError as exc: - logging.error("Unable to copy image to server") - logging.error(exc) + self.logger.error("Unable to copy image to server") + self.logger.error(exc) aux_data_record['data_array'].append( {'data_name': "camera_name", 'data_value': source['source_name']} diff --git a/misc/framegrab_aux/settings.py b/misc/framegrab_aux/settings.py.dist similarity index 99% rename from misc/framegrab_aux/settings.py rename to misc/framegrab_aux/settings.py.dist index 2a71f8b..df165be 100644 --- a/misc/framegrab_aux/settings.py +++ b/misc/framegrab_aux/settings.py.dist @@ -41,4 +41,4 @@ 'filename_prefix': '', 'filename_suffix': '.jpg' } -] \ No newline at end of file +] diff --git a/misc/influx_sealog/aux_data_record_builder.py b/misc/influx_sealog/aux_data_record_builder.py index c1fdeb9..b609d74 100644 --- a/misc/influx_sealog/aux_data_record_builder.py +++ b/misc/influx_sealog/aux_data_record_builder.py @@ -17,10 +17,10 @@ Copyright (C) OceanDataTools.org 2025 ''' import sys -import logging + from datetime import datetime, timedelta -from urllib3.exceptions import NewConnectionError from influxdb_client.rest import ApiException +from urllib3.exceptions import NewConnectionError from os.path import dirname, realpath sys.path.append(dirname(dirname(dirname(realpath(__file__))))) @@ -48,17 +48,22 @@ def __init__(self, influxdb_client, aux_data_config, influxdb_bucket=INFLUXDB_BU if 'query_bucket' in aux_data_config else influxdb_bucket ) - @staticmethod - def _build_query_range(ts): + def _build_query_range(self, ts): ''' Builds the temporal range for the influxDB query based on the provided timestamp (ts). + + Args: + ts (str): Timestamp in ISO 8601 format (YYYY-MM-DDTHH:MM:SS.fffZ) + + Returns: + str or None: Query range string ''' try: start_ts = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ") - timedelta(minutes=1) return f'start: {start_ts.strftime("%Y-%m-%dT%H:%M:%S.%fZ")}, stop: {ts}' except ValueError as exc: - logging.debug(str(exc)) + self.logger.debug(str(exc)) return None def _build_query(self, ts): @@ -87,52 +92,50 @@ def _build_query(self, ts): query += '|> sort(columns: ["_time"], desc: true)\n' query += '|> limit(n:1)' - logging.debug("Query: %s", query) + self.logger.debug("Query: %s", query) return query - + def open_connections(self): ''' Open any necessary connections to external data sources. For Influx, no persistent connection is needed. ''' - pass - + def close_connections(self): ''' Close any open connections to external data sources. For Influx, no persistent connection is needed. ''' - pass def build_aux_data_record(self, event): ''' Build the aux_data record for the given event. ''' - logging.debug("building query") + self.logger.debug("building query") query = self._build_query(event['ts']) - logging.debug("Query: %s", query) + self.logger.debug("Query: %s", query) # run the query against the influxDB try: query_result = self._influxdb_client.query(query=query) except NewConnectionError: - logging.error("InfluxDB connection error, verify URL: %s", INFLUXDB_URL) + self.logger.error("InfluxDB connection error, verify URL: %s", INFLUXDB_URL) except ApiException as exc: _, value, _ = sys.exc_info() if str(value).startswith("(400)"): - logging.error("InfluxDB API error, verify org: %s", INFLUXDB_ORG) + self.logger.error("InfluxDB API error, verify org: %s", INFLUXDB_ORG) elif str(value).startswith("(401)"): - logging.error("InfluxDB API error, verify token: %s", INFLUXDB_AUTH_TOKEN) + self.logger.error("InfluxDB API error, verify token: %s", INFLUXDB_AUTH_TOKEN) elif str(value).startswith("(404)"): - logging.error("InfluxDB API error, verify bucket: %s", self._influxdb_bucket) + self.logger.error("InfluxDB API error, verify bucket: %s", self._influxdb_bucket) else: - logging.error("Error with query:") - logging.error(query.replace("|>", '\n')) - logging.error(str(exc)) + self.logger.error("Error with query:") + self.logger.error(query.replace("|>", '\n')) + self.logger.error(str(exc)) raise exc else: # Parse InfluxDB result into a dictionary format @@ -145,4 +148,4 @@ def build_aux_data_record(self, event): return aux_data_record - return None \ No newline at end of file + return None diff --git a/misc/influx_sealog/aux_data_record_builder_v1.py b/misc/influx_sealog/aux_data_record_builder_v1.py index c1f8271..1202f9e 100644 --- a/misc/influx_sealog/aux_data_record_builder_v1.py +++ b/misc/influx_sealog/aux_data_record_builder_v1.py @@ -17,10 +17,10 @@ Copyright (C) OceanDataTools.org 2025 ''' import sys -import logging + from datetime import datetime, timedelta -from urllib3.exceptions import NewConnectionError from influxdb.exceptions import InfluxDBClientError, InfluxDBServerError +from urllib3.exceptions import NewConnectionError from os.path import dirname, realpath sys.path.append(dirname(dirname(dirname(realpath(__file__))))) @@ -34,7 +34,7 @@ ) -class SealogInfluxV1AuxDataRecordBuilder(AuxDataRecordBuilder): # pylint: disable=too-few-public-methods # noqa: E501 +class SealogInfluxV1AuxDataRecordBuilder(AuxDataRecordBuilder): ''' Class that handles the construction of an influxDB query and using the resulting data to build a sealog aux_data record. @@ -49,15 +49,20 @@ def __init__(self, influxdb_client, aux_data_config, influxdb_bucket=INFLUXDB_BU if 'query_bucket' in aux_data_config else influxdb_bucket ) - @staticmethod - def _build_query_range(ts): + def _build_query_range(self, ts): ''' Builds the temporal range for the influxDB query based on the provided timestamp (ts). + + Args: + ts (str): Timestamp in ISO 8601 format (YYYY-MM-DDTHH:MM:SS.fffZ) + + Returns: + str or None: Query range string ''' str_start_ts = datetime.strftime( - datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ") - timedelta(seconds=20), - "%Y-%m-%dT%H:%M:%SZ" + datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ") - timedelta(seconds=20), + "%Y-%m-%dT%H:%M:%SZ" ) # Note: in the new code, you subtract a whole minute. Did we choose 20 s on purpose? @@ -86,7 +91,7 @@ def _build_query(self, ts): WHERE {str_filters} ORDER BY DESC LIMIT 1''' - logging.debug("Query: %s", query) + self.logger.debug("Query: %s", query) return query @@ -95,44 +100,42 @@ def open_connections(self): Open any necessary connections to external data sources. For Influx, no persistent connection is needed. ''' - pass - + def close_connections(self): ''' Close any open connections to external data sources. For Influx, no persistent connection is needed. ''' - pass - + def build_aux_data_record(self, event): ''' Build the aux_data record for the given event. ''' - logging.debug("building query") + self.logger.debug("building query") query = self._build_query(event['ts']) - logging.debug("Query: %s", query) + self.logger.debug("Query: %s", query) # run the query against the influxDB try: query_result = self._influxdb_client.query(query=query) except NewConnectionError: - logging.error("InfluxDB connection error, verify URL: %s", INFLUXDB_URL) + self.logger.error("InfluxDB connection error, verify URL: %s", INFLUXDB_URL) except (InfluxDBClientError, InfluxDBServerError) as exc: _, value, _ = sys.exc_info() if str(value).startswith("(400)"): - logging.error("InfluxDB API error, verify org: %s", INFLUXDB_ORG) + self.logger.error("InfluxDB API error, verify org: %s", INFLUXDB_ORG) elif str(value).startswith("(401)"): - logging.error("InfluxDB API error, verify token: %s", INFLUXDB_AUTH_TOKEN) + self.logger.error("InfluxDB API error, verify token: %s", INFLUXDB_AUTH_TOKEN) elif str(value).startswith("(404)"): - logging.error("InfluxDB API error, verify bucket: %s", self._influxdb_bucket) + self.logger.error("InfluxDB API error, verify bucket: %s", self._influxdb_bucket) else: - logging.error("Error with query:") - logging.error(query.replace("|>", '\n')) - logging.error(str(exc)) + self.logger.error("Error with query:") + self.logger.error(query.replace("|>", '\n')) + self.logger.error(str(exc)) raise exc else: # Parse InfluxDB result into a dictionary format diff --git a/misc/influx_sealog/settings.py.dist b/misc/influx_sealog/settings.py.dist index a629e05..054e9e2 100644 --- a/misc/influx_sealog/settings.py.dist +++ b/misc/influx_sealog/settings.py.dist @@ -27,4 +27,4 @@ INFLUXDB_VERIFY_SSL = False INFLUX_HOST = '10.1.90.40' INFLUX_PORT = 8086 INFLUX_USER = '' -INFLUX_PASS = '' \ No newline at end of file +INFLUX_PASS = '' diff --git a/misc/python_sealog/event_aux_data.py b/misc/python_sealog/event_aux_data.py index be3782e..1b62fa6 100644 --- a/misc/python_sealog/event_aux_data.py +++ b/misc/python_sealog/event_aux_data.py @@ -104,21 +104,6 @@ def get_event_aux_data_by_lowering(lowering_uid, datasource=None, limit=0, logging.error(str(exc)) raise exc -def get_event_aux_data_by_event(aux_data_uid, - api_server_url=API_SERVER_URL, - headers=HEADERS): - ''' - Get the aux_data records for an event - ''' - try: - url = f'{api_server_url}{EVENT_AUX_DATA_API_PATH}' - req = requests.post(url, headers=headers, data=json.dumps(payload), timeout=(2, None)) - logging.debug(req.text) - - except requests.exceptions.RequestException as exc: - logging.error(str(exc)) - raise exc - def create_event_aux_data(payload, api_server_url=API_SERVER_URL, headers=HEADERS): ''' diff --git a/misc/sealog_aux_data_manager_coriolix.py.dist b/misc/sealog_aux_data_manager_coriolix.py.dist index 557dabc..d6ab523 100644 --- a/misc/sealog_aux_data_manager_coriolix.py.dist +++ b/misc/sealog_aux_data_manager_coriolix.py.dist @@ -1,8 +1,8 @@ #!/usr/bin/env python3 ''' -FILE: sealog_aux_data_inserter_coriolix.py +FILE: sealog_aux_data_manager_coriolix.py -DESCRIPTION: This service listens for new events submitted to Sealog, create +DESCRIPTION: This service listens for new events submitted to Sealog, creates aux_data records containing the specified real-time data and associates the aux data records with the newly created event. @@ -105,7 +105,7 @@ if __name__ == '__main__': def builder_factory(aux_data_config): '''Build an AuxDataRecordBuilder''' return SealogCORIOLIXAuxDataRecordBuilder(aux_data_config) - + def cleaner_factory(aux_data_config): '''Build an AuxDataFileCleaner''' return DoNothingAuxDataFileCleaner(aux_data_config) diff --git a/misc/sealog_aux_data_manager_framegrab.py.dist b/misc/sealog_aux_data_manager_framegrab.py.dist index 6df56de..3240dde 100644 --- a/misc/sealog_aux_data_manager_framegrab.py.dist +++ b/misc/sealog_aux_data_manager_framegrab.py.dist @@ -45,7 +45,7 @@ INLINE_CONFIG = ''' EXCLUDE_SET = () # needs to be unique for all currently active dataInserter scripts. -CLIENT_WSID = f'aux_data_inserter_vehicleRealtimeFramegrabberData' +CLIENT_WSID = 'aux_data_inserter_vehicleRealtimeFramegrabberData' # ------------------------------------------------------------------------------------- diff --git a/misc/sealog_aux_data_manager_influx.py.dist b/misc/sealog_aux_data_manager_influx.py.dist index cf8661a..c7674fe 100644 --- a/misc/sealog_aux_data_manager_influx.py.dist +++ b/misc/sealog_aux_data_manager_influx.py.dist @@ -106,7 +106,7 @@ if __name__ == '__main__': def builder_factory(aux_data_config): '''Build an AuxDataRecordBuilder''' return SealogInfluxAuxDataRecordBuilder(client, aux_data_config) - + def cleaner_factory(aux_data_config): '''Build an AuxDataFileCleaner''' return DoNothingAuxDataFileCleaner(aux_data_config) diff --git a/misc/sealog_aux_data_manager_influx_v1.py.dist b/misc/sealog_aux_data_manager_influx_v1.py.dist index c0f1c8b..bee105c 100644 --- a/misc/sealog_aux_data_manager_influx_v1.py.dist +++ b/misc/sealog_aux_data_manager_influx_v1.py.dist @@ -1,6 +1,6 @@ #!/usr/bin/env python3 ''' -FILE: sealog_aux_data_manager_influx.py +FILE: sealog_aux_data_manager_influx_v1.py DESCRIPTION: This service listens for new events submitted to Sealog, create aux_data records containing the specified real-time data and diff --git a/routes/api/v1/events.js b/routes/api/v1/events.js index 9a18bdc..159e065 100644 --- a/routes/api/v1/events.js +++ b/routes/api/v1/events.js @@ -50,6 +50,7 @@ const _renameAndClearFields = (doc) => { }; const _deleteEventsWithAuxData = async (db, server, query = {}, limit = 0, offset = 0, sort = { ts: 1 }) => { + // Find the events const eventsToDelete = await db.collection(eventsTable) .find(query) @@ -70,10 +71,12 @@ const _deleteEventsWithAuxData = async (db, server, query = {}, limit = 0, offse .toArray(); const eventIDToAuxData = allAuxData.reduce((dict, auxData) => { + const eventId = auxData.event_id.toString(); if (!dict[eventId]) { dict[eventId] = []; } + dict[eventId].push(auxData); return dict; }, {}); @@ -87,7 +90,7 @@ const _deleteEventsWithAuxData = async (db, server, query = {}, limit = 0, offse // Delete event records const results = await db.collection(eventsTable).deleteMany({ _id: { $in: eventIDs } }); - + return { deletedCount: results.deletedCount }; }; @@ -1307,4 +1310,4 @@ exports.plugin = { } }); } -}; \ No newline at end of file +}; From c361480e83df44f94e928940a0d903f2e654b9ff Mon Sep 17 00:00:00 2001 From: ljones Date: Thu, 19 Mar 2026 10:11:13 -0700 Subject: [PATCH 17/17] bugfix: add cleaner data_source public property to avoid crash when there's no aux data while debugging --- misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py b/misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py index f44a173..41b7113 100644 --- a/misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py +++ b/misc/aux_data_file_cleaners/base_aux_data_file_cleaner.py @@ -90,3 +90,10 @@ def clean_aux_data_record(self, event, dry_run): Returns: dict or None: Aux data record or None if no data available ''' + + @property + def data_source(self): + ''' + Getter method for the data_source property + ''' + return self._data_source