diff --git a/container_templates/matlab/container_example/.gitignore b/container_templates/matlab/container_example/.gitignore new file mode 100644 index 0000000000..d2d9075c73 --- /dev/null +++ b/container_templates/matlab/container_example/.gitignore @@ -0,0 +1,2 @@ +.idea/ +venv diff --git a/container_templates/matlab/container_example/Dockerfile b/container_templates/matlab/container_example/Dockerfile new file mode 100644 index 0000000000..7af4cf3fd7 --- /dev/null +++ b/container_templates/matlab/container_example/Dockerfile @@ -0,0 +1,49 @@ +# Modified from https://github.com/mathworks-ref-arch/matlab-dockerfile. + +# To specify which MATLAB release to install in the container, edit the value of the MATLAB_RELEASE argument. +ARG MATLAB_RELEASE=r2021b + +# When you start the build stage, this Dockerfile by default uses the Ubuntu-based matlab-deps image. +# To check the available matlab-deps images, see: https://hub.docker.com/r/mathworks/matlab-deps +FROM mathworks/matlab-deps:${MATLAB_RELEASE} + +# Declare the global argument to use at the current build stage +ARG MATLAB_RELEASE + +# Install mpm dependencies +RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \ + apt-get install --no-install-recommends --yes \ + wget \ + unzip \ + ca-certificates && \ + apt-get clean && apt-get autoremove + +# Run mpm to install MATLAB in the target location and delete the mpm installation afterwards. +# If mpm fails to install successfully then output the logfile to the terminal, otherwise cleanup. +RUN wget -q https://www.mathworks.com/mpm/glnxa64/mpm && \ + chmod +x mpm && \ + ./mpm install \ + --release=${MATLAB_RELEASE} \ + --destination=/opt/matlab \ + --products MATLAB || \ + (echo "MPM Installation Failure. See below for more information:" && cat /tmp/mathworks_root.log && false) && \ + rm -f mpm /tmp/mathworks_root.log && \ + ln -s /opt/matlab/bin/matlab /usr/local/bin/matlab + +# Add "matlab" user and grant sudo permission. +RUN adduser --shell /bin/bash --disabled-password --gecos "" matlab && \ + echo "matlab ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/matlab && \ + chmod 0440 /etc/sudoers.d/matlab + +ARG LICENSE_SERVER + +# Specify the host and port of the machine that serves the network licenses +ENV MLM_LICENSE_FILE=$LICENSE_SERVER + +# Set user and work directory +WORKDIR /home/matlab + +# copy the content of the local directory to the working directory +COPY src/ . + +ENTRYPOINT ["matlab", "-r"] diff --git a/container_templates/matlab/container_example/README.md b/container_templates/matlab/container_example/README.md new file mode 100644 index 0000000000..a45b569c25 --- /dev/null +++ b/container_templates/matlab/container_example/README.md @@ -0,0 +1,11 @@ +# Container Example Code + +This the basic code for creating a Docker container for the Level 2 pipeline. +For questions contact: laura.sandoval@lasp.colorado.edu or contact +Laura Sandoval via the libera_l2 Slack channel. + +## Setup +[Here](docs/setup.md) + +## Building and running a Docker image +[Here](docs/docker.md) diff --git a/container_templates/matlab/container_example/container_example_data/dropbox/input_manifest_20220923t000000.json b/container_templates/matlab/container_example/container_example_data/dropbox/input_manifest_20220923t000000.json new file mode 100644 index 0000000000..9b152cf3b0 --- /dev/null +++ b/container_templates/matlab/container_example/container_example_data/dropbox/input_manifest_20220923t000000.json @@ -0,0 +1,8 @@ +{ + "manifest_type": "INPUT", + "files": [ + {"filename": "/opt/data/source_data/filename_example_1.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"}, + {"filename": "/opt/data/source_data/filename_example_2.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"} + ], + "configuration": null +} diff --git a/container_templates/matlab/container_example/container_example_data/dropbox/input_manifest_20220923t000000_aws.json b/container_templates/matlab/container_example/container_example_data/dropbox/input_manifest_20220923t000000_aws.json new file mode 100644 index 0000000000..634c7e8b92 --- /dev/null +++ b/container_templates/matlab/container_example/container_example_data/dropbox/input_manifest_20220923t000000_aws.json @@ -0,0 +1,8 @@ +{ + "manifest_type": "INPUT", + "files": [ + {"filename": "s3://l0-ingest-dropbox-liberasdplcs-dev/source_data/filename_example_1.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"}, + {"filename": "s3://l0-ingest-dropbox-liberasdplcs-dev/source_data/filename_example_2.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"} + ], + "configuration": null +} diff --git a/container_templates/matlab/container_example/container_example_data/dropbox/input_manifest_20220923t000000_local.json b/container_templates/matlab/container_example/container_example_data/dropbox/input_manifest_20220923t000000_local.json new file mode 100644 index 0000000000..a60f693dec --- /dev/null +++ b/container_templates/matlab/container_example/container_example_data/dropbox/input_manifest_20220923t000000_local.json @@ -0,0 +1,8 @@ +{ + "manifest_type": "INPUT", + "files": [ + {"filename": "../container_example_data/source_data/filename_example_1.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"}, + {"filename": "../container_example_data/source_data/filename_example_2.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"} + ], + "configuration": null +} diff --git a/container_templates/matlab/container_example/container_example_data/source_data/filename_example_1.h5 b/container_templates/matlab/container_example/container_example_data/source_data/filename_example_1.h5 new file mode 100644 index 0000000000..e805e3c817 Binary files /dev/null and b/container_templates/matlab/container_example/container_example_data/source_data/filename_example_1.h5 differ diff --git a/container_templates/matlab/container_example/container_example_data/source_data/filename_example_2.h5 b/container_templates/matlab/container_example/container_example_data/source_data/filename_example_2.h5 new file mode 100644 index 0000000000..e805e3c817 Binary files /dev/null and b/container_templates/matlab/container_example/container_example_data/source_data/filename_example_2.h5 differ diff --git a/container_templates/matlab/container_example/docs/docker.md b/container_templates/matlab/container_example/docs/docker.md new file mode 100644 index 0000000000..d451eb1269 --- /dev/null +++ b/container_templates/matlab/container_example/docs/docker.md @@ -0,0 +1,38 @@ +# Building and running a docker image locally + +To build the image run the following command from the Dockerfile directory. +Note: for non-LASP users you must retrieve your MATLAB licensing information +from your own institute. More information is available here: +https://github.com/mathworks-ref-arch/matlab-dockerfile + +```shell +docker build -t matlab-image --rm . --build-arg \ +LICENSE_SERVER=27000@lasp-lmgr.colorado.edu +``` + +Now we can run our image using bind mounting. + +# For local directory + +```shell +docker run --rm -it -e PROCESSING_DROPBOX=/opt/data/dropbox/ \ +--volume="$(pwd)/container_example_data:/opt/data" matlab-image:latest \ +"algorithm_example /opt/data/dropbox/input_manifest_20220923t000000.json" +``` + +# Building and running a docker image in AWS + +You will be given AWS console logins in the future. Don't worry about this for now. + +In order to push a Docker Image to the ECR, navigate to the AWS Console to +retrieve the AWS ECR registry URI: + +https://us-west-2.console.aws.amazon.com/ecr/repositories?region=us-west-2 + +Follow the instructions in the link to push to the AWS ECR. + +https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html + +The ECR can be browsed through the AWS Console: + +https://us-west-2.console.aws.amazon.com/ecr/repositories?region=us-west-2 diff --git a/container_templates/matlab/container_example/docs/setup.md b/container_templates/matlab/container_example/docs/setup.md new file mode 100644 index 0000000000..cfb5a492ad --- /dev/null +++ b/container_templates/matlab/container_example/docs/setup.md @@ -0,0 +1,3 @@ +# Environment + +Ensure MATLAB >=r2021b is installed for compatibility. diff --git a/container_templates/matlab/container_example/src/algorithm_example.m b/container_templates/matlab/container_example/src/algorithm_example.m new file mode 100644 index 0000000000..0b5e5eff91 --- /dev/null +++ b/container_templates/matlab/container_example/src/algorithm_example.m @@ -0,0 +1,137 @@ +% algorithm_example.md - template to read/write to local directory +% or S3 bucket +% --------------------------------------------------------------------- +% +% Example: +% +% in terminal - +% matlab -nodisplay -r "cd src; algorithm_example " +% +% in MATLAB console - +% setenv("PROCESSING_DROPBOX",""); +% algorithm_example() +% +% Notes: +% - If running from the terminal add +% export PROCESSING_DROPBOX="" to your bash profile +% - may either be the path to the local directory e.g. +% '../container_example_data/dropbox/input_manifest_20220923t000000_local.json' +% or the path to the s3 bucket (e.g. s3://bucketname/). +% - For instructions on using this code with Docker refer to +% docker.md +% +% Authors: Laura Sandoval, Matt Watwood, Gavin Medley +% +% External libraries used: +% +% Stefan Stoll (2022). MD5 signature of a file +% (https://www.mathworks.com/matlabcentral/fileexchange/5498-md5-signature-of-a-file), +% MATLAB Central File Exchange. Retrieved November 9, 2022. +% +% Matthew Spaethe (2022). Matlab logging facility +% (https://www.mathworks.com/matlabcentral/fileexchange/42078-matlab-logging-facility), +% MATLAB Central File Exchange. Retrieved November 15, 2022. +% --------------------------------------------------------------------- + +function []=algorithm_example(manifest_in, varargin) +% Requires 1 input and has optional input. + +addpath('logging') +import logging.* + +% get the global LogManager +LogManager.getLogManager(); + +% add a logger instance to this script +logger = Logger.getLogger('AlgorithmExample'); +logger.setLevel( Level.ALL ); + +defaultInt = 2; + +logger.info(strcat('Command executed in: ', mfilename('filename'),'.m')); +logger.info('Parsing CLI arguments.'); + +p = inputParser; +addRequired(p,'manifest_in',@ischar); +addOptional(p,'multiplier',defaultInt,@isnumeric); +parse(p,manifest_in,varargin{:}); + +logger.info(strcat('Manifest file to read: ', p.Results.manifest_in)); +logger.info(strcat('Additional options passed are multiplier = ', ... + num2str(p.Results.multiplier))); + +% only want 1 optional input at most +numvarargs = length(varargin); +if numvarargs > 1 + error('my_script:TooManyInputs', ... + 'requires at most 1 optional inputs'); +end + +processing_dropbox = getenv('PROCESSING_DROPBOX'); + +str = fileread(p.Results.manifest_in); +json_content = jsondecode(str); + +logger.info(strcat('Manifest type is ', json_content.manifest_type)); +logger.info(strcat('Manifest contains ', json_content.files.filename)); + +% Get the data from each file and put data into some format that will be +% used in the algorithm +for n = 1 : length(json_content.files) + + checksum = json_content.files(n).checksum; + filename = json_content.files(n).filename; + + assert(isequal(checksum,md5(filename)), ... + 'algorithm_example:checksumError','Checksums do not match!') + logger.info(strcat('Checksum matches for ', filename)); + + data_in = h5read(filename,'/HDFEOS/SWATHS/Swath1/DataField/Temperature'); + logger.info(strcat('Found input data in HDF5 file:',mat2str(data_in))) + +end + +%fake data product to write as output +data_out = data_in * p.Results.multiplier; + +%Create hdf5 data +output_filepath = append(processing_dropbox, 'example_output.h5'); +h5create(output_filepath,'/data/array1',[height(data_out) width(data_out)]); + +logger.info(strcat('Writing output file: ', output_filepath)) + +h5write(output_filepath, '/data/array1', data_out); +h5writeatt(output_filepath,'/','someattr','hello, world'); + +% Need this to create empty group; if you are creating a group +% that contains data, then use h5create and h5write +plist = 'H5P_DEFAULT'; +fid = H5F.open(output_filepath,'H5F_ACC_RDWR',plist); +gid = H5G.create(fid,'new_group',plist,plist,plist); +H5G.close(gid); +H5F.close(fid); + +json_out = append(processing_dropbox, 'output_manifest_20220923t111111.json'); + +logger.info(strcat('Writing output manifest: ', json_out)) + +%Get checksum of written file +checksum_written = md5(output_filepath); + +s1.filename = output_filepath; +s1.checksum = checksum_written; +struct_1 = jsonencode(s1); + +s.manifest_type = "OUTPUT"; +s.files = [struct_1]; +s.configuration = string(nan); + +fid = fopen(json_out,'w'); + +fprintf(fid,'%s',strrep(jsonencode(s, "PrettyPrint", true),'\','')); +fclose(fid); + +logger.info('Algorithm complete. Exiting.') + +exit +end diff --git a/container_templates/matlab/container_example/src/logging/+logging/ConsoleHandler.m b/container_templates/matlab/container_example/src/logging/+logging/ConsoleHandler.m new file mode 100755 index 0000000000..40f641d949 --- /dev/null +++ b/container_templates/matlab/container_example/src/logging/+logging/ConsoleHandler.m @@ -0,0 +1,35 @@ +classdef ConsoleHandler < logging.Handler + + % Matthew Spaethe + % Ideas borrowed from Java logging, morphed into Matlab. + + methods + function obj = ConsoleHandler() + end + end + + methods + function [] = emit(obj, level, name, varargin) + %emit Used to pass log message to Handler from Logger. + + import logging.* + + % only print text if local 'logger level' equals or exceeds 'handler level' + if level >= obj.level + fprintf(1, '%s: ', Level.getName( level )); + fprintf(1, '%s: ', name); + fprintf(1, varargin{:}{:}); + fprintf(1, '\n'); + end + end + + function [] = close(obj) + end + + function [] = flush(obj) + end + + end + +end + diff --git a/container_templates/matlab/container_example/src/logging/+logging/FileHandler.m b/container_templates/matlab/container_example/src/logging/+logging/FileHandler.m new file mode 100755 index 0000000000..8509f9fc75 --- /dev/null +++ b/container_templates/matlab/container_example/src/logging/+logging/FileHandler.m @@ -0,0 +1,74 @@ +classdef FileHandler < logging.Handler + + % Matthew Spaethe + % Ideas borrowed from Java logging, morphed into Matlab. + + properties + fileId + fileOpen = false; + filename + logger + end + + methods + function obj = FileHandler(filename) + %FileHandler Handler subclass used to log messages to file. + % FileHandler(filename) Log output to file, creating / overwriting filename. + + import logging.* + + obj.logger = Logger.getLogger( ); + obj.logger.setLevel( Level.FINEST ); + obj.logger.log(Level.FINEST, 'constructor'); + + obj.logger.fine('opening file: %s', filename); + obj.fileId = fopen(filename, 'w'); + + if obj.fileId == -1 + obj.logger.log(Level.ERROR, 'error in opening file %s', filename); + obj.fileOpen = false; + obj.filename = []; + else + obj.fileOpen = true; + obj.filename = filename; + end + + end + + function delete(obj) + end + + function emit(obj, level, name, varargin) + %emit Used to pass log message to Handler from Logger. + + import logging.* + + % only print text if logged message level equals or exceeds 'handler level' + if level >= obj.level + fprintf(obj.fileId, '%s: ', Level.getName( level )); + fprintf(obj.fileId, '%s: ', name); + fprintf(obj.fileId, varargin{:}{:}); + fprintf(obj.fileId, '\n'); + end + end + + function [] = close(obj) + %close Called by Logger when emit() will no longer be called. Close any relevant i/o. + if obj.fileOpen + try + fclose( obj.fileId ); + obj.logger.fine('closed file %s', obj.filename); + obj.fileOpen = false; + catch e + obj.logger.error(e.message); + end + end + end + + function [] = flush(obj) + + end + + end +end + diff --git a/container_templates/matlab/container_example/src/logging/+logging/Handler.m b/container_templates/matlab/container_example/src/logging/+logging/Handler.m new file mode 100755 index 0000000000..ed1c536a60 --- /dev/null +++ b/container_templates/matlab/container_example/src/logging/+logging/Handler.m @@ -0,0 +1,48 @@ +classdef Handler < handle + %Handler Used by Logger to "emit" log messages. + % Additional handler classes can be written by subclassing Handler. + + % Matthew Spaethe + % Ideas borrowed from Java logging, morphed into Matlab. + + properties + level = logging.Level.OFF; + sticky = false; + end + + methods + function setLevel(obj, level) + %setLevel Set handler logging level. + % Log messages presented by the Logger with a logging level greater than or equal to the Handler logging level should be emitted. + obj.level = level; + end + + function setSticky(obj, sticky) + %setSticky Set the sticky flag. + % If a Handler is marked as sticky, it will not be removed from the Logger instance when the Logger is reset. + obj.sticky = sticky; + end + + function [level] = getLevel(obj) + %getLevel Return the handler logging level. + % Log messages presented by the Logger with a logging level greater than or equal to the Handler logging level should be emitted. + level = obj.level; + end + + function [sticky] = getSticky(obj) + %getSticky Return the sticky flag. + % If a Handler is marked as sticky, it will not be removed from the Logger instance when the Logger is reset. + sticky = obj.sticky; + end + end + + methods (Abstract) + %emit Method used to pass log message to Handler from Logger. + [] = emit(obj, level, name, varargin); + %close Method called by Logger when emit() will no longer be called. Close any relevant i/o. + [] = close(obj); + %flush Method called by Logger to request buffer to be flushed. + [] = flush(obj); + end + +end diff --git a/container_templates/matlab/container_example/src/logging/+logging/Level.m b/container_templates/matlab/container_example/src/logging/+logging/Level.m new file mode 100755 index 0000000000..f18c61c9a5 --- /dev/null +++ b/container_templates/matlab/container_example/src/logging/+logging/Level.m @@ -0,0 +1,80 @@ +classdef Level + %Level Class containing defined logging levels. + + % Matthew Spaethe + % Ideas borrowed from Java logging, morphed into Matlab. + + properties (Constant) + % Logging level + % + % Java + % • SEVERE 1000 + % • WARNING 900 + % • INFO 800 + % • CONFIG 700 + % • FINE 500 + % • FINER 400 + % • FINEST 300 + + % Python + % • CRITICAL 50 + % • ERROR 40 + % • WARNING 30 + % • INFO 20 + % • DEBUG 10 + % • NOTSET 0 + + ERROR = 60; + WARNING = 50; + INFO = 40; + FINE = 30; + FINER = 20; + FINEST = 10; + + OFF = 255; + ALL = 0; + end + + methods (Static) + function [str] = getName(level) + %getName Returns string name for numeric logging level. + + import logging.* + + switch level + case Level.ERROR, str = 'ERROR'; + case Level.WARNING, str = 'WARNING'; + case Level.INFO, str = 'INFO'; + case Level.FINE, str = 'FINE'; + case Level.FINER, str = 'FINER'; + case Level.FINEST, str = 'FINEST'; + + case Level.OFF, str = 'OFF'; + case Level.ALL, str = 'ALL'; + + otherwise, str = ['CUSTOM (' num2str(level) ')']; + end + end + + function level = getLevel(name) + %getLevel Returns numeric logging level for corresponding string. + + import logging.* + + switch name + case 'ERROR', level = Level.ERROR; + case 'WARNING', level = Level.WARNING; + case 'INFO', level = Level.INFO; + case 'FINE', level = Level.FINE; + case 'FINER', level = Level.FINER; + case 'FINEST', level = Level.FINEST; + case 'OFF', level = Level.OFF; + case 'ALL', level = Level.ALL; + otherwise, level = -1; + end + end + + end + +end + diff --git a/container_templates/matlab/container_example/src/logging/+logging/List.m b/container_templates/matlab/container_example/src/logging/+logging/List.m new file mode 100755 index 0000000000..124d9f5dbc --- /dev/null +++ b/container_templates/matlab/container_example/src/logging/+logging/List.m @@ -0,0 +1,46 @@ +classdef List < handle + %List List based on cell array. + + % Matthew Spaethe + % Ideas borrowed from Java logging, morphed into Matlab. + + properties (Access=private) + list + end + + methods + function obj = List() + obj.list = {}; + end + end + + methods + function [] = add(obj, x) + %add Add item to List. + obj.list = [obj.list {x}]; + end + + function [val] = getList(obj) + %getList Return cell array. + val = obj.list; + end + + function [] = remove(obj, x) + %remove Remove item from List. + + % TODO: cleaner method of removing handle from cell array? + for i=1:length(obj.list) + if x == obj.list{i} + obj.removeIndex(i); + break + end + end + end + + function [] = removeIndex(obj, index) + %removeIndex Remove item at specific position from List. + obj.list(index) = []; + end + end + +end \ No newline at end of file diff --git a/container_templates/matlab/container_example/src/logging/+logging/LogManager.m b/container_templates/matlab/container_example/src/logging/+logging/LogManager.m new file mode 100755 index 0000000000..33a064c16f --- /dev/null +++ b/container_templates/matlab/container_example/src/logging/+logging/LogManager.m @@ -0,0 +1,177 @@ +classdef LogManager < logging.Singleton + %LogManager Global LogManager instance + % There is a global LogManager object (Singleton). + % Call static method LogManager.getLogManager() to return the global LogManager object. + % + % Matthew Spaethe + % Ideas borrowed from Java logging, morphed into Matlab. + + properties (Constant) + ROOT_LOGGER_LOGGING_LEVEL = logging.Level.INFO; + ROOT_CONSOLE_HANDLER_LOGGING_LEVEL = logging.Level.ALL; + + LOGGER_LOGGING_LEVEL = logging.Level.INFO; + USE_PARENT_HANDLERS = true; + USE_STACK_CALL_NAME = true; + end + + properties + loggerMap + end + + methods (Access = private) + function obj = LogManager() + obj.loggerMap = containers.Map(); + end + end + + methods (Static) + function obj = instance() + persistent singleInstance + + if isempty( singleInstance ) + obj = logging.LogManager(); + singleInstance = obj; + + % create root logger and its console handler + obj.createRootLogger(); + else + obj = singleInstance; + end + + end + + function obj = getLogManager() + %getLogManager Return the global LogManager object. + % Wrapper to make method call similar to Java. + obj = logging.LogManager.instance(); + end + end + + methods (Access = private) + function [] = createRootLogger(obj) + %createRootLogger Add the default root logger and its console handler. + % We set the 'sticky' attribute so the console handler is not deleted by reset(). + import logging.* + + rootConsoleHandler = ConsoleHandler(); + rootConsoleHandler.setLevel( LogManager.ROOT_CONSOLE_HANDLER_LOGGING_LEVEL ); + rootConsoleHandler.setSticky( true ); + + rootLogger = Logger.getLogger(''); + rootLogger.setLevel( LogManager.ROOT_LOGGER_LOGGING_LEVEL ); + rootLogger.addHandler( rootConsoleHandler ); + end + end + + methods + + function delete(obj) + end + + function [logger] = getLogger(obj, name) + %getLogger Returns handle to named Logger, or empty array if not found. + % Java: matching logger or null if none is found + % + if ~obj.loggerMap.isKey( name ) + logger = []; + else + logger = obj.loggerMap( name ); + end + end + + function [success] = addLogger(obj, logger) + %addLogger Logger class will call this method to add a new Logger object to LogManager. + % Java: Add a named logger. This does nothing and returns false if a logger with the same name is already registered. + if ~obj.loggerMap.isKey( logger.getName() ) + % add to map + obj.loggerMap( logger.getName() ) = logger; + success = true; + else + success = false; + end + end + + function [] = reset(obj) + %reset Close and remove handlers. + % Each logger will instruct its registerd handlers to close(). + % Each logger will remove any registered handlers that are not "sticky" (e.g. root console handler). + import logging.* + + keys = obj.loggerMap.keys(); + for key = keys + logger = obj.loggerMap( key{1} ); + logger.reset(); + end + end + + function [] = resetAll(obj) + %resetAll Close and remove handlers, remove loggers, reset root logger and root console handler logging levels. + % Logger reset: + % Each logger will instruct its associated handlers to close(). + % Each logger will remove any handlers that are not "sticky" (e.g. root console handler). + % Delete loggers (except the root logger). + % Set the root logger back to its default logging level. + % Set the root console handler back to its default logging level. + import logging.* + + keys = obj.loggerMap.keys(); + for key = keys + logger = obj.loggerMap( key{1} ); + logger.reset(); + if ~strcmp(logger.name, '') + %delete( obj.loggerMap(key{1}) ); + obj.loggerMap.remove( key{1} ); + else + logger.setLevel( LogManager.ROOT_LOGGER_LOGGING_LEVEL ); + % get handle to the root console handler and restore its default logging level + consoleHandlers = logger.getHandlers('logging.ConsoleHandler'); + consoleHandlers{1}.setLevel( LogManager.ROOT_CONSOLE_HANDLER_LOGGING_LEVEL ); + end + end + end + + function [keys] = getLoggerNames(obj) + %getLoggerNames Returns cell array of Logger registered names. + keys = obj.loggerMap.keys(); + end + + function [] = printLoggers(obj) + %printLoggers Prints table of loggers, associated handlers, and 'use parent handler flag' (Y/N). + fprintf('\n'); + fprintf('--- LogManager table ---\n'); + + loggerNames = obj.loggerMap.keys(); + for loggerName = loggerNames + logger = obj.loggerMap( loggerName{1} ); + if strcmp( logger.getName(), '' ) + indent = 0; + else + indent = 2; + end + + for i = 1:indent + fprintf(' '); + end + + switch logger.getUseParentHandlers() + case 0, useParentLogger = 'N'; + case 1, useParentLogger = 'Y'; + end + + fprintf( '+ LOGGER ''%s'' %s [%s]\n', logger.getName(), logging.Level.getName(logger.getLevel()), useParentLogger ); + + handlers = logger.getHandlers(); + for handler = handlers + for i = 1:indent + fprintf(' '); + end + fprintf( ' HANDLER %s %s\n', class(handler{1}), logging.Level.getName(handler{1}.getLevel()) ); + end + end + fprintf('\n'); + end + + end + +end \ No newline at end of file diff --git a/container_templates/matlab/container_example/src/logging/+logging/Logger.m b/container_templates/matlab/container_example/src/logging/+logging/Logger.m new file mode 100755 index 0000000000..c0535824e9 --- /dev/null +++ b/container_templates/matlab/container_example/src/logging/+logging/Logger.m @@ -0,0 +1,309 @@ +classdef Logger < handle + %Logger Used for logging messages. + % Always call Logger.getLogger() to create a Logger or return an existing Logger. + % Calling Logger.getLogger() without an argument will instruct the method to use the call stack to determine the string name + % for the Logger. + % Calling Logger.getLogger(name) will use the provided name to create or return an existing Logger. + % Calling Logger.getLogger('') with an empty char array will return the root Logger. + % + % There are only two levels to the Logger hierarchy. The root logger is created by default, and is always the parent to any + % new logger instances. + % + % By default, a Logger instance will also use its parent's handlers (useParentHandlers). The easiest way to visualize this + % is to call the method printLoggers of LogManager. There may be times you want to disable this behavior. For example, you + % have a communications library that will log to a separate log file. You can disable useParentHandlers and add a FileHandler + % to your communications library logger. Then log messages will be handled by only your logger's handlers. + % + % Matthew Spaethe + % Ideas borrowed from Java logging, morphed into Matlab. + + properties + handlerList + level + name + parent + useParentHandlers + useStackCallName + end + + methods (Access = private) + function obj = Logger() + import logging.* + + obj.handlerList = List(); + obj.level = LogManager.LOGGER_LOGGING_LEVEL; + obj.useParentHandlers = LogManager.USE_PARENT_HANDLERS; + obj.useStackCallName = LogManager.USE_STACK_CALL_NAME; + end + end + + methods (Static) + function logger = getLogger( varargin ) + %getLogger Always use this static method to create or return an existing Logger instance. + % Calling Logger.getLogger() without an argument will instruct the method to use the call stack to determine the string name + % for the Logger. + % Calling Logger.getLogger(name) will use the provided name to create or return an existing Logger. + % Calling Logger.getLogger('') with an empty char array will return the root Logger. + % Java: Find or create a logger for a named subsystem. + + % access global LogManager instance + logManager = logging.LogManager.getLogManager(); + + if nargin == 0 + % no name provided, generate name + st = dbstack(1); + + if isempty(st) + name = 'MatlabCommandWindow'; + else + for i = length(st):-1:1 + if i == length(st) + name = st(i).name; + else + name = [name '>' st(i).name]; + end + end + end + + else + % class name (or similar) given + name = varargin{1}; + end + + logger = logManager.getLogger( name ); + + if isempty( logger ) + logger = logging.Logger(); + logger.setName( name ); + + % set handle to parent logger + if ~strcmp(name, '') + logger.setParent( logManager.getLogger('') ) + else + logger.setParent( [] ); + end + + % register new Logger instance with LogManager + logManager.addLogger( logger ); + end + + end + end + + methods + function addHandler(obj, handler) + %addHandler Add handler instance to this logger. + obj.handlerList.add( handler ); + end + + function setLevel(obj, level) + %setLevel Set logging level. + % A log message with a logging level greater than or equal to the logger's logging level will be passed to handlers. + obj.level = level; + end + + function setName(obj, name) + %setName Set the logger's registered name. + obj.name = name; + end + + function setParent(obj, parent) + %setParent Set handle to the logger's parent. + obj.parent = parent; + end + + function setUseParentHandlers(obj, useParentHandlers) + %setUseParentHandlers Set useParentHandlers flag. + % If true, the parent's handlers will be used in addition to the logger's handlers. That is, log messages will + % be forwarded to the logger's handlers and the parent's handlers if the logging level has been met. + obj.useParentHandlers = useParentHandlers; + end + + function setUseStackCallName(obj, useStackCallName) + %setUseStackCallName Set useStackCallName flag. + % When passing log messages to the handlers, this flag determines if the call stack (at point of log call) or the + % registered name should be used. + obj.useStackCallName = useStackCallName; + end + + function [parent] = getParent(obj) + %getParent Returns handle to parent Logger. + parent = obj.parent; + end + + function [handlers] = getHandlers(obj, varargin) + %getHandlers Returns a cell array of handles to Handler objects. + % getHandlers() returns a cell array of handles to all Handler objects for this Logger. + % getHandlers(classname) returns a cell array of handles to all Handler objects for this Logger of type classname. + handlers = []; + + if nargin == 1 + handlers = obj.handlerList.getList(); + else + for i = obj.handlerList.getList() + if isa(i{1}, varargin{1}) + handlers = [handlers i]; + end + end + end + end + + function [level] = getLevel(obj) + %getLevel Return the logging level for this Logger instance. + % A log message with a logging level greater than or equal to the logger's logging level will be passed to handlers. + level = obj.level; + end + + function [name] = getName(obj) + %getName Return the logger's registered name. + name = obj.name; + end + + function [useParentHandlers] = getUseParentHandlers(obj) + %getUseParentHandlers Return the useParentHandlers flag. + % If true, the parent's handlers will be used in addition to the logger's handlers. That is, log messages will + % be forwarded to the logger's handlers and the parent's handlers if the logging level has been met. + useParentHandlers = obj.useParentHandlers; + end + + function [useStackCallName] = getUseStackCallName(obj) + %getUseStackCallName Return the useStackCallName flag. + % When passing log messages to the handlers, this flag determines if the call stack (at point of log call) or the + % registered name should be used. + useStackCallName = obj.useStackCallName; + end + + function [] = reset(obj) + %reset Close and remove handlers. + % Instruct registered handlers to close(). + % Remove any registered handlers that are not "sticky" (e.g. root console handler). + for handler = obj.handlerList.getList() + if ~handler{1}.getSticky() + obj.handlerList.remove( handler{1} ); + handler{1}.close(); + delete( handler{1} ); + end + end + end + + + function log(obj, level, varargin) + %log Generate log message of specified logging level. + % Example usage: + % + % import logging.* + % log(Level.INFO, 'Received %d packets', receivedPacketCount); + % + % log(logging.Level.INFO, 'Received %d packets', receivedPacketCount); + obj.logm( 2, level, varargin{:} ); + end + + function error(obj, varargin) + %error Convenience method for logging message with level ERROR. + obj.logm( 2, logging.Level.ERROR, varargin{:} ); + end + + function warning(obj, varargin) + %warning Convenience method for logging message with level WARNING. + obj.logm( 2, logging.Level.WARNING, varargin{:} ); + end + + function info(obj, varargin) + %info Convenience method for logging message with level INFO. + obj.logm( 2, logging.Level.INFO, varargin{:} ); + end + + function fine(obj, varargin) + %fine Convenience method for logging message with level FINE. + obj.logm( 2, logging.Level.FINE, varargin{:} ); + end + + function finer(obj, varargin) + %finer Convenience method for logging message with level FINER. + obj.logm( 2, logging.Level.FINER, varargin{:} ); + end + + function finest(obj, varargin) + %finest Convenience method for logging message with level FINEST. + obj.logm( 2, logging.Level.FINEST, varargin{:} ); + end + end + + methods (Access = private) + function logm(obj, stackPtr, level, varargin) + %logm Internal method used to log messages. + if level >= obj.level + + if obj.useStackCallName + st = dbstack(stackPtr); + + if isempty(st) + stackCallName = 'MatlabCommandWindow'; + else + for i = length(st):-1:1 + if i == length(st) + stackCallName = st(i).name; + else + stackCallName = [stackCallName '>' st(i).name]; + end + end + end + end + + for handler = obj.handlerList.getList() + % stackCallName results in similar log message to Java logp() + if obj.useStackCallName + handler{1}.emit( level, stackCallName, varargin ); + else + handler{1}.emit( level, obj.name, varargin ); + end + end + + % --- use parent handlers --- + + if obj.useParentHandlers + logger = obj.parent; + else + logger = []; + end + + % root logger will have empty parent handle + while ~isempty( logger ) + for handler = logger.handlerList.getList() + if obj.useStackCallName + handler{1}.emit( level, stackCallName, varargin ); + else + handler{1}.emit( level, obj.name, varargin ); + end + end + + % continue up hierarchy + if logger.useParentHandlers + logger = logger.parent; + else + logger = []; + end + end + + % ----------------------- + + end + +% % --- use parent loggers --- +% +% % propagate logged message up the chain? +% if obj.useParentHandlers +% logger = obj.parent; +% % root logger will have empty parent handle +% if ~isempty( logger ) +% logger.logm( stackPtr + 1, level, varargin{:} ); +% end +% end +% +% % ---------------------- + + end + end + +end + diff --git a/container_templates/matlab/container_example/src/logging/+logging/Singleton.m b/container_templates/matlab/container_example/src/logging/+logging/Singleton.m new file mode 100755 index 0000000000..998f6f9685 --- /dev/null +++ b/container_templates/matlab/container_example/src/logging/+logging/Singleton.m @@ -0,0 +1,80 @@ +classdef Singleton < handle + %SINGLETON Abstract Class for Singleton OOP Design Pattern + % Intent: Ensures a class only has one instance and provide a + % global point of access to it [1]. + % Motivation: It's important for some classes to have exactly one + % instance. For example, it can be desirable to have only a + % single instance of a GUI. With a MATLAB GUIDE built GUI, the driver + % or main function provides a global point of access which is + % executed to, + % 1. initially instantiate the GUI; and + % 2. subsequently bring the existing GUI into focus *not* creating + % a new one. + % Implementation: MATLAB OOP doesn't have the notion of static + % properties. Properties become available once the constructor + % of the class is invoked. In the case of the Singleton Pattern, it + % is then undesirable to use the constructor as a global point of + % access since it creates a new instance before you can check if an + % instance already exists. The solution is to use a persistent + % variable within a unique static method instance() which calls the + % constructor to create a unique 'singleton' instance. The persistent + % variable can be interrogated prior to object creation and after + % object creation to check if the singleton object exists. There are + % two conditions that all subclasses must satisfy: + % 1. Constructor must be hidden from the user (Access=private). + % 2. The concrete implementation of instance() must use a persistent + % variable to store the unique instance of the subclass. + % + % Refer to pp.127-134 Gamma et al.[1] for more information on the + % Singleton Design Pattern. + % + % Written by Bobby Nedelkovski + % The MathWorks Australia Pty Ltd + % Copyright 2009, The MathWorks, Inc. + % + % Reference: + % [1] Gamma, E., Helm, R., Johnson, R. and Vlissides, J. + % Design Patterns : Elements of Reusable Object-Oriented Software. + % Boston: Addison-Wesley, 1995. + + % Unique properties of Singleton. This can be only accessed + % with the getter/setter methods via any subclass of Singleton. + properties(Access=private) + singletonData = []; + end + + methods(Abstract, Static) + % This method serves as the global point of access in creating a + % single instance *or* acquiring a reference to the singleton. + % If the object doesn't exist, create it otherwise return the + % existing one in persistent memory. + % Input: + % + % Output: + % obj = reference to a persistent instance of the class + obj = instance(); + end + + methods % Public Access + % Accessor method for querying the singleton data. + % Input: + % obj = reference to the singleton instance of the subclass + % Output: + % singletonData = internal data store for Singleton object + function singletonData = getSingletonData(obj) + singletonData = obj.singletonData; + end + + % Accessor method for modifying the singleton data. + % Input: + % obj = reference to the singleton instance of the subclass + % singletonData = new data to set internal data store + % Output: + % + function setSingletonData(obj, singletonData) + obj.singletonData = singletonData; + end + + end + +end diff --git a/container_templates/matlab/container_example/src/logging/TestObject.m b/container_templates/matlab/container_example/src/logging/TestObject.m new file mode 100755 index 0000000000..540cb3cfe2 --- /dev/null +++ b/container_templates/matlab/container_example/src/logging/TestObject.m @@ -0,0 +1,39 @@ +classdef TestObject < handle + + % Matthew Spaethe + % Ideas borrowed from Java logging, morphed into Matlab. + + properties (Access = private) + logger + end + + methods + + function obj = TestObject() + import logging.* + + % use class name when registering this logger + % multiple instances of this class will use the same logger instance since we are registering via class name + obj.logger = Logger.getLogger( class(obj) ); + obj.logger.setLevel( Level.INFO ); + obj.logger.log(Level.FINEST, 'constructor'); + end + + function [] = run(obj) + obj.logger.log(logging.Level.INFO, 'example printf type %s', 'call'); + + % blah blah + % blah blah + + obj.logger.info('using convenience method for logging'); + + % blah blah + % blah blah + + obj.logger.setUseStackCallName( false ); + obj.logger.warning('disabled stack call name generation; use registered name in log messages'); + + end + + end +end diff --git a/container_templates/matlab/container_example/src/logging/TestObject2.m b/container_templates/matlab/container_example/src/logging/TestObject2.m new file mode 100755 index 0000000000..d76dfce8f2 --- /dev/null +++ b/container_templates/matlab/container_example/src/logging/TestObject2.m @@ -0,0 +1,36 @@ +classdef TestObject2 < handle + + % Matthew Spaethe + % Ideas borrowed from Java logging, morphed into Matlab. + + properties (Constant) + logger = logging.Logger.getLogger('TestObject2'); + end + + methods + + function obj = TestObject2() + import logging.* + + TestObject2.logger.setLevel( Level.INFO ); + TestObject2.logger.log(Level.FINEST, 'constructor'); + end + + function [] = run(obj) + TestObject2.logger.log(logging.Level.INFO, 'example printf type %s', 'call'); + + % blah blah + % blah blah + + TestObject2.logger.info('using convenience method for logging'); + + % blah blah + % blah blah + + TestObject2.logger.setUseStackCallName( false ); + TestObject2.logger.warning('disabled stack call name generation; use registered name in log messages'); + + end + + end +end diff --git a/container_templates/matlab/container_example/src/logging/TestScript.m b/container_templates/matlab/container_example/src/logging/TestScript.m new file mode 100755 index 0000000000..ee2fe0fff9 --- /dev/null +++ b/container_templates/matlab/container_example/src/logging/TestScript.m @@ -0,0 +1,110 @@ +% Matthew Spaethe +% Ideas borrowed from Java logging, morphed into Matlab. + +import logging.* + + +% --- BASIC CONSOLE --- + +% get the global LogManager +logManager = LogManager.getLogManager(); + +% add a logger instance to this script +logger = Logger.getLogger('TestScript'); +logger.setLevel( Level.ALL ); + +logger.info('Hi, this is info!'); +logger.warning('Hi, this is warning!'); + +% side-effect is to close all handlers +logManager.resetAll(); + + +% --- BASIC CONSOLE WITH FILE --- + +% get the global LogManager +logManager = LogManager.getLogManager(); + +% add a file handler to the root logger +fileHandler = FileHandler('./Basic-RootFileHandler.log'); +fileHandler.setLevel( Level.ALL ); +rootLogger = logManager.getLogger(''); +rootLogger.addHandler( fileHandler ); + +% add a logger instance to this script +% will use stack to generate name for logger since a name is not being provided +logger = Logger.getLogger( ); +logger.setLevel( Level.ALL ); + +logger.info('Hi, this is info!'); +logger.warning('Hi, this is warning!'); + +% side-effect is to close all handlers +logManager.resetAll(); + + +% --- A LITTLE BEYOND --- + +% get the global LogManager +logManager = LogManager.getLogManager(); + +% in case LogManager exists from a previous run +% removes all handlers -- except for the root console handler +% removes all loggers -- except for the root logger +% logManager.resetAll(); + +% removes all handlers -- except for the root console handler +% logManager.reset(); + +% obtain handle to root logger and set its logging level +% output log messages of level INFO and above +rootLogger = logManager.getLogger(''); +rootLogger.setLevel( Level.INFO ); + +% add a file handler to the root logger +fileHandler = FileHandler('./RootFileHandler.log'); +fileHandler.setLevel( Level.ALL ); +rootLogger.addHandler( fileHandler ); + +% add a logger instance to this script +% will use stack to generate name for logger since a name is not being provided +logger = Logger.getLogger( ); +logger.setLevel( Level.ALL ); + +% add a file handler to this script's logger +fileHandler = FileHandler('./ScriptFileHandler.log'); +fileHandler.setLevel( Level.ALL ); +logger.addHandler( fileHandler ); + +logger.info('hi'); + +% don't propagate log messages to the parent's handlers +logger.setUseParentHandlers( false ); +logger.log( Level.WARNING, 'yo' ); + +% create an instance of TestObject +testObject = TestObject(); +testObject.run(); + +% create an instance of TestObject2 +testObject2 = TestObject2(); +testObject2.run(); + +% display table of all registered loggers and their associated handlers +logManager = LogManager.getLogManager(); +logManager.printLoggers(); + +% change the root console handler's logging level +logManager = LogManager.getLogManager(); +logger = logManager.getLogger(''); +consoleHandlers = logger.getHandlers('logging.ConsoleHandler'); +consoleHandlers{1}.setLevel( Level.INFO ); + +% side-effect is to close all handlers (e.g. file handler) +% removes all handlers -- except for the root console handler +logManager.reset(); +logManager.printLoggers(); + +% removes all loggers and handlers -- except for the root logger and the root console handler +logManager.resetAll(); +logManager.printLoggers(); \ No newline at end of file diff --git a/container_templates/matlab/container_example/src/logging/license.txt b/container_templates/matlab/container_example/src/logging/license.txt new file mode 100644 index 0000000000..9434f0da5a --- /dev/null +++ b/container_templates/matlab/container_example/src/logging/license.txt @@ -0,0 +1,28 @@ +Copyright (c) 2013, Matthew Spaethe +Copyright (c) 2009, The MathWorks, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution + * Neither the name of the The MathWorks, Inc. nor the names + of its contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/container_templates/matlab/container_example/src/md5.m b/container_templates/matlab/container_example/src/md5.m new file mode 100644 index 0000000000..0e632a05b1 --- /dev/null +++ b/container_templates/matlab/container_example/src/md5.m @@ -0,0 +1,198 @@ +% md5 Compute MD5 hash function for files +% +% d = md5(FileName) +% +% md5() computes the MD5 hash function of +% the file specified in the string FileName +% and returns it as a 64-character array d. + + +% The MD5 message-digest algorithm is specified +% in RFC 1321. + +% The code below is for instructional and illustrational +% purposes only. It is very clear, but very slow. + +% (C) Stefan Stoll, ETH Zurich, 2006 + +function Digest = md5(FileName) + +% Guard against old Matlab versions +MatlabVersion = version; +if MatlabVersion(1)<'7' + error('md5() requires Matlab 7.0 or later!'); +end + +% Run autotest if no parameters are given +if (nargin==0) + md5autotest; + return; +end + +% Read in entire file into uint32 vector +[Message,nBits] = readmessagefromfile(FileName); + +%-------------------------------------------------- + +% Append a bit-1 to the last bit read from file +BytesInLastInt = mod(nBits,32)/8; +if BytesInLastInt + Message(end) = bitset(Message(end),BytesInLastInt*8+8); +else + Message = [Message; uint32(128)]; +end + +% Append zeros +nZeros = 16 - mod(numel(Message)+2,16); +Message = [Message; zeros(nZeros,1,'uint32')]; + +% Append bit length of original message as uint64, lower significant uint32 first +Lower32 = uint32(nBits); +Upper32 = uint32(bitshift(uint64(nBits),-32)); +Message = [Message; Lower32; Upper32]; + +%-------------------------------------------------- + +% 64-element transformation array +T = uint32(fix(4294967296*abs(sin(1:64)))); + +% 64-element array of number of bits for circular left shift +S = repmat([7 12 17 22; 5 9 14 20; 4 11 16 23; 6 10 15 21].',4,1); +S = S(:).'; + +% 64-element array of indices into X +idxX = [0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 ... + 1 6 11 0 5 10 15 4 9 14 3 8 13 2 7 12 ... + 5 8 11 14 1 4 7 10 13 0 3 6 9 12 15 2 ... + 0 7 14 5 12 3 10 1 8 15 6 13 4 11 2 9] + 1; + +% Initial state of buffer (consisting of A, B, C and D) +A = uint32(hex2dec('67452301')); +B = uint32(hex2dec('efcdab89')); +C = uint32(hex2dec('98badcfe')); +D = uint32(hex2dec('10325476')); + +%-------------------------------------------------- + +Message = reshape(Message,16,[]); + +% Loop over message blocks each 16 uint32 long +for iBlock = 1:size(Message,2) + + % Extract next block + X = Message(:,iBlock); + + % Store current buffer state + AA = A; + BB = B; + CC = C; + DD = D; + + % Transform buffer using message block X and the + % parameters from S, T and idxX + k = 0; + for iRound = 1:4 + for q = 1:4 + A = Fun(iRound,A,B,C,D,X(idxX(k+1)),S(k+1),T(k+1)); + D = Fun(iRound,D,A,B,C,X(idxX(k+2)),S(k+2),T(k+2)); + C = Fun(iRound,C,D,A,B,X(idxX(k+3)),S(k+3),T(k+3)); + B = Fun(iRound,B,C,D,A,X(idxX(k+4)),S(k+4),T(k+4)); + k = k + 4; + end + end + + % Add old buffer state + A = bitadd32(A,AA); + B = bitadd32(B,BB); + C = bitadd32(C,CC); + D = bitadd32(D,DD); + +end + +%-------------------------------------------------- + +% Combine uint32 from buffer to form message digest +Str = lower(dec2hex([A;B;C;D])); +Str = Str(:,[7 8 5 6 3 4 1 2]).'; +Digest = Str(:).'; + +%================================================== + +function y = Fun(iRound,a,b,c,d,x,s,t) +switch iRound +case 1 + q = bitor(bitand(b,c),bitand(bitcmp(b),d)); +case 2 + q = bitor(bitand(b,d),bitand(c,bitcmp(d))); +case 3 + q = bitxor(bitxor(b,c),d); +case 4 + q = bitxor(c,bitor(b,bitcmp(d))); +end +y = bitadd32(b,rotateleft32(bitadd32(a,q,x,t),s)); + +%-------------------------------------------- + +function y = rotateleft32(x,s) +y = bitor(bitshift(x,s),bitshift(x,s-32)); + +%-------------------------------------------- + +function sum = bitadd32(varargin) +sum = varargin{1}; +for k = 2:nargin + add = varargin{k}; + carry = bitand(sum,add); + sum = bitxor(sum,add); + for q = 1:32 + shift = bitshift(carry,1); + carry = bitand(shift,sum); + sum = bitxor(shift,sum); + end +end + +function [Message,nBits] = readmessagefromfile(FileName) +[hFile,ErrMsg] = fopen(FileName,'r'); +error(ErrMsg); +%Message = fread(hFile,inf,'bit32=>uint32'); +Message = fread(hFile,inf,'ubit32=>uint32'); +fclose(hFile); +d = dir(FileName); +nBits = d.bytes*8; + +%============================================ + +function md5autotest + +disp('Running md5 autotest...'); + +Messages{1} = ''; +Messages{2} = 'a'; +Messages{3} = 'abc'; +Messages{4} = 'message digest'; +Messages{5} = 'abcdefghijklmnopqrstuvwxyz'; +Messages{6} = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; +Messages{7} = char(128:255); + +CorrectDigests{1} = 'd41d8cd98f00b204e9800998ecf8427e'; +CorrectDigests{2} = '0cc175b9c0f1b6a831c399e269772661'; +CorrectDigests{3} = '900150983cd24fb0d6963f7d28e17f72'; +CorrectDigests{4} = 'f96b697d7cb7938d525a2f31aaf161d0'; +CorrectDigests{5} = 'c3fcd3d76192e4007dfb496cca67e13b'; +CorrectDigests{6} = 'd174ab98d277d9f5a5611c2c9f419d9f'; +CorrectDigests{7} = '16f404156c0500ac48efa2d3abc5fbcf'; + +TmpFile = tempname; +for k=1:numel(Messages) + [h,ErrMsg] = fopen(TmpFile,'w'); + error(ErrMsg); + fwrite(h,Messages{k},'char'); + fclose(h); + Digest = md5(TmpFile); + fprintf('%d: %s\n',k,Digest); + if ~strcmp(Digest,CorrectDigests{k}) + error('md5 autotest failed on the following string: %s',Messages{k}); + end +end +delete(TmpFile); +disp('md5 autotest passed!'); diff --git a/container_templates/python/container_example/.gitignore b/container_templates/python/container_example/.gitignore new file mode 100644 index 0000000000..b3ed9b7ea5 --- /dev/null +++ b/container_templates/python/container_example/.gitignore @@ -0,0 +1,8 @@ +# pyenv specific file +.python-version + +# PyCharm config files +.idea/ + +# Remnants of Installing locally # +*.egg-info \ No newline at end of file diff --git a/container_templates/python/container_example/Dockerfile b/container_templates/python/container_example/Dockerfile new file mode 100644 index 0000000000..bf05098197 --- /dev/null +++ b/container_templates/python/container_example/Dockerfile @@ -0,0 +1,28 @@ +#Dockerfile is used to create an image + +#Set the base image. +FROM python:3.9-slim + +RUN apt-get update +RUN apt-get install -y gcc libpq-dev + +#Set the work directory in the container +WORKDIR /home + +# copy the dependencies file to the working directory +COPY requirements.txt . + +# Ensure pip is up to date +RUN pip install --upgrade pip + +# install dependencies +RUN pip install -r requirements.txt + +# copy the content of the local src directory to the working directory +COPY src/ . + +# Define the entrypoint of the container. Passing arguments (known as the COMMAND) when running the +# container will be passed as arguments to the function. Ultimately what is executed is the ENTRYPOINT concatenated +# with the COMMAND. You may add custom options to ENTRYPOINT but note that those options will always be set when +# running in AWS. +ENTRYPOINT ["python","algorithm_example.py"] diff --git a/container_templates/python/container_example/README.md b/container_templates/python/container_example/README.md new file mode 100644 index 0000000000..5694817594 --- /dev/null +++ b/container_templates/python/container_example/README.md @@ -0,0 +1,56 @@ + # Container Example Code + +This the basic code for creating a Docker container for the Level 2 pipeline. +For questions contact: laura.sandoval@lasp.colorado.edu or contact Laura Sandoval via Slack. + +## Python Setup + +It is recommended that a virtual environment is created within the project: +1. Make a virtual environment using `python -m venv venv` and activate it +with `source venv/bin/activate`. Python 3.10 is required. +2. Generate requirements.txt only if additional libraries are required. Otherwise, use the default project requirements.txt. +3. pip install -r requirements.txt + +Note: If an error is thrown when installing psycopg2, you may need to read requirements for installing psycopg2: +https://www.psycopg.org/docs/install.html#build-prerequisites. On Mac simply install Postgres.app, +which comes with a Postgres server, client, and command line utilities (https://postgresapp.com/) + + +## Example Algorithm Usage + +To run +`python src/algorithm_example.py --help` + + +## Building and Running a Docker Image Locally + +To build the image run the following command from the Dockerfile directory. You might add -t option to tag your image +and --rm to remove intermediate containers after the build is done. + +`docker build -t my-image --rm .` + +Now we can run our image using bind mounting. In our example, we will name the container ‘my_app’. +Adding --rm option will remove the container automatically after the container exits. + +`docker run --rm -it \ + -e PROCESSING_DROPBOX=/opt/data/dropbox \ + --volume="$(pwd)/container_example_data:/opt/data" \ + my-image:latest /opt/data/dropbox/libera_input_manifest_20220923t000000.json` + +The script `run_container_example.sh` should contain this code as well. You can build and run the example container +with `./run_container_example.sh`. Docker must be running, you must have permissions to execute the script, and +you must be in the same directory as the script. If you get a permission error, +run `chmod 755 run_container_example.sh` + +## Building and running a docker image in AWS + +You will be given AWS console logins in the future. Don't worry about this for now. + +In order to push a Docker Image to the ECR, navigate to the AWS Console to retrieve the AWS ECR registry URI: +https://us-west-2.console.aws.amazon.com/ecr/repositories?region=us-west-2 + +Follow the instructions in the link to push to the AWS ECR. +https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html + +The ECR can be browsed through the AWS Console: +https://us-west-2.console.aws.amazon.com/ecr/repositories?region=us-west-2 diff --git a/container_templates/python/container_example/container_example_data/dropbox/libera_input_manifest_20220923t000000.json b/container_templates/python/container_example/container_example_data/dropbox/libera_input_manifest_20220923t000000.json new file mode 100644 index 0000000000..9b152cf3b0 --- /dev/null +++ b/container_templates/python/container_example/container_example_data/dropbox/libera_input_manifest_20220923t000000.json @@ -0,0 +1,8 @@ +{ + "manifest_type": "INPUT", + "files": [ + {"filename": "/opt/data/source_data/filename_example_1.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"}, + {"filename": "/opt/data/source_data/filename_example_2.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"} + ], + "configuration": null +} diff --git a/container_templates/python/container_example/container_example_data/dropbox/libera_input_manifest_20220923t000000_aws.json b/container_templates/python/container_example/container_example_data/dropbox/libera_input_manifest_20220923t000000_aws.json new file mode 100644 index 0000000000..66c13db5fc --- /dev/null +++ b/container_templates/python/container_example/container_example_data/dropbox/libera_input_manifest_20220923t000000_aws.json @@ -0,0 +1,8 @@ +{ + "manifest_type": "INPUT", + "files": [ + {"filename": "s3://bucket-path-dev/source-data/filename_example_1.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"}, + {"filename": "s3://bucket-path-dev/source-data/filename_example_2.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"} + ], + "configuration": null +} diff --git a/container_templates/python/container_example/container_example_data/dropbox/libera_input_manifest_20220923t000000_local.json b/container_templates/python/container_example/container_example_data/dropbox/libera_input_manifest_20220923t000000_local.json new file mode 100644 index 0000000000..a60f693dec --- /dev/null +++ b/container_templates/python/container_example/container_example_data/dropbox/libera_input_manifest_20220923t000000_local.json @@ -0,0 +1,8 @@ +{ + "manifest_type": "INPUT", + "files": [ + {"filename": "../container_example_data/source_data/filename_example_1.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"}, + {"filename": "../container_example_data/source_data/filename_example_2.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"} + ], + "configuration": null +} diff --git a/container_templates/python/container_example/container_example_data/dropbox/libera_output_manifest_20230508t212437.json b/container_templates/python/container_example/container_example_data/dropbox/libera_output_manifest_20230508t212437.json new file mode 100644 index 0000000000..c6ca471a42 --- /dev/null +++ b/container_templates/python/container_example/container_example_data/dropbox/libera_output_manifest_20230508t212437.json @@ -0,0 +1 @@ +{"manifest_type": "OUTPUT", "files": [], "configuration": {}} \ No newline at end of file diff --git a/container_templates/python/container_example/container_example_data/dropbox/libera_output_manifest_20230508t212736.json b/container_templates/python/container_example/container_example_data/dropbox/libera_output_manifest_20230508t212736.json new file mode 100644 index 0000000000..c6ca471a42 --- /dev/null +++ b/container_templates/python/container_example/container_example_data/dropbox/libera_output_manifest_20230508t212736.json @@ -0,0 +1 @@ +{"manifest_type": "OUTPUT", "files": [], "configuration": {}} \ No newline at end of file diff --git a/container_templates/python/container_example/container_example_data/dropbox/libera_output_manifest_20230508t224157.json b/container_templates/python/container_example/container_example_data/dropbox/libera_output_manifest_20230508t224157.json new file mode 100644 index 0000000000..c6ca471a42 --- /dev/null +++ b/container_templates/python/container_example/container_example_data/dropbox/libera_output_manifest_20230508t224157.json @@ -0,0 +1 @@ +{"manifest_type": "OUTPUT", "files": [], "configuration": {}} \ No newline at end of file diff --git a/container_templates/python/container_example/container_example_data/dropbox/libera_output_manifest_20230508t224239.json b/container_templates/python/container_example/container_example_data/dropbox/libera_output_manifest_20230508t224239.json new file mode 100644 index 0000000000..c6ca471a42 --- /dev/null +++ b/container_templates/python/container_example/container_example_data/dropbox/libera_output_manifest_20230508t224239.json @@ -0,0 +1 @@ +{"manifest_type": "OUTPUT", "files": [], "configuration": {}} \ No newline at end of file diff --git a/container_templates/python/container_example/container_example_data/source_data/filename_example_1.h5 b/container_templates/python/container_example/container_example_data/source_data/filename_example_1.h5 new file mode 100644 index 0000000000..e805e3c817 Binary files /dev/null and b/container_templates/python/container_example/container_example_data/source_data/filename_example_1.h5 differ diff --git a/container_templates/python/container_example/container_example_data/source_data/filename_example_2.h5 b/container_templates/python/container_example/container_example_data/source_data/filename_example_2.h5 new file mode 100644 index 0000000000..e805e3c817 Binary files /dev/null and b/container_templates/python/container_example/container_example_data/source_data/filename_example_2.h5 differ diff --git a/container_templates/python/container_example/requirements.txt b/container_templates/python/container_example/requirements.txt new file mode 100644 index 0000000000..738ea0374e --- /dev/null +++ b/container_templates/python/container_example/requirements.txt @@ -0,0 +1,5 @@ +libera-utils +numpy +h5py +pandas +libera-utils[db,spice] == 2.0.0 \ No newline at end of file diff --git a/container_templates/python/container_example/run_container_example.sh b/container_templates/python/container_example/run_container_example.sh new file mode 100755 index 0000000000..009eb703f4 --- /dev/null +++ b/container_templates/python/container_example/run_container_example.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e +echo "Building docker container" +docker build -t algorithm-example:latest . +echo "Finished." +echo +echo "Running container example." +docker run --rm -it \ + -e PROCESSING_DROPBOX=/opt/data/dropbox \ + --volume="$(pwd)/container_example_data:/opt/data" \ + algorithm-example:latest /opt/data/dropbox/input_manifest_20220923t000000.json +echo "Algorithm complete" \ No newline at end of file diff --git a/container_templates/python/container_example/src/algorithm_example.py b/container_templates/python/container_example/src/algorithm_example.py new file mode 100644 index 0000000000..20f8f69ab9 --- /dev/null +++ b/container_templates/python/container_example/src/algorithm_example.py @@ -0,0 +1,164 @@ +""" +Container example for Level 2 developers + +Authors: Laura Sandoval, Gavin Medley, Matt Watwood +""" +# Standard +import argparse +from hashlib import md5 +import logging +import os +import sys +# Installed +import h5py as h5 +import numpy as np +import pandas as pd +# Local +from libera_utils.io.smart_open import smart_open +from libera_utils.io.manifest import Manifest, ManifestType +from static.l2_quality_flags import L2QualityFlag + + +logger = logging.getLogger(__name__) + + +def main(): + # Initialize logging. You can configure logging however you like + logging.basicConfig(level="DEBUG") + + # Get logger instance, named for the current function for traceability + logger.debug(f"Command executed in container: {sys.argv}") + logger.info("Parsing CLI arguments.") + args = parse_cli_args(sys.argv[1:]) + + manifest = args.manifest + logger.info(f"Manifest file to read: {manifest}") + + logger.info("Additional options passed are " + f"exampleint={args.exampleint}, examplefloat={args.examplefloat}, examplebool={args.examplebool}") + + processing_dropbox = os.environ['PROCESSING_DROPBOX'] + + # read json information + manifest = Manifest.from_file(manifest) + if not manifest.manifest_type == ManifestType.INPUT: + raise ValueError("Incorrect manifest type received as input.") + + logger.debug(f"Manifest type is {manifest.manifest_type}") + logger.debug(f"Manifest contains files:\n{manifest.files}") + + # read hdf5 + for record in manifest.files: + checksum = record['checksum'] + filename = record['filename'] + # Validate checksums + with smart_open(filename, 'rb') as fh: + if checksum != md5(fh.read()).hexdigest(): + raise ValueError("Checksums do not match!") + logger.debug(f"Checksum matches for {filename}") + + with h5.File(smart_open(filename, 'rb'), 'r') as h5file: + # Get the data from each file and put data into some format that will be used in the algorithm + data_in = np.array(h5file['HDFEOS/SWATHS/Swath1/DataField/Temperature']) + logger.info(f"Found input data in HDF5 file:\n{data_in}") + + df = generate_example_data() + + # write example output data to a new HDF5 file + output_files = [] + output_filepath = os.path.join(processing_dropbox, 'example_output.h5') + logger.info(f"Writing output file: {output_filepath}") + with h5.File(smart_open(output_filepath, 'xb'), 'x') as hdf: + hdf.create_group('new_group') + hdf.attrs['someattr'] = "hello, world" + hdf.create_dataset('data/array1', data=df.data_out) + hdf.create_dataset('quality_out/array1', data=df.quality_int) + + # get the checksum of the written file + with smart_open(output_filepath, 'rb') as fh: + checksum = md5(fh.read()).hexdigest() + + output_files.append({"filename": output_filepath, "checksum": checksum}) + + # Write output manifest file containing a list of the product files that the processing created + output_manifest_path = os.path.join(processing_dropbox, "libera_output_manifest_20220923t111111.json") + logger.info(f"Writing output manifest: {output_filepath}") + output_manifest = Manifest(manifest_type=ManifestType.OUTPUT, + filename=output_manifest_path, + files=output_files, + configuration={}) + output_manifest.write(output_manifest_path) + logger.info("Algorithm complete. Exiting.") + + +def generate_example_data(): + """ + Function created to make up data and quality flags + + Returns + ------- + pandas.core.frame.DataFrame + Dataframe containing data and quality flag columns + """ + + # generate fake data product to write as output + df = pd.DataFrame() + df['data_out'] = [-np.inf, -np.inf, 30, 40, 50, 60] + df['quality'] = np.full(len(df.data_out), fill_value=L2QualityFlag.NONE) + df['quality_int'] = np.full(len(df.data_out), fill_value=0) + + # flag neg infinite values + was_inf = np.equal(df.data_out, -np.inf) + df.loc[was_inf, 'quality'] = df.loc[was_inf, 'quality'][0] | L2QualityFlag.INF | L2QualityFlag.NEG + df.loc[was_inf, 'quality_int'] = (df.loc[was_inf, 'quality'][0] | L2QualityFlag.INF | L2QualityFlag.NEG).value + + # view decomposed flag and value + view_summary = df.quality.values[0].summary + view_decomposed = (df.quality.values[0]).decompose() + print(view_summary) + print(view_decomposed) + + return df + + +def parse_cli_args(cli_args: list): + """ + Function that parses CLI arguments + + Parameters + ---------- + cli_args : list + List of string arguments to parse + + Returns + ------- + Namespace + A Namespace object containing the parsed arguments as attributes + """ + parser = argparse.ArgumentParser(description='program arguments.') + + parser.add_argument('--exampleint', + type=int, + default=42, + help='An example integer input option' + ) + + parser.add_argument('--examplefloat', + type=float, + default=3.14159, + help='An example float input option') + + parser.add_argument('--examplebool', + type=bool, + default=True, + help='An example boolean input option') + + parser.add_argument('manifest', + type=str, + help="Input JSON manifest file containing a list of files to use in the processing algorithm.") + + return parser.parse_args(cli_args) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/container_templates/python/container_example/src/static/l2_quality_flags.py b/container_templates/python/container_example/src/static/l2_quality_flags.py new file mode 100644 index 0000000000..23a4e683ef --- /dev/null +++ b/container_templates/python/container_example/src/static/l2_quality_flags.py @@ -0,0 +1,23 @@ +""" +Quality flag example for Level 2 developers + +Authors: Laura Sandoval, Gavin Medley, Matt Watwood +""" +import libera_utils.quality_flags as qf + + +@qf.with_all_none +class L2QualityFlag(qf.QualityFlag, metaclass=qf.FrozenFlagMeta): + """Quality flag for L2 observation""" + INF = qf.FlagBit( + 2**0, # bit 0 + message="Infinite value.") + MISSING_TELEM = qf.FlagBit( + 2**1, # bit 1 + message="Missing telemetry.") + NEG = qf.FlagBit( + 2**2, # bit 2 + message="Negative value.") + UNEXPECTED_TELEM_VALUE_CHANGE = qf.FlagBit( + 2**3, # bit 3 + message="Value changed within the observation that should not have.") diff --git a/container_templates/python/container_example/tests/test_quality_flags.py b/container_templates/python/container_example/tests/test_quality_flags.py new file mode 100644 index 0000000000..0a224c8d31 --- /dev/null +++ b/container_templates/python/container_example/tests/test_quality_flags.py @@ -0,0 +1,13 @@ +"""Test coverage for the static.l2_quality_flags module""" + +import src.static.l2_quality_flags as qf + + +def test_L1QualityFlag(): + """Test behavior of the L2QualityFlag class""" + for f in qf.L2QualityFlag: + assert f.value + assert f.value.message + assert f.summary + + assert len(qf.L2QualityFlag.ALL.summary[1]) == 4