adding container_templates

container_templates/matlab/container_example/.gitignore

@@ -0,0 +1,2 @@
+.idea/
+venv

container_templates/matlab/container_example/Dockerfile

@@ -0,0 +1,49 @@
+# Modified from https://github.com/mathworks-ref-arch/matlab-dockerfile.
+
+# To specify which MATLAB release to install in the container, edit the value of the MATLAB_RELEASE argument.
+ARG MATLAB_RELEASE=r2021b
+
+# When you start the build stage, this Dockerfile by default uses the Ubuntu-based matlab-deps image.
+# To check the available matlab-deps images, see: https://hub.docker.com/r/mathworks/matlab-deps
+FROM mathworks/matlab-deps:${MATLAB_RELEASE}
+
+# Declare the global argument to use at the current build stage
+ARG MATLAB_RELEASE
+
+# Install mpm dependencies
+RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \
+    apt-get install --no-install-recommends --yes \
+        wget \
+        unzip \
+        ca-certificates && \
+    apt-get clean && apt-get autoremove
+
+# Run mpm to install MATLAB in the target location and delete the mpm installation afterwards.
+# If mpm fails to install successfully then output the logfile to the terminal, otherwise cleanup.
+RUN wget -q https://www.mathworks.com/mpm/glnxa64/mpm && \ 
+    chmod +x mpm && \
+    ./mpm install \
+        --release=${MATLAB_RELEASE} \
+        --destination=/opt/matlab \
+        --products MATLAB || \
+    (echo "MPM Installation Failure. See below for more information:" && cat /tmp/mathworks_root.log && false) && \
+    rm -f mpm /tmp/mathworks_root.log && \
+    ln -s /opt/matlab/bin/matlab /usr/local/bin/matlab
+
+# Add "matlab" user and grant sudo permission.
+RUN adduser --shell /bin/bash --disabled-password --gecos "" matlab && \
+    echo "matlab ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/matlab && \
+    chmod 0440 /etc/sudoers.d/matlab
+
+ARG LICENSE_SERVER
+
+# Specify the host and port of the machine that serves the network licenses 
+ENV MLM_LICENSE_FILE=$LICENSE_SERVER
+
+# Set user and work directory
+WORKDIR /home/matlab
+
+# copy the content of the local directory to the working directory
+COPY src/ .
+
+ENTRYPOINT ["matlab", "-r"]

container_templates/matlab/container_example/README.md

@@ -0,0 +1,11 @@
+# Container Example Code
+
+This the basic code for creating a Docker container for the Level 2 pipeline. 
+For questions contact: laura.sandoval@lasp.colorado.edu or contact 
+Laura Sandoval via the libera_l2 Slack channel.
+
+## Setup
+[Here](docs/setup.md)
+
+## Building and running a Docker image
+[Here](docs/docker.md)

container_templates/matlab/container_example/container_example_data/dropbox/input_manifest_20220923t000000.json

@@ -0,0 +1,8 @@
+{
+    "manifest_type": "INPUT",
+    "files": [
+        {"filename": "/opt/data/source_data/filename_example_1.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"},
+        {"filename": "/opt/data/source_data/filename_example_2.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"}
+    ],
+    "configuration": null
+}

container_templates/matlab/container_example/container_example_data/dropbox/input_manifest_20220923t000000_aws.json

@@ -0,0 +1,8 @@
+{
+    "manifest_type": "INPUT",
+    "files": [
+        {"filename": "s3://l0-ingest-dropbox-liberasdplcs-dev/source_data/filename_example_1.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"},
+        {"filename": "s3://l0-ingest-dropbox-liberasdplcs-dev/source_data/filename_example_2.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"}
+    ],
+    "configuration": null
+}

container_templates/matlab/container_example/container_example_data/dropbox/input_manifest_20220923t000000_local.json

@@ -0,0 +1,8 @@
+{
+    "manifest_type": "INPUT",
+    "files": [
+        {"filename": "../container_example_data/source_data/filename_example_1.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"},
+        {"filename": "../container_example_data/source_data/filename_example_2.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"}
+    ],
+    "configuration": null
+}

container_templates/matlab/container_example/docs/docker.md

@@ -0,0 +1,38 @@
+# Building and running a docker image locally
+
+To build the image run the following command from the Dockerfile directory.
+Note: for non-LASP users you must retrieve your MATLAB licensing information
+from your own institute. More information is available here:
+https://github.com/mathworks-ref-arch/matlab-dockerfile 
+
+```shell
+docker build -t matlab-image --rm . --build-arg \
+LICENSE_SERVER=27000@lasp-lmgr.colorado.edu
+```
+
+Now we can run our image using bind mounting. 
+
+# For local directory
+
+```shell
+docker run --rm -it   -e PROCESSING_DROPBOX=/opt/data/dropbox/   \
+--volume="$(pwd)/container_example_data:/opt/data"   matlab-image:latest \
+"algorithm_example /opt/data/dropbox/input_manifest_20220923t000000.json"
+```
+
+# Building and running a docker image in AWS
+
+You will be given AWS console logins in the future. Don't worry about this for now.
+
+In order to push a Docker Image to the ECR, navigate to the AWS Console to 
+retrieve the AWS ECR registry URI:
+
+https://us-west-2.console.aws.amazon.com/ecr/repositories?region=us-west-2
+
+Follow the instructions in the link to push to the AWS ECR.
+
+https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html
+
+The ECR can be browsed through the AWS Console:
+
+https://us-west-2.console.aws.amazon.com/ecr/repositories?region=us-west-2

container_templates/matlab/container_example/docs/setup.md

@@ -0,0 +1,3 @@
+# Environment
+
+Ensure MATLAB >=r2021b is installed for compatibility.

container_templates/matlab/container_example/src/algorithm_example.m

@@ -0,0 +1,137 @@
+% algorithm_example.md - template to read/write to local directory 
+% or S3 bucket
+% ---------------------------------------------------------------------
+%     
+% Example:
+%  
+%  in terminal -
+%  matlab -nodisplay -r "cd src; algorithm_example <manifest_in path>"
+%  
+%  in MATLAB console - 
+%  setenv("PROCESSING_DROPBOX","<dropbox path>");
+%  algorithm_example(<manifest_in path>)
+% 
+% Notes:
+%   - If running from the terminal add
+%   export PROCESSING_DROPBOX="<dropbox path>" to your bash profile
+%   - <path> may either be the path to the local directory e.g.
+%   '../container_example_data/dropbox/input_manifest_20220923t000000_local.json' 
+%   or the path to the s3 bucket (e.g. s3://bucketname/).
+%   - For instructions on using this code with Docker refer to
+%   docker.md
+%
+% Authors: Laura Sandoval, Matt Watwood, Gavin Medley
+%
+% External libraries used:
+%
+% Stefan Stoll (2022). MD5 signature of a file 
+% (https://www.mathworks.com/matlabcentral/fileexchange/5498-md5-signature-of-a-file), 
+% MATLAB Central File Exchange. Retrieved November 9, 2022.
+% 
+% Matthew Spaethe (2022). Matlab logging facility 
+% (https://www.mathworks.com/matlabcentral/fileexchange/42078-matlab-logging-facility), 
+% MATLAB Central File Exchange. Retrieved November 15, 2022.
+% ---------------------------------------------------------------------
+
+function []=algorithm_example(manifest_in, varargin)
+% Requires 1 input and has optional input.
+
+addpath('logging')
+import logging.*
+
+% get the global LogManager
+LogManager.getLogManager();
+
+% add a logger instance to this script
+logger = Logger.getLogger('AlgorithmExample');
+logger.setLevel( Level.ALL );
+
+defaultInt = 2;
+
+logger.info(strcat('Command executed in: ', mfilename('filename'),'.m'));
+logger.info('Parsing CLI arguments.');
+
+p = inputParser;
+addRequired(p,'manifest_in',@ischar);
+addOptional(p,'multiplier',defaultInt,@isnumeric);
+parse(p,manifest_in,varargin{:});
+
+logger.info(strcat('Manifest file to read: ', p.Results.manifest_in));
+logger.info(strcat('Additional options passed are multiplier = ', ...
+    num2str(p.Results.multiplier)));
+
+% only want 1 optional input at most
+numvarargs = length(varargin);
+if numvarargs > 1
+    error('my_script:TooManyInputs', ...
+        'requires at most 1 optional inputs');
+end
+
+processing_dropbox = getenv('PROCESSING_DROPBOX');
+
+str = fileread(p.Results.manifest_in);
+json_content = jsondecode(str);
+
+logger.info(strcat('Manifest type is ', json_content.manifest_type));
+logger.info(strcat('Manifest contains ', json_content.files.filename));
+
+% Get the data from each file and put data into some format that will be 
+% used in the algorithm
+for n = 1 : length(json_content.files)
+
+    checksum = json_content.files(n).checksum;
+    filename = json_content.files(n).filename;
+
+    assert(isequal(checksum,md5(filename)), ...
+        'algorithm_example:checksumError','Checksums do not match!')
+    logger.info(strcat('Checksum matches for ', filename));
+
+  data_in = h5read(filename,'/HDFEOS/SWATHS/Swath1/DataField/Temperature');
+  logger.info(strcat('Found input data in HDF5 file:',mat2str(data_in)))
+
+end
+
+%fake data product to write as output
+data_out = data_in * p.Results.multiplier;
+
+%Create hdf5 data
+output_filepath = append(processing_dropbox, 'example_output.h5');
+h5create(output_filepath,'/data/array1',[height(data_out) width(data_out)]);
+
+logger.info(strcat('Writing output file: ', output_filepath))
+
+h5write(output_filepath, '/data/array1', data_out);
+h5writeatt(output_filepath,'/','someattr','hello, world');
+
+% Need this to create empty group; if you are creating a group 
+% that contains data, then use h5create and h5write
+plist = 'H5P_DEFAULT';
+fid = H5F.open(output_filepath,'H5F_ACC_RDWR',plist);
+gid = H5G.create(fid,'new_group',plist,plist,plist);
+H5G.close(gid);
+H5F.close(fid);
+
+json_out = append(processing_dropbox, 'output_manifest_20220923t111111.json');
+
+logger.info(strcat('Writing output manifest: ', json_out))
+
+%Get checksum of written file
+checksum_written = md5(output_filepath);
+
+s1.filename = output_filepath;
+s1.checksum = checksum_written;
+struct_1 = jsonencode(s1);
+
+s.manifest_type = "OUTPUT";
+s.files = [struct_1];
+s.configuration = string(nan);
+
+fid = fopen(json_out,'w');
+
+fprintf(fid,'%s',strrep(jsonencode(s, "PrettyPrint", true),'\',''));
+fclose(fid);
+
+logger.info('Algorithm complete. Exiting.')
+
+exit
+end

container_templates/matlab/container_example/src/logging/+logging/ConsoleHandler.m

@@ -0,0 +1,35 @@
+classdef ConsoleHandler < logging.Handler
+
+  % Matthew Spaethe
+  % Ideas borrowed from Java logging, morphed into Matlab.
+
+  methods
+    function obj = ConsoleHandler()
+    end
+  end
+
+  methods
+    function [] = emit(obj, level, name, varargin)
+      %emit  Used to pass log message to Handler from Logger.
+
+      import logging.*
+
+      % only print text if local 'logger level' equals or exceeds 'handler level'
+      if level >= obj.level
+        fprintf(1, '%s: ', Level.getName( level ));
+        fprintf(1, '%s: ', name);
+        fprintf(1, varargin{:}{:});
+        fprintf(1, '\n');
+      end
+    end
+
+    function [] = close(obj)
+    end
+
+    function [] = flush(obj)
+    end
+
+  end
+
+end
+

container_templates/matlab/container_example/src/logging/+logging/FileHandler.m

@@ -0,0 +1,74 @@
+classdef FileHandler < logging.Handler
+
+  % Matthew Spaethe
+  % Ideas borrowed from Java logging, morphed into Matlab.
+
+  properties
+    fileId
+    fileOpen = false;
+    filename
+    logger
+  end
+
+  methods
+    function obj = FileHandler(filename)
+      %FileHandler  Handler subclass used to log messages to file.
+      %  FileHandler(filename)  Log output to file, creating / overwriting filename.
+
+      import logging.*
+
+      obj.logger = Logger.getLogger( );
+      obj.logger.setLevel( Level.FINEST );
+      obj.logger.log(Level.FINEST, 'constructor');
+
+      obj.logger.fine('opening file: %s', filename);
+      obj.fileId = fopen(filename, 'w');
+
+      if obj.fileId == -1
+        obj.logger.log(Level.ERROR, 'error in opening file %s', filename);
+        obj.fileOpen = false;
+        obj.filename = [];
+      else
+        obj.fileOpen = true;
+        obj.filename = filename;
+      end
+
+    end
+
+    function delete(obj)
+    end
+
+    function emit(obj, level, name, varargin)
+      %emit  Used to pass log message to Handler from Logger.
+
+      import logging.*
+
+      % only print text if logged message level equals or exceeds 'handler level'
+      if level >= obj.level
+        fprintf(obj.fileId, '%s: ', Level.getName( level ));
+        fprintf(obj.fileId, '%s: ', name);
+        fprintf(obj.fileId, varargin{:}{:});
+        fprintf(obj.fileId, '\n');
+      end
+    end
+
+    function [] = close(obj)
+      %close  Called by Logger when emit() will no longer be called.  Close any relevant i/o.
+      if obj.fileOpen
+        try
+          fclose( obj.fileId );
+          obj.logger.fine('closed file %s', obj.filename);
+          obj.fileOpen = false;
+        catch e
+          obj.logger.error(e.message);
+        end
+      end
+    end
+
+    function [] = flush(obj)
+
+    end
+
+  end
+end
+