Add Image Support To Deepforge. Closes #1218

src/common/viz/PlotlyDescExtractor.js

@@ -48,11 +48,11 @@ define([], function () {
             ylabel: node.getAttribute('ylabel'),
         };
 
-        const lineIds = node.getChildrenIds();
-        desc.lines = lineIds.map((lineId) => {
-            let lineNode = this._client.getNode(lineId);
-            return this.getLineDesc(lineNode);
-        });
+        const children = node.getChildrenIds().map(id => this._client.getNode(id));
+        desc.lines = children.filter((node) => node.getValidAttributeNames().includes('points'))
+            .map(lineNode => this.getLineDesc(lineNode));
+        desc.images = children.filter(node => node.getValidAttributeNames().includes('rgbaMatrix'))
+            .map(imageNode => this.getImageDesc(imageNode));
         return desc;
     };
 
@@ -86,6 +86,28 @@ define([], function () {
         return desc;
     };
 
+    PlotlyDescExtractor.prototype.getImageDesc = function (imageNode) {
+        const id = imageNode.getId(),
+            subGraphId = imageNode.getParentId(),
+            graphId = this._client.getNode(subGraphId).getParentId(),
+            jobId = this._client.getNode(graphId).getParentId(),
+            execId = this._client.getNode(jobId).getParentId(),
+            imageHeight = imageNode.getAttribute('height'),
+            imageWidth = imageNode.getAttribute('width'),
+            numChannels = imageNode.getAttribute('numChannels');
+        const colorModel = numChannels === 3 ? 'rgb' : 'rgba';
+        return {
+            id: id,
+            execId: execId,
+            subgraphId: this._client.getNode(imageNode.getParentId()).getAttribute('id'),
+            type: 'image',
+            width: imageWidth,
+            height: imageHeight,
+            colorModel: colorModel,
+            visible: imageNode.getAttribute('visible'),
+            rgbaMatrix: base64ToImageArray(imageNode.getAttribute('rgbaMatrix'), imageWidth, imageHeight, numChannels)
+        };
+    };
 
     PlotlyDescExtractor.prototype.compareSubgraphIDs = function (desc1, desc2) {
         if (desc1.subgraphId >= desc2.subgraphId) return 1;
@@ -97,10 +119,10 @@ define([], function () {
         if (desc) {
             plotlyJSON.layout = createLayout(desc);
             let dataArr = desc.subGraphs.map((subGraph, index) => {
-                return createScatterTraces(subGraph, index);
+                return createTraces(subGraph, index);
             });
             plotlyJSON.data = flatten(dataArr);
-            const axesData = addAxesLabels(desc.subGraphs);
+            const axesData = addAxesLabelsAndTicks(desc.subGraphs);
             Object.keys(axesData).forEach((axis) => {
                 plotlyJSON.layout[axis] = axesData[axis];
             });
@@ -152,6 +174,32 @@ define([], function () {
         return layout;
     };
 
+
+    const base64ToImageArray = function (base64String, width, height, numChannels) {
+        const decodedString = atob(base64String);
+        let bytes = new Uint8Array(decodedString.length);
+        for (let i = 0; i < decodedString.length; i++) {
+            bytes[i] = decodedString.charCodeAt(i);
+        }
+        return reshape(bytes, width, height, numChannels);
+    };
+
+    const reshape = function (bytesArray, width, height, numChannels) {
+        let pixelArray = [], oneRow = [], rgbaArray = [];
+        let i, j = 0;
+        for (i = 0; i < height * numChannels; i += numChannels) {
+            while (j < width * numChannels) {
+                pixelArray = Array.from(bytesArray
+                    .slice(i * width + j, i * width + j + numChannels).values());
+                oneRow.push(pixelArray);
+                j += numChannels;
+            }
+            j = 0;
+            rgbaArray.push(oneRow);
+            oneRow = [];
+        }
+        return rgbaArray;
+    };
     // https://github.com/plotly/plotly.js/issues/2746#issuecomment-528342877
     // At present the only hacky way to add subplots title
     const addAnnotations = function (subGraphs) {
@@ -163,7 +211,7 @@ define([], function () {
                 font: {
                     family: 'Arial',
                     color: 'black',
-                    size: 14
+                    size: 12
                 },
                 showarrow: false,
                 xref: 'paper',
@@ -175,7 +223,7 @@ define([], function () {
         });
     };
 
-    const createScatterTraces = function (subGraph, index) {
+    const createTraces = function (subGraph, index) {
         let traceArr = subGraph.lines.map(line => {
             let points = pointsToCartesianArray(line.points);
             let traceData = {
@@ -195,10 +243,23 @@ define([], function () {
             }
             return traceData;
         });
+
+        traceArr.push(...subGraph.images.map(image => {
+            let traceData = {
+                type: TraceTypes.IMAGE,
+                z: image.rgbaMatrix,
+                colormodel: image.colorModel
+            };
+            if (index !== 0) {
+                traceData.xaxis = `x${index + 1}`;
+                traceData.yaxis = `y${index + 1}`;
+            }
+            return traceData;
+        }));
         return traceArr;
     };
 
-    const addAxesLabels = function (subGraphs) {
+    const addAxesLabelsAndTicks = function (subGraphs) {
         let axesData = {};
         subGraphs.forEach((subGraph, index) => {
             let xAxisName = `xaxis${index + 1}`;
@@ -221,6 +282,10 @@ define([], function () {
                     standoff: 0
                 }
             };
+            if (subGraph.images.length >= 1) {
+                axesData[xAxisName].visible = false;
+                axesData[yAxisName].visible = false;
+            }
         });
         return axesData;
     };

src/plugins/ExecuteJob/ExecuteJob.Metadata.js

@@ -13,38 +13,34 @@ define([
         this.createdMetadataIds = {};
         this.subGraphs = {};
         this.plotLines = {};
+        this.images = {};
+    };
+
+    ExecuteJob.prototype.SEPARATORS = {
+        JOB_GRAPH: '/',
+        GRAPH_SUBGRAPH: '/',
+        SUBGRAPH_LINE: '/L',
+        SUBGRAPH_IMAGE: '/I'
+    };
+
+    ExecuteJob.prototype.createId = function (parentId, childRelID, sep) {
+        return parentId + sep + childRelID;
     };
 
     // TODO: Add tests
     ExecuteJob.prototype[CONSTANTS.PLOT_UPDATE] = async function (job, state) {
         const jobId = this.core.getPath(job);
         // Check if the graph already exists
         // use the id to look up the graph
-        let id = jobId + '/' + state.id;
-        let graph = this.getExistingMetadataById(job, 'Graph', id);
-        if (!graph) {
-            graph = this.core.createNode({
-                parent: job,
-                base: this.META.Graph
-            });
-            this.core.setAttribute(graph, 'id', id);
-            this.core.setAttribute(graph, 'title', state.title);
-            this.logger.info(`Adding graph titled ${state.title}`);
-        }
-        this._metadata[id] = graph;
-
-        const subGraphs = await this.core.loadChildren(graph);
-        subGraphs.forEach(subGraph => this.core.deleteNode(this.core.getPath(subGraph)));
-
-        if (this.subGraphs[id])
-            this.subGraphs[id].forEach(subGraphId => this._deleteByMetaDataId(subGraphId));
+        let id = this.createId(jobId, state.id, this.SEPARATORS.JOB_GRAPH);
+        let graph = await this.createGraphNode(job, id, state);
 
         // Apply whatever updates are needed
         // Set the sub-plot title (axes => SubGraph)
         const axeses = state.axes;
         this.subGraphs[id] = [];
         axeses.forEach((axes, index) => {
-            const axesId = id + '/' + index;
+            const axesId = this.createId(id, index, this.SEPARATORS.GRAPH_SUBGRAPH);
             let axesNode = this.getExistingMetadataById(job, 'SubGraph', axesId);
             if (!axesNode) {
                 axesNode = this.core.createNode({
@@ -59,41 +55,94 @@ define([
                 this.core.setAttribute(axesNode, 'ylim', axes.ylim);
                 this.core.setAttribute(axesNode, 'id', axesId);
                 this.logger.info(`Adding subgraph with title ${axes.title}`);
+                // Add Lines
+                this.addAxesLines(axesNode, job, axes, axesId);
+                // Add Images
+                this.addAxesImage(axesNode, job, axes, axesId);
 
-                // Now check for line Nodes
-                const lines = axes.lines;
-                this.plotLines[axesId] = [];
-                lines.forEach((line, index) => {
-                    const lineId = axesId + '/' + index;
-                    let lineNode = this.getExistingMetadataById(job, 'Line', lineId);
-                    if (!lineNode) {
-                        lineNode = this.core.createNode({
-                            parent: axesNode,
-                            base: this.META.Line
-                        });
-                        this.plotLines[axesId].push(lineId);
-                        this.core.setAttribute(lineNode, 'color', line.color);
-                        this.core.setAttribute(lineNode, 'label', line.label || `line ${index + 1}`);
-                        this.core.setAttribute(lineNode, 'lineStyle', line.lineStyle);
-                        this.core.setAttribute(lineNode, 'marker', line.marker);
-                        let points = line.points.map(pts => pts.join(',')).join(';');
-                        this.core.setAttribute(lineNode, 'points', points);
-                        this.core.setAttribute(lineNode, 'lineWidth', line.lineWidth);
-                    }
-                    this._metadata[lineId] = lineNode;
-                });
                 this._metadata[axesId] = axesNode;
             }
         });
     };
 
+    ExecuteJob.prototype.addAxesLines = function (axesNode, job, axes, axesId) {
+        // Check for line Nodes
+        const lines = axes.lines;
+        this.plotLines[axesId] = [];
+        lines.forEach((line, index) => {
+            const lineId = this.createId(axesId, index, this.SEPARATORS.SUBGRAPH_LINE);
+            let lineNode = this.getExistingMetadataById(job, 'Line', lineId);
+            if (!lineNode) {
+                lineNode = this.core.createNode({
+                    parent: axesNode,
+                    base: this.META.Line
+                });
+                this.plotLines[axesId].push(lineId);
+                this.core.setAttribute(lineNode, 'color', line.color);
+                this.core.setAttribute(lineNode, 'label', line.label || `line ${index + 1}`);
+                this.core.setAttribute(lineNode, 'lineStyle', line.lineStyle);
+                this.core.setAttribute(lineNode, 'marker', line.marker);
+                let points = line.points.map(pts => pts.join(',')).join(';');
+                this.core.setAttribute(lineNode, 'points', points);
+                this.core.setAttribute(lineNode, 'lineWidth', line.lineWidth);
+            }
+            this._metadata[lineId] = lineNode;
+        });
+    };
+
+    ExecuteJob.prototype.addAxesImage = function (axesNode, job, axes, axesId) {
+        // Check for Image Nodes
+        const images = axes.images;
+        this.images[axesId] = [];
+        images.forEach((image, index) => {
+            const imageId = this.createId(axesId, index, this.SEPARATORS.SUBGRAPH_IMAGE);
+            let imageNode = this.getExistingMetadataById(job, 'Image', imageId);
+            if (!imageNode) {
+                imageNode = this.core.createNode({
+                    parent: axesNode,
+                    base: this.META.Image
+                });
+                this.logger.debug(`Created Image Node with id ${imageId}`);
+                this.images[axesId].push(imageNode);
+                this.core.setAttribute(imageNode, 'rgbaMatrix', image.rgbaMatrix);
+                this.core.setAttribute(imageNode, 'height', image.height);
+                this.core.setAttribute(imageNode, 'width', image.width);
+                this.core.setAttribute(imageNode, 'visible', image.visible);
+                this.core.setAttribute(imageNode, 'numChannels', image.numChannels);
+            }
+            this._metadata[imageId] = imageNode;
+        });
+    };
+
     ExecuteJob.prototype._deleteByMetaDataId = function (id) {
         if (this._metadata[id]) {
             const nodeId = this.core.getPath(this._metadata[id]);
             this.deleteNode(nodeId);
         }
     };
 
+    ExecuteJob.prototype.createGraphNode = async function (job, id, state) {
+        let graph = this.getExistingMetadataById(job, 'Graph', id);
+        if (!graph) {
+            graph = this.core.createNode({
+                parent: job,
+                base: this.META.Graph
+            });
+            this.core.setAttribute(graph, 'id', id);
+            this.core.setAttribute(graph, 'title', state.title);
+            this.logger.info(`Adding graph titled ${state.title}`);
+        }
+        this._metadata[id] = graph;
+
+        const subGraphs = await this.core.loadChildren(graph);
+        subGraphs.forEach(subGraph => this.core.deleteNode(this.core.getPath(subGraph)));
+
+        if (this.subGraphs[id]) {
+            this.subGraphs[id].forEach(subGraphId => this._deleteByMetaDataId(subGraphId));
+        }
+        return graph;
+    };
+
     ExecuteJob.prototype[CONSTANTS.GRAPH_CREATE_LINE] = function (job, graphId, id) {
         var jobId = this.core.getPath(job),
             graph = this._metadata[jobId + '/' + graphId],

src/plugins/GenerateJob/templates/backend_deepforge.py

@@ -64,9 +64,12 @@
 
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
+import base64
 
 import six
 
+import numpy as np
+
 from matplotlib._pylab_helpers import Gcf
 from matplotlib.backend_bases import (
      FigureCanvasBase, FigureManagerBase, GraphicsContextBase, RendererBase)
@@ -267,6 +270,9 @@ def figure_to_state(self):
             axes_data['xlim'] = axes.get_xlim()
             axes_data['ylim'] = axes.get_ylim()
             axes_data['lines'] = []
+            axes_data['images'] = []
+
+            # Line Data
             for i, line in enumerate(axes.lines):
                 lineDict = {}
                 lineDict['points'] = line.get_xydata().tolist()
@@ -280,9 +286,37 @@ def figure_to_state(self):
                     lineDict['label'] = line.get_label()
 
                 axes_data['lines'].append(lineDict)
+
+            # Image data
+            for i, image in enumerate(axes.images):
+                imageDict = {}
+                properties_dict = image.properties()
+                imageDict['height'] = properties_dict['size'][0]
+                imageDict['width'] = properties_dict['size'][1]
+                imageDict['visible'] = properties_dict['visible']
+                (imageDict['rgbaMatrix'], imageDict['numChannels']) = self.umask_b64_encode(properties_dict['array'])
+
+                axes_data['images'].append(imageDict)
+
             state['axes'].append(axes_data)
         return state
 
+    def umask_b64_encode(self, masked_array):
+        # Unmask invalid data if present
+        if masked_array.mask:
+            masked_array.fill_value = 0
+        image_array = masked_array.filled()
+        if np.all(np.where(image_array <= 1, True, False)):
+            array = (image_array * 255).astype(np.uint8)
+        else:
+            array = image_array.astype(np.uint8)
+        if not array.flags['C_CONTIGUOUS']:  # Needed for base64 encoding
+            array = array.copy(order='c')
+        if len(array.shape) == 2:  # In Case a grayscale Image
+            array = np.stack((array, array, array), axis=-1)
+        encoded_array = base64.b64encode(array)
+        return encoded_array, array.shape[-1]
+
     # You should provide a print_xxx function for every file format
     # you can write.
 

src/seeds/pipeline/version.txt

@@ -1 +1 @@
-0.14.0
+0.15.0