Incorporate Plotly JSON to metamodel (Closes #1745)

environment.worker.yml

@@ -5,3 +5,4 @@ dependencies:
       - pillow
       - matplotlib==3.2.2
       - simplejson
+      - plotly

src/common/updates/Updates.js

@@ -1,13 +1,51 @@
 /* globals define */
 define([
     'deepforge/storage/index',
+    'deepforge/viz/PlotlyDescExtractor',
+    'deepforge/viz/FigureExtractor',
     './Version',
     'q'
 ], function(
     Storage,
+    PlotlyDescExtractor,
+    FigureExtractor,
     Version,
     Q
 ) {
+    const GRAPH = 'Graph';
+    const getGraphNodes = async function(core, rootNode, graphNodes=[]) {
+        const children = await core.loadChildren(rootNode);
+        for(let i = 0; i < children.length; i++) {
+            if (core.getAttribute(children[i], 'name') === GRAPH && !core.isMetaNode(children[i])) {
+                graphNodes.push(children[i]);
+            }
+            await getGraphNodes(core, children[i], graphNodes);
+        }
+    };
+
+    const addMetadataMixinToNodeSubTree = async function(core, META, node) {
+        const METADATA_NODE_PATH =  core.getPath(META['pipeline.Metadata']);
+        const IMPLICIT_OPERATION_NODE = META['pipeline.ImplicitOperation'];
+        const graphNodeChildren = (await core.loadSubTree(node))
+            .filter(node => {
+                return IMPLICIT_OPERATION_NODE ?
+                    !core.isTypeOf(node, IMPLICIT_OPERATION_NODE) :
+                    true;
+            });
+
+        graphNodeChildren.forEach(node => {
+            core.addMixin(node, METADATA_NODE_PATH);
+        });
+    };
+
+    const getPipelineLibraryVersion = function(core, rootNode) {
+        const pipelineRoot = core.getLibraryRoot(rootNode, 'pipeline');
+        const hasPipelineLibrary = !!pipelineRoot;
+        if (hasPipelineLibrary) {
+            const versionString = core.getAttribute(pipelineRoot, 'version');
+            return new Version(versionString);
+        }
+    };
 
     const allUpdates = [
         {
@@ -48,12 +86,9 @@ define([
         {
             name: 'UpdateDataNodesToUserAssets',
             isNeeded: async function(core, rootNode) {
-                const pipelineRoot = core.getLibraryRoot(rootNode, 'pipeline');
-                const hasPipelineLibrary = !!pipelineRoot;
-                if (hasPipelineLibrary) {
-                    const versionString = core.getAttribute(pipelineRoot, 'version');
-                    const version = new Version(versionString);
-                    return version.lessThan(new Version('0.13.0'));
+                const pipelineLibraryVersion = getPipelineLibraryVersion(core, rootNode);
+                if(pipelineLibraryVersion) {
+                    return pipelineLibraryVersion.lessThan(new Version('0.13.0'));
                 }
             },
             apply: async function(core, rootNode, META) {
@@ -72,6 +107,42 @@ define([
                     }
                 }
             }
+        },
+        {
+            name: 'UpdateGraphContainment',
+            beforeLibraryUpdates: true,
+            isNeeded: async function(core, rootNode) {
+                const pipelineLibraryVersion = getPipelineLibraryVersion(core, rootNode);
+                if (pipelineLibraryVersion) {
+                    return pipelineLibraryVersion.lessThan(new Version('0.22.0')) &&
+                           pipelineLibraryVersion.greaterThan(new Version('0.19.1'));
+                }
+            },
+            apply: async function(core, rootNode, META) {
+                let graphNodes = [];
+                await getGraphNodes(core, rootNode, graphNodes);
+                const coreFigureExtractor = new FigureExtractor.CoreFigureExtractor(core, rootNode);
+                const pipelineVersion = getPipelineLibraryVersion(core, rootNode);
+                const shouldAddMetadataMixin = pipelineVersion ?
+                    pipelineVersion.lessThan(new Version('0.21.1')) :
+                    false;
+
+                for (let i = 0; i < graphNodes.length; i++){
+                    const graphNode = graphNodes[i];
+                    if(shouldAddMetadataMixin){
+                        await addMetadataMixinToNodeSubTree(core, META, graphNode);
+                    }
+                    const desc = await coreFigureExtractor.extract(graphNode);
+                    const plotlyJSON = PlotlyDescExtractor.descToPlotlyJSON(desc);
+                    const parentNode = core.getParent(graphNode);
+                    const updatedGraphNode = core.createNode({
+                        parent: parentNode,
+                        base: META['pipeline.Graph']
+                    });
+                    core.setAttribute(updatedGraphNode, 'data', JSON.stringify(plotlyJSON));
+                    core.deleteNode(graphNode);
+                }
+            }
         }
     ];
 

src/plugins/ExecuteJob/metadata/Figure.js

@@ -1,26 +1,12 @@
 /* globals define */
 define([
-    './Metadata',
+    './Metadata'
 ], function(
-    Metadata,
+    Metadata
 ) {
     class Figure extends Metadata {
         async update(state) {
-            this.core.setAttribute(this.node, 'title', state.title);
-            await this.clearSubGraphs();
-
-            state.axes.forEach(axes => {
-                const axesNode = this.core.createNode({
-                    parent: this.node,
-                    base: axes.is3D ? this.META.Plot3D : this.META.Plot2D
-                });
-                this.setAxesProperties(axesNode, axes);
-                this.addAxesLines(axesNode, this.node, axes);
-                if(!axes.is3D){
-                    this.addAxesImage(axesNode, this.node, axes);
-                }
-                this.addAxesScatterPoints(axesNode, this.node, axes);
-            });
+            this.core.setAttribute(this.node, 'data', JSON.stringify(state));
         }
 
         setAxesProperties(axesNode, axes){

src/plugins/GenerateJob/GenerateJob.js

@@ -103,7 +103,7 @@ define([
         this.result.setSuccess(true);
         this.result.addArtifact(hash);
         callback(null, this.result);
-    }; 
+    };
 
     GenerateJob.prototype.createRunScript = async function (files) {
         let runDebug = Templates.RUN_DEBUG;
@@ -291,7 +291,7 @@ define([
         files.addFile('config.json', JSON.stringify(configs));
         files.addFile('start.js', Templates.START);
         files.addFile('utils.build.js', Templates.UTILS);
-        files.addFile('backend_deepforge.py', Templates.MATPLOTLIB_BACKEND);
+        files.addFile('plotly_backend.py', Templates.MATPLOTLIB_BACKEND);
 
         inputs.forEach(pair => {
             const dataInfo = this.core.getAttribute(pair[2], 'data');

src/plugins/GenerateJob/templates/backend_deepforge.py

@@ -354,9 +354,9 @@ def draw(self):
         """
         Draw the figure using the renderer
         """
-        self.send_deepforge_update()
         renderer = RendererTemplate(self.figure.dpi)
         self.figure.draw(renderer)
+        self.send_deepforge_update()
 
     def send_deepforge_update(self):
         state = self.figure_to_state()

src/plugins/GenerateJob/templates/index.js

@@ -4,7 +4,7 @@ define([
     'text!./run-debug.js',
     'text!./main.ejs',
     'text!./deepforge.ejs',
-    'text!./backend_deepforge.py',
+    'text!./plotly_backend.py',
     'text!./deepforge__init__.py',
     'text!./serialize.ejs',
     'text!./deserialize.ejs',