From 953c565d866bc97e27ae414caae546ecd22d5efe Mon Sep 17 00:00:00 2001 From: Matt Thomas Date: Sat, 11 Apr 2020 16:14:59 -0400 Subject: [PATCH 1/9] refactor: Change 'new' endpoint to POST with model.options.id --- front-end/src/API.js | 10 ++++++++-- front-end/src/components/Workspace.js | 4 ++-- vp/workflow/views.py | 11 ++++++++--- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/front-end/src/API.js b/front-end/src/API.js index 93050e4..8df7b4b 100644 --- a/front-end/src/API.js +++ b/front-end/src/API.js @@ -98,10 +98,16 @@ export async function getNodes() { /** * Start a new workflow on the server + * @param {DiagramModel} model - Diagram model * @returns {Promise} - server response */ -export async function initWorkflow() { - return fetchWrapper("/workflow/new"); +export async function initWorkflow(model) { + const options = { + method: "POST", + body: JSON.stringify(model.options.id) + }; + + return fetchWrapper("/workflow/new", options); } diff --git a/front-end/src/components/Workspace.js b/front-end/src/components/Workspace.js index 7f9dca1..5618f96 100644 --- a/front-end/src/components/Workspace.js +++ b/front-end/src/components/Workspace.js @@ -32,7 +32,7 @@ class Workspace extends React.Component { API.getNodes() .then(nodes => this.setState({nodes: nodes})) .catch(err => console.log(err)); - API.initWorkflow().catch(err => console.log(err)); + API.initWorkflow(this.model).catch(err => console.log(err)); } /** @@ -51,7 +51,7 @@ class Workspace extends React.Component { clear() { if (window.confirm("Clear diagram? You will lose all work.")) { this.model.getNodes().forEach(n => n.remove()); - API.initWorkflow().catch(err => console.log(err)); + API.initWorkflow(this.model).catch(err => console.log(err)); this.engine.repaintCanvas(); } } diff --git a/vp/workflow/views.py b/vp/workflow/views.py index c54f370..2b29436 100644 --- a/vp/workflow/views.py +++ b/vp/workflow/views.py @@ -8,13 +8,13 @@ from drf_yasg.utils import swagger_auto_schema -@swagger_auto_schema(method='get', +@swagger_auto_schema(method='post', operation_summary='Create a new workflow.', operation_description='Creates a new workflow with empty DiGraph.', responses={ 200: 'Created new DiGraph' }) -@api_view(['GET']) +@api_view(['POST']) def new_workflow(request): """Create a new workflow. @@ -23,8 +23,13 @@ def new_workflow(request): Return: 200 - Created new DiGraph """ + try: + workflow_id = json.loads(request.body) + except json.JSONDecodeError as e: + return JsonResponse({'No React model ID provided': str(e)}, status=500) + # Create new Workflow - request.pyworkflow = Workflow(root_dir=settings.MEDIA_ROOT) + request.pyworkflow = Workflow(name=workflow_id, root_dir=settings.MEDIA_ROOT) request.session.update(request.pyworkflow.to_session_dict()) return JsonResponse(Workflow.to_graph_json(request.pyworkflow.graph)) From f524304b63df419d07a747c9e9c1af8c95aea6e1 Mon Sep 17 00:00:00 2001 From: Matt Thomas Date: Sat, 11 Apr 2020 16:19:06 -0400 Subject: [PATCH 2/9] refactor: Change format of JSON save data Move all back-end info to 'pyworkflow' key of JSON object. Update 'filename' accessor on Workflow object. --- pyworkflow/pyworkflow/workflow.py | 4 ++++ vp/workflow/views.py | 11 +++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pyworkflow/pyworkflow/workflow.py b/pyworkflow/pyworkflow/workflow.py index effe898..d56593e 100644 --- a/pyworkflow/pyworkflow/workflow.py +++ b/pyworkflow/pyworkflow/workflow.py @@ -116,6 +116,10 @@ def name(self): def name(self, name: str): self._name = name + @property + def filename(self): + return self.name + '.json' + def add_edge(self, node_from: Node, node_to: Node): """ Add a Node object to the graph. diff --git a/vp/workflow/views.py b/vp/workflow/views.py index 2b29436..c7c17de 100644 --- a/vp/workflow/views.py +++ b/vp/workflow/views.py @@ -116,11 +116,14 @@ def save_workflow(request): # serialized graph try: combined_json = json.dumps({ + 'filename': request.pyworkflow.filename, 'react': json.loads(request.body), - 'networkx': Workflow.to_graph_json(request.pyworkflow.graph), - 'flow_vars': Workflow.to_graph_json(request.pyworkflow.flow_vars), - 'filename': request.pyworkflow.file_path, - 'root_dir': request.pyworkflow.root_dir, + 'pyworkflow': { + 'name': request.pyworkflow.name, + 'root_dir': request.pyworkflow.root_dir, + 'graph': Workflow.to_graph_json(request.pyworkflow.graph), + 'flow_vars': Workflow.to_graph_json(request.pyworkflow.flow_vars), + } }) return HttpResponse(combined_json, content_type='application/json') From fa1247d61c145c6b741c170f61d73ce13147e046 Mon Sep 17 00:00:00 2001 From: Matt Thomas Date: Sat, 11 Apr 2020 16:20:15 -0400 Subject: [PATCH 3/9] refactor: Only send 'react' to front-end on 'open' endpoint --- front-end/src/components/Workspace.js | 2 +- vp/workflow/views.py | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/front-end/src/components/Workspace.js b/front-end/src/components/Workspace.js index 5618f96..57c0f32 100644 --- a/front-end/src/components/Workspace.js +++ b/front-end/src/components/Workspace.js @@ -128,7 +128,7 @@ function FileUpload(props) { const form = new FormData(); form.append("file", file); API.uploadWorkflow(form).then(json => { - props.handleData(json.react); + props.handleData(json); }).catch(err => { console.log(err); }); diff --git a/vp/workflow/views.py b/vp/workflow/views.py index c7c17de..d34f202 100644 --- a/vp/workflow/views.py +++ b/vp/workflow/views.py @@ -76,7 +76,9 @@ def open_workflow(request): request.pyworkflow = Workflow.from_request(combined_json['networkx']) request.session.update(request.pyworkflow.to_session_dict()) - react = combined_json['react'] + + # Send back front-end workflow + return JsonResponse(combined_json['react']) except KeyError as e: return JsonResponse({'open_workflow': 'Missing data for ' + str(e)}, status=500) except json.JSONDecodeError as e: @@ -84,12 +86,6 @@ def open_workflow(request): except WorkflowException as e: return JsonResponse({e.action: e.reason}, status=404) - # Construct response - return JsonResponse({ - 'react': react, - 'networkx': Workflow.to_graph_json(request.pyworkflow.graph), - }) - @swagger_auto_schema(method='post', operation_summary='Save workflow to JSON file', From ed2a0760a2bbe597b5aca68923b1ef86e8428fd4 Mon Sep 17 00:00:00 2001 From: Matt Thomas Date: Sat, 11 Apr 2020 16:24:22 -0400 Subject: [PATCH 4/9] refactor: Unify Workflow.from_X methods to load JSON Adds exception handling if any info is missing and combines `from_session`, `from_file`, and `from_request` into a single `from_json` method --- pyworkflow/pyworkflow/workflow.py | 62 +++++++++++-------------------- vp/workflow/middleware.py | 2 +- vp/workflow/views.py | 2 +- 3 files changed, 23 insertions(+), 43 deletions(-) diff --git a/pyworkflow/pyworkflow/workflow.py b/pyworkflow/pyworkflow/workflow.py index d56593e..3afcedc 100644 --- a/pyworkflow/pyworkflow/workflow.py +++ b/pyworkflow/pyworkflow/workflow.py @@ -330,20 +330,18 @@ def retrieve_node_data(node_to_retrieve): raise WorkflowException('retrieve node data', str(e)) @staticmethod - def read_graph_json(file_like): + def read_graph_json(json_data): """Deserialize JSON NetworkX graph Args: - file_like: file-like object from which to read JSON-serialized graph + json_data: JSON data from which to read JSON-serialized graph Returns: NetworkX DiGraph object Raises: - OSError: on file error NetworkXError: on issue with loading JSON graph data """ - json_data = json.load(file_like) return nx.readwrite.json_graph.node_link_graph(json_data) @staticmethod @@ -361,48 +359,30 @@ def generate_file_name(workflow, node_id): return os.path.join(workflow.root_dir, file_name + '-' + str(node_id)) @classmethod - def from_session(cls, data): - """Create instance from graph (JSON) data and filename - - Typically takes Django session as argument, which contains - `graph` and `file_path` keys. + def from_json(cls, json_data): + """Load Workflow from JSON data. Args: - data: dict-like with keys `file_path` and `graph` - """ - file_path = data.get('file_path') - graph_data = data.get('graph') - name = data.get('name') - flow_vars_data = data.get('flow_vars') - root_dir = data.get('root_dir') - - if graph_data is None: - graph = None - else: - graph = nx.readwrite.json_graph.node_link_graph(graph_data) - - if flow_vars_data is None: - flow_vars = None - else: - flow_vars = nx.readwrite.json_graph.node_link_graph(flow_vars_data) - - return cls(graph, file_path, name, flow_vars, root_dir) - - @classmethod - def from_file(cls, file_like): - """ + json_data: JSON-like data from session, or uploaded file - """ - graph = cls.read_graph_json(file_like) - return cls(graph) - - @classmethod - def from_request(cls, json_data): - """ + Returns: + New Workflow object + Raises: + WorkflowException: on missing data (KeyError) or on + malformed NetworkX graph data (NetworkXError) """ - graph = nx.readwrite.json_graph.node_link_graph(json_data) - return cls(graph) + try: + name = json_data['name'] + root_dir = json_data['root_dir'] + graph = Workflow.read_graph_json(json_data['graph']) + flow_vars = Workflow.read_graph_json(json_data['flow_vars']) + + return cls(name=name, root_dir=root_dir, graph=graph, flow_vars=flow_vars) + except KeyError as e: + raise WorkflowException('from_json', str(e)) + except nx.NetworkXError as e: + raise WorkflowException('from_json', str(e)) @staticmethod def to_graph_json(graph): diff --git a/vp/workflow/middleware.py b/vp/workflow/middleware.py index 5425732..6d0fdb4 100644 --- a/vp/workflow/middleware.py +++ b/vp/workflow/middleware.py @@ -25,7 +25,7 @@ def __call__(self, request): pass else: # All other cases, load workflow from session - request.pyworkflow = Workflow.from_session(request.session) + request.pyworkflow = Workflow.from_json(request.session) # Check if a graph is present if request.pyworkflow.graph is None: diff --git a/vp/workflow/views.py b/vp/workflow/views.py index d34f202..e081f44 100644 --- a/vp/workflow/views.py +++ b/vp/workflow/views.py @@ -74,7 +74,7 @@ def open_workflow(request): uploaded_file = request.FILES.get('file') combined_json = json.load(uploaded_file) - request.pyworkflow = Workflow.from_request(combined_json['networkx']) + request.pyworkflow = Workflow.from_json(combined_json['pyworkflow']) request.session.update(request.pyworkflow.to_session_dict()) # Send back front-end workflow From 8f08a0527781de83ed1ac3089bb4c0df151bd5bb Mon Sep 17 00:00:00 2001 From: Matt Thomas Date: Sat, 11 Apr 2020 16:34:13 -0400 Subject: [PATCH 5/9] refactor: Remove unused get/setters. Adds Workflow.path --- pyworkflow/pyworkflow/workflow.py | 69 +++++++++++++------------------ 1 file changed, 29 insertions(+), 40 deletions(-) diff --git a/pyworkflow/pyworkflow/workflow.py b/pyworkflow/pyworkflow/workflow.py index 3afcedc..deb951c 100644 --- a/pyworkflow/pyworkflow/workflow.py +++ b/pyworkflow/pyworkflow/workflow.py @@ -5,54 +5,43 @@ from .node import Node from .node_factory import node_factory -from django.conf import settings - class Workflow: """ Workflow object Attributes: + name: Name of the workflow + root_dir: Used for reading/writing files to/from disk graph: A NetworkX Directed Graph - file_path: Location of a workflow file + flow_vars: Global flow variables associated with workflow """ - def __init__(self, graph=nx.DiGraph(), file_path=None, name='a-name', flow_vars=nx.Graph(), root_dir=settings.MEDIA_ROOT): - #TODO: need to discuss a way to generating the workflow name. For now passing a default name. - self._graph = graph - self._file_path = file_path + def __init__(self, name="Untitled", root_dir=None, graph=nx.DiGraph(), flow_vars=nx.Graph()): self._name = name - self._flow_vars = flow_vars + + if root_dir is None: + root_dir = os.getcwd() if not os.path.exists(root_dir): os.makedirs(root_dir) + self._root_dir = root_dir + self._graph = graph + self._flow_vars = flow_vars + @property def graph(self): return self._graph - @graph.setter - def graph(self, graph): - self._graph = graph - - def path(self, file_name): - return os.path.join(self.root_dir, file_name) + @staticmethod + def path(workflow, file_name): + return os.path.join(workflow.root_dir, file_name) @property def root_dir(self): return self._root_dir - @property - def file_path(self): - return self._file_path - - @file_path.setter - def file_path(self, file_path: str): - if file_path is None or file_path[-5:] == '.json': - self._file_path = file_path - else: - raise WorkflowException('set_file_path', 'File ' + file_path + ' is not JSON.') - @property def flow_vars(self): return self._flow_vars @@ -251,7 +240,7 @@ def execution_order(self): def upload_file(self, uploaded_file, node_id): try: file_name = f"{node_id}-{uploaded_file.name}" - to_open = os.path.join(self.root_dir, file_name) + to_open = Workflow.path(self, file_name) # TODO: Change to a stream/other method for large files? with open(to_open, 'wb') as f: @@ -269,14 +258,13 @@ def download_file(self, node_id): try: # TODO: Change to generic "file" option to allow for more than WriteCsv - to_open = os.path.join(self.root_dir, node.options["path_or_buf"]) + to_open = Workflow.path(self, node.options['path_or_buf']) return open(to_open) except KeyError: raise WorkflowException('download_file', '%s does not have an associated file' % node_id) except OSError as e: raise WorkflowException('download_file', str(e)) - @staticmethod def store_node_data(workflow, node_id, data): """Store Node data @@ -292,9 +280,10 @@ def store_node_data(workflow, node_id, data): """ file_name = Workflow.generate_file_name(workflow, node_id) + file_path = Workflow.path(workflow, file_name) try: - with open(file_name, 'w') as f: + with open(file_path, 'w') as f: f.write(data) return file_name except Exception as e: @@ -348,15 +337,13 @@ def read_graph_json(json_data): def generate_file_name(workflow, node_id): """Generates a file name for saving intermediate execution data. - Current format is workflow_name - node_id + Current format is 'workflow_name - node_id' Args: workflow: the workflow node_id: the id of the workflow """ - #TODO: need to add validation - file_name = workflow.name if workflow.name else 'a-name' - return os.path.join(workflow.root_dir, file_name + '-' + str(node_id)) + return f"{workflow.name}-{node_id}" @classmethod def from_json(cls, json_data): @@ -391,13 +378,15 @@ def to_graph_json(graph): def to_session_dict(self): """Store Workflow information in the Django session. """ - out = dict() - out['graph'] = Workflow.to_graph_json(self.graph) - out['file_path'] = self.file_path - out['name'] = self.name - out['flow_vars'] = Workflow.to_graph_json(self.flow_vars) - out['root_dir'] = self.root_dir - return out + try: + out = dict() + out['name'] = self.name + out['root_dir'] = self.root_dir + out['graph'] = Workflow.to_graph_json(self.graph) + out['flow_vars'] = Workflow.to_graph_json(self.flow_vars) + return out + except nx.NetworkXError as e: + raise WorkflowException('to_session_dict', str(e)) class WorkflowException(Exception): From c230276f4319a8a4451fd4c994250551aa480d15 Mon Sep 17 00:00:00 2001 From: Matt Thomas Date: Sun, 12 Apr 2020 11:06:16 -0400 Subject: [PATCH 6/9] feat: Edit Workflow information (name) --- vp/workflow/urls.py | 1 + vp/workflow/views.py | 27 +++++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/vp/workflow/urls.py b/vp/workflow/urls.py index 3aaa034..26f1ad7 100644 --- a/vp/workflow/urls.py +++ b/vp/workflow/urls.py @@ -4,6 +4,7 @@ urlpatterns = [ path('new', views.new_workflow, name='new workflow'), path('open', views.open_workflow, name='open workflow'), + path('edit', views.edit_workflow, name='edit workflow'), path('save', views.save_workflow, name='save'), path('execute', views.execute_workflow, name='execute workflow'), path('execute//successors', views.get_successors, name='get node successors'), diff --git a/vp/workflow/views.py b/vp/workflow/views.py index e081f44..a76c28c 100644 --- a/vp/workflow/views.py +++ b/vp/workflow/views.py @@ -54,12 +54,17 @@ def open_workflow(request): request: Django request Object, should follow the pattern: { react: {react-diagrams JSON}, - networkx: {networkx graph as JSON}, + pyworkflow: { + name: Workflow name, + root_dir: File storage, + graph: Computational graph, + flow_vars: Global flow variables, + }, } Raises: JSONDecodeError: invalid JSON data - KeyError: request missing either 'react' or 'networkx' data + KeyError: request missing either 'react' or 'pyworkflow' data WorkflowException: error loading JSON into NetworkX DiGraph Returns: @@ -87,6 +92,24 @@ def open_workflow(request): return JsonResponse({e.action: e.reason}, status=404) +@swagger_auto_schema(method='post', + operation_summary='Edit workflow information', + operation_description='Edits workflow information.', + responses={ + 200: 'Workflow info updated', + 500: 'No valid JSON in request body' + }) +@api_view(['POST']) +def edit_workflow(request): + try: + json_data = json.loads(request.body) + request.pyworkflow.name = json_data['name'] + + return JsonResponse({'message': 'Workflow successfully updated.'}) + except Exception as e: + return JsonResponse({'message': str(e)}, status=404) + + @swagger_auto_schema(method='post', operation_summary='Save workflow to JSON file', operation_description='Saves workflow to JSON file for download.', From 8ffda9b31db577dcbd28fecc90172edea0b07aab Mon Sep 17 00:00:00 2001 From: Matt Thomas Date: Sun, 12 Apr 2020 11:06:57 -0400 Subject: [PATCH 7/9] fix: Workflow.path call, handle middleware exception --- vp/node/views.py | 2 +- vp/workflow/middleware.py | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/vp/node/views.py b/vp/node/views.py index f365ad4..91ea9e2 100644 --- a/vp/node/views.py +++ b/vp/node/views.py @@ -263,7 +263,7 @@ def create_node(request): if info["type"] == "file" or info["name"] == "Filename": opt_value = json_data["options"][field] if opt_value is not None: - json_data["options"][field] = request.pyworkflow.path(opt_value) + json_data["options"][field] = Workflow.path(request.pyworkflow, opt_value) try: return node_factory(json_data) diff --git a/vp/workflow/middleware.py b/vp/workflow/middleware.py index 6d0fdb4..f6b0893 100644 --- a/vp/workflow/middleware.py +++ b/vp/workflow/middleware.py @@ -1,4 +1,4 @@ -from pyworkflow import Workflow +from pyworkflow import Workflow, WorkflowException from django.http import JsonResponse @@ -25,13 +25,16 @@ def __call__(self, request): pass else: # All other cases, load workflow from session - request.pyworkflow = Workflow.from_json(request.session) - - # Check if a graph is present - if request.pyworkflow.graph is None: - return JsonResponse({ - 'message': 'A workflow has not been created yet.' - }, status=404) + try: + request.pyworkflow = Workflow.from_json(request.session) + + # Check if a graph is present + if request.pyworkflow.graph is None: + return JsonResponse({ + 'message': 'A workflow has not been created yet.' + }, status=404) + except WorkflowException as e: + return JsonResponse({e.action: e.reason}, status=500) response = self.get_response(request) From 394498b929f6072c9ec26556c026e25af79d56a5 Mon Sep 17 00:00:00 2001 From: Matt Thomas Date: Sun, 12 Apr 2020 12:50:18 -0400 Subject: [PATCH 8/9] fix: Make `path` instance method --- pyworkflow/pyworkflow/workflow.py | 11 +++++------ vp/node/views.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pyworkflow/pyworkflow/workflow.py b/pyworkflow/pyworkflow/workflow.py index deb951c..55c5fb3 100644 --- a/pyworkflow/pyworkflow/workflow.py +++ b/pyworkflow/pyworkflow/workflow.py @@ -34,9 +34,8 @@ def __init__(self, name="Untitled", root_dir=None, graph=nx.DiGraph(), flow_vars def graph(self): return self._graph - @staticmethod - def path(workflow, file_name): - return os.path.join(workflow.root_dir, file_name) + def path(self, file_name): + return os.path.join(self.root_dir, file_name) @property def root_dir(self): @@ -240,7 +239,7 @@ def execution_order(self): def upload_file(self, uploaded_file, node_id): try: file_name = f"{node_id}-{uploaded_file.name}" - to_open = Workflow.path(self, file_name) + to_open = self.path(file_name) # TODO: Change to a stream/other method for large files? with open(to_open, 'wb') as f: @@ -258,7 +257,7 @@ def download_file(self, node_id): try: # TODO: Change to generic "file" option to allow for more than WriteCsv - to_open = Workflow.path(self, node.options['path_or_buf']) + to_open = self.path(node.options['path_or_buf']) return open(to_open) except KeyError: raise WorkflowException('download_file', '%s does not have an associated file' % node_id) @@ -280,7 +279,7 @@ def store_node_data(workflow, node_id, data): """ file_name = Workflow.generate_file_name(workflow, node_id) - file_path = Workflow.path(workflow, file_name) + file_path = workflow.path(file_name) try: with open(file_path, 'w') as f: diff --git a/vp/node/views.py b/vp/node/views.py index 91ea9e2..f365ad4 100644 --- a/vp/node/views.py +++ b/vp/node/views.py @@ -263,7 +263,7 @@ def create_node(request): if info["type"] == "file" or info["name"] == "Filename": opt_value = json_data["options"][field] if opt_value is not None: - json_data["options"][field] = Workflow.path(request.pyworkflow, opt_value) + json_data["options"][field] = request.pyworkflow.path(opt_value) try: return node_factory(json_data) From ab7ea538ccfa301ad494fd0b4220867b3050c819 Mon Sep 17 00:00:00 2001 From: Matt Thomas Date: Sun, 12 Apr 2020 18:20:44 -0400 Subject: [PATCH 9/9] refactor: Move 'root_dir' logic to WorkflowUtils --- pyworkflow/pyworkflow/workflow.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/pyworkflow/pyworkflow/workflow.py b/pyworkflow/pyworkflow/workflow.py index 55c5fb3..c540b4e 100644 --- a/pyworkflow/pyworkflow/workflow.py +++ b/pyworkflow/pyworkflow/workflow.py @@ -18,15 +18,7 @@ class Workflow: def __init__(self, name="Untitled", root_dir=None, graph=nx.DiGraph(), flow_vars=nx.Graph()): self._name = name - - if root_dir is None: - root_dir = os.getcwd() - - if not os.path.exists(root_dir): - os.makedirs(root_dir) - - self._root_dir = root_dir - + self._root_dir = WorkflowUtils.set_root_dir(root_dir) self._graph = graph self._flow_vars = flow_vars @@ -388,6 +380,18 @@ def to_session_dict(self): raise WorkflowException('to_session_dict', str(e)) +class WorkflowUtils: + @staticmethod + def set_root_dir(root_dir): + if root_dir is None: + root_dir = os.getcwd() + + if not os.path.exists(root_dir): + os.makedirs(root_dir) + + return root_dir + + class WorkflowException(Exception): def __init__(self, action: str, reason: str): self.action = action