From 23f1c40d7ad2ce7aa8f5ce53dd2ae84b4a18f052 Mon Sep 17 00:00:00 2001 From: Matt Thomas Date: Sun, 19 Apr 2020 09:36:40 -0400 Subject: [PATCH 1/5] feat: Upload custom node file --- pyworkflow/pyworkflow/workflow.py | 14 ++++++++++++++ vp/workflow/urls.py | 3 ++- vp/workflow/views.py | 27 +++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/pyworkflow/pyworkflow/workflow.py b/pyworkflow/pyworkflow/workflow.py index 7930beb..964b897 100644 --- a/pyworkflow/pyworkflow/workflow.py +++ b/pyworkflow/pyworkflow/workflow.py @@ -19,9 +19,14 @@ class Workflow: def __init__(self, name="Untitled", root_dir=None, graph=nx.DiGraph(), flow_vars=nx.Graph()): self._name = name self._root_dir = WorkflowUtils.set_root_dir(root_dir) + self._custom_node_dir = WorkflowUtils.set_custom_nodes_dir() self._graph = graph self._flow_vars = flow_vars + @property + def custom_node_dir(self): + return self._custom_node_dir + @property def graph(self): return self._graph @@ -404,6 +409,15 @@ def execute_workflow(workflow_location): class WorkflowUtils: + @staticmethod + def set_custom_nodes_dir(): + custom_node_dir = os.path.join(os.getcwd(), '../pyworkflow/custom_nodes') + + if not os.path.exists(custom_node_dir): + os.makedirs(custom_node_dir) + + return custom_node_dir + @staticmethod def set_root_dir(root_dir): if root_dir is None: diff --git a/vp/workflow/urls.py b/vp/workflow/urls.py index 26f1ad7..dceaca1 100644 --- a/vp/workflow/urls.py +++ b/vp/workflow/urls.py @@ -10,5 +10,6 @@ path('execute//successors', views.get_successors, name='get node successors'), path('globals', views.global_vars, name="retrieve global variables"), path('upload', views.upload_file, name='upload file'), - path('download', views.download_file, name='download file') + path('download', views.download_file, name='download file'), + path('custom_node', views.add_custom_node, name='add custom node') ] diff --git a/vp/workflow/views.py b/vp/workflow/views.py index a76c28c..9ebd497 100644 --- a/vp/workflow/views.py +++ b/vp/workflow/views.py @@ -216,6 +216,33 @@ def get_successors(request, node_id): return JsonResponse(order, safe=False) +@swagger_auto_schema(method='post', + operation_summary='Uploads a custom node to server.', + operation_description='Uploads a custom node to server location.', + responses={ + 200: 'File uploaded', + 404: 'No specified file' + }) +@api_view(['POST']) +def add_custom_node(request): + node_file = request.FILES.get('file') + + if node_file is None: + return JsonResponse("Empty content", status=404) + + to_open = os.path.join(request.pyworkflow.custom_node_dir, node_file.name) + + try: + with open(to_open, 'wb') as f: + f.write(node_file.read()) + except OSError as e: + return JsonResponse({'message': str(e)}, status=500) + + node_file.close() + + return JsonResponse({"filename": to_open}, status=201, safe=False) + + @swagger_auto_schema(method='post', operation_summary='Uploads a file to server.', operation_description='Uploads a new file to server location.', From 3e77c5ac0607d07d86fba86c742f207389b91304 Mon Sep 17 00:00:00 2001 From: Matt Thomas Date: Sun, 19 Apr 2020 09:37:04 -0400 Subject: [PATCH 2/5] feat: Retrieve available custom nodes --- vp/vp/urls.py | 1 + vp/vp/views.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/vp/vp/urls.py b/vp/vp/urls.py index 88caaaf..85bbe74 100644 --- a/vp/vp/urls.py +++ b/vp/vp/urls.py @@ -39,6 +39,7 @@ path('admin/', admin.site.urls), path('info/', views.info), path('nodes/', views.retrieve_nodes_for_user), + path('custom_nodes/', views.retrieve_custom_nodes_for_user), path('node/', include('node.urls')), path('workflow/', include('workflow.urls')) ] diff --git a/vp/vp/views.py b/vp/vp/views.py index 898c095..2c55761 100644 --- a/vp/vp/views.py +++ b/vp/vp/views.py @@ -3,6 +3,9 @@ from drf_yasg.utils import swagger_auto_schema from pyworkflow import Node +import os +import inspect + @swagger_auto_schema(method='get', responses={200:'JSON response with data'}) @api_view(['GET']) @@ -62,4 +65,55 @@ def retrieve_nodes_for_user(request): data[key].append(child_node) - return JsonResponse(data) \ No newline at end of file + return JsonResponse(data) + + +@swagger_auto_schema(method='get', + operation_summary='Retrieve a list of custom Nodes', + operation_description='Retrieves a list of custom Nodes, in JSON.', + responses={ + 200: 'List of installed Nodes, in JSON', + }) +@api_view(['GET']) +def retrieve_custom_nodes_for_user(request): + data = dict() + + # TODO: Workflow loading excluded in middleware for this route + # Should probably have a way to access the 'custom_node` dir dynamically + custom_node_path = os.path.join(os.getcwd(), '../pyworkflow/custom_nodes') + + try: + nodes = os.listdir(custom_node_path) + except OSError as e: + return JsonResponse({"message": str(e)}, status=500) + + for node in nodes: + # Parse file type + node_name, ext = os.path.splitext(node) + + try: + package = __import__('custom_nodes.' + node_name) + module = getattr(package, node_name) + except ModuleNotFoundError as e: + # TODO: This will only catch the first missing package. Can we get more? + data[node_name] = f"Please install missing packages and restart the server. Missing '{e.name}'" + continue + + for name, klass in inspect.getmembers(module): + if inspect.isclass(klass) and klass.__module__.startswith('custom_nodes.'): + custom_node = { + 'name': klass.name, + 'node_key': name, + 'node_type': node_name, + 'num_in': klass.num_in, + 'num_out': klass.num_out, + 'color': klass.color or 'black', + 'doc': klass.__doc__, + 'options': {k: v.get_value() for k, v in klass.options.items()}, + 'option_types': klass.option_types, + 'download_result': getattr(klass, "download_result", False) + } + + data[node_name] = custom_node + + return JsonResponse(data, safe=False) From 8e272686be190d7d46345fcc754892fc3f064cda Mon Sep 17 00:00:00 2001 From: Matt Thomas Date: Sun, 19 Apr 2020 09:37:22 -0400 Subject: [PATCH 3/5] feat: Add custom node creation to `node_factory` --- pyworkflow/pyworkflow/node_factory.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pyworkflow/pyworkflow/node_factory.py b/pyworkflow/pyworkflow/node_factory.py index e295790..3fde202 100644 --- a/pyworkflow/pyworkflow/node_factory.py +++ b/pyworkflow/pyworkflow/node_factory.py @@ -14,7 +14,7 @@ def node_factory(node_info): elif node_type == 'FlowNode': new_node = flow_node(node_key, node_info) else: - new_node = None + new_node = custom_node(node_type, node_key, node_info) return new_node @@ -46,3 +46,15 @@ def manipulation_node(node_key, node_info): return FilterNode(node_info) else: return None + +def custom_node(filename, node_key, node_info): + try: + package = __import__('custom_nodes.' + filename) + module = getattr(package, filename) + my_class = getattr(module, node_key) + instance = my_class(node_info) + + return instance + except Exception as e: + print(str(e)) + return None From 896d3e6285c18ae0a91dec912ae458bb3f9cbd2a Mon Sep 17 00:00:00 2001 From: Matt Thomas Date: Tue, 21 Apr 2020 19:45:27 -0400 Subject: [PATCH 4/5] refactor: Merge node listings to one endpoint Factors out some duplicate code. PR #57 adds a `to_json()` method that might be able to replace the `extract_node_info()` method here. TBD. --- pyworkflow/pyworkflow/node_factory.py | 1 + vp/vp/urls.py | 1 - vp/vp/views.py | 116 ++++++++++++++------------ 3 files changed, 63 insertions(+), 55 deletions(-) diff --git a/pyworkflow/pyworkflow/node_factory.py b/pyworkflow/pyworkflow/node_factory.py index 3fde202..578a5aa 100644 --- a/pyworkflow/pyworkflow/node_factory.py +++ b/pyworkflow/pyworkflow/node_factory.py @@ -47,6 +47,7 @@ def manipulation_node(node_key, node_info): else: return None + def custom_node(filename, node_key, node_info): try: package = __import__('custom_nodes.' + filename) diff --git a/vp/vp/urls.py b/vp/vp/urls.py index 85bbe74..88caaaf 100644 --- a/vp/vp/urls.py +++ b/vp/vp/urls.py @@ -39,7 +39,6 @@ path('admin/', admin.site.urls), path('info/', views.info), path('nodes/', views.retrieve_nodes_for_user), - path('custom_nodes/', views.retrieve_custom_nodes_for_user), path('node/', include('node.urls')), path('workflow/', include('workflow.urls')) ] diff --git a/vp/vp/views.py b/vp/vp/views.py index 2c55761..e8e9f58 100644 --- a/vp/vp/views.py +++ b/vp/vp/views.py @@ -2,9 +2,11 @@ from rest_framework.decorators import api_view from drf_yasg.utils import swagger_auto_schema from pyworkflow import Node +from modulefinder import ModuleFinder import os import inspect +import sys @swagger_auto_schema(method='get', responses={200:'JSON response with data'}) @@ -42,78 +44,84 @@ def retrieve_nodes_for_user(request): """ data = dict() - # Iterate through node 'types' + # Iterate through installed Nodes for parent in Node.__subclasses__(): key = getattr(parent, "display_name", parent.__name__) data[key] = list() # Iterate through node 'keys' for child in parent.__subclasses__(): - # TODO: check attribute-scope is handled correctly - child_node = { - 'name': child.name, - 'node_key': child.__name__, - 'node_type': parent.__name__, - 'num_in': child.num_in, - 'num_out': child.num_out, - 'color': child.color or parent.color, - 'doc': child.__doc__, - 'options': {k: v.get_value() for k, v in child.options.items()}, - 'option_types': child.option_types, - 'download_result': getattr(child, "download_result", False) - } - - data[key].append(child_node) + node = extract_node_info(parent, child) + data[key].append(node) + + # Check for any installed Custom Nodes + # TODO: Workflow loading excluded in middleware for this route + # Should probably have a way to access the 'custom_node` dir dynamically + custom_node_path = os.path.join(os.getcwd(), '../pyworkflow/custom_nodes') + data['CustomNode'] = import_custom_node(custom_node_path) return JsonResponse(data) -@swagger_auto_schema(method='get', - operation_summary='Retrieve a list of custom Nodes', - operation_description='Retrieves a list of custom Nodes, in JSON.', - responses={ - 200: 'List of installed Nodes, in JSON', - }) -@api_view(['GET']) -def retrieve_custom_nodes_for_user(request): - data = dict() +def check_missing_packages(node_path): + finder = ModuleFinder(node_path) + finder.run_script(node_path) + + uninstalled = list() + for missing_package in finder.badmodules.keys(): + if missing_package not in sys.modules: + uninstalled.append(missing_package) + + return uninstalled + + +def extract_node_info(parent, child): + # TODO: check attribute(s) accessing is handled correctly + return { + 'name': child.name, + 'node_key': child.__name__, + 'node_type': str(parent), + 'num_in': child.num_in, + 'num_out': child.num_out, + 'color': child.color or parent.color or 'black', + 'doc': child.__doc__, + 'options': {k: v.get_value() for k, v in child.options.items()}, + 'option_types': child.option_types, + 'download_result': getattr(child, "download_result", False) + } - # TODO: Workflow loading excluded in middleware for this route - # Should probably have a way to access the 'custom_node` dir dynamically - custom_node_path = os.path.join(os.getcwd(), '../pyworkflow/custom_nodes') +def import_custom_node(root_path): + # Get list of files in path try: - nodes = os.listdir(custom_node_path) + files = os.listdir(root_path) except OSError as e: - return JsonResponse({"message": str(e)}, status=500) + return None - for node in nodes: - # Parse file type - node_name, ext = os.path.splitext(node) + data = list() + for file in files: + # Check file is not a dir + node_path = os.path.join(root_path, file) + if not os.path.isfile(node_path): + continue + + node, ext = os.path.splitext(file) try: - package = __import__('custom_nodes.' + node_name) - module = getattr(package, node_name) - except ModuleNotFoundError as e: - # TODO: This will only catch the first missing package. Can we get more? - data[node_name] = f"Please install missing packages and restart the server. Missing '{e.name}'" + package = __import__('custom_nodes.' + node) + module = getattr(package, node) + except ModuleNotFoundError: + data.append({ + "name": node, + "missing_packages": check_missing_packages(node_path) + }) continue for name, klass in inspect.getmembers(module): if inspect.isclass(klass) and klass.__module__.startswith('custom_nodes.'): - custom_node = { - 'name': klass.name, - 'node_key': name, - 'node_type': node_name, - 'num_in': klass.num_in, - 'num_out': klass.num_out, - 'color': klass.color or 'black', - 'doc': klass.__doc__, - 'options': {k: v.get_value() for k, v in klass.options.items()}, - 'option_types': klass.option_types, - 'download_result': getattr(klass, "download_result", False) - } - - data[node_name] = custom_node - - return JsonResponse(data, safe=False) + custom_node = extract_node_info(node, klass) + data.append(custom_node) + + return data + + From 6746f61b21e4d99efd8cf7199fffefb56b1cdec4 Mon Sep 17 00:00:00 2001 From: Matt Thomas Date: Tue, 21 Apr 2020 19:46:43 -0400 Subject: [PATCH 5/5] feat: Add missing package check on upload `check_missing_packages()` is duplicated from `vp/views.py`. It should find a home. --- vp/workflow/views.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/vp/workflow/views.py b/vp/workflow/views.py index 9ebd497..9b3c998 100644 --- a/vp/workflow/views.py +++ b/vp/workflow/views.py @@ -1,5 +1,6 @@ import os import json +import sys from django.http import JsonResponse, HttpResponse from django.conf import settings @@ -7,6 +8,7 @@ from pyworkflow import Workflow, WorkflowException from drf_yasg.utils import swagger_auto_schema +from modulefinder import ModuleFinder @swagger_auto_schema(method='post', operation_summary='Create a new workflow.', @@ -240,7 +242,10 @@ def add_custom_node(request): node_file.close() - return JsonResponse({"filename": to_open}, status=201, safe=False) + return JsonResponse({ + "filename": to_open, + "missing_packages": check_missing_packages(to_open) + }, status=201, safe=False) @swagger_auto_schema(method='post', @@ -302,3 +307,14 @@ def download_file(request): except WorkflowException as e: return JsonResponse({e.action: e.reason}, status=500) + +def check_missing_packages(node_path): + finder = ModuleFinder(node_path) + finder.run_script(node_path) + + uninstalled = list() + for missing_package in finder.badmodules.keys(): + if missing_package not in sys.modules: + uninstalled.append(missing_package) + + return uninstalled