From f00a662e4c5b1c657092ecc8cdc113a777908382 Mon Sep 17 00:00:00 2001 From: Samir Reddigari Date: Sat, 4 Apr 2020 19:09:55 -0400 Subject: [PATCH 1/6] feat: use node ID in filenames; return to client --- vp/workflow/views.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/vp/workflow/views.py b/vp/workflow/views.py index f5fe196..b8a76c0 100644 --- a/vp/workflow/views.py +++ b/vp/workflow/views.py @@ -155,6 +155,7 @@ def execute_workflow(request): return JsonResponse(order, safe=False) + @swagger_auto_schema(method='get', operation_summary='Retrieve sorted list of successors from a node.', operation_description='Retrieves a list of successor nodes, sorted in execution order.', @@ -196,6 +197,7 @@ def retrieve_csv(request, node_id): return response + @swagger_auto_schema(method='post', operation_summary='Uploads a file to server.', operation_description='Uploads a new file to server location.', @@ -207,9 +209,8 @@ def retrieve_csv(request, node_id): def upload_file(request): if 'file' not in request.data: return JsonResponse("Empty content", status=404) - f = request.data['file'] - - fs.save(f.name, f) - - return JsonResponse("File Uploaded", status=201, safe=False) \ No newline at end of file + node_id = request.data.get('nodeId', '') + save_name = f"{node_id}-{f.name}" + fs.save(save_name, f) + return JsonResponse({"filename": save_name}, status=201, safe=False) \ No newline at end of file From 1bd3e4ee21f61a8fa28f43bd1fa971a9deb52b7d Mon Sep 17 00:00:00 2001 From: Samir Reddigari Date: Sat, 4 Apr 2020 20:31:02 -0400 Subject: [PATCH 2/6] feat(ui): add API functions for file upload endpoint --- front-end/src/API.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/front-end/src/API.js b/front-end/src/API.js index 04f8043..3219c18 100644 --- a/front-end/src/API.js +++ b/front-end/src/API.js @@ -148,3 +148,17 @@ export async function addEdge(link) { export async function deleteEdge(link) { return handleEdge(link, "DELETE"); } + + +/** + * Upload a data file to be stored on the server + * @param {FormData} formData - FormData with file and nodeId + * @returns {Promise} - server response + */ +export async function uploadDataFile(formData) { + const options = { + method: "POST", + body: formData + }; + return fetchWrapper("/workflow/upload", options); +} From 1027bfa7913b7d03c53e57c0fa4f28772d66a0fa Mon Sep 17 00:00:00 2001 From: Samir Reddigari Date: Sat, 4 Apr 2020 20:32:11 -0400 Subject: [PATCH 3/6] feat(ui): upload file from node config menu Prevents submission of config form until server responds that file was received. --- .../src/components/CustomNode/NodeConfig.js | 79 +++++++++++++++++-- 1 file changed, 72 insertions(+), 7 deletions(-) diff --git a/front-end/src/components/CustomNode/NodeConfig.js b/front-end/src/components/CustomNode/NodeConfig.js index babb73b..de521d2 100644 --- a/front-end/src/components/CustomNode/NodeConfig.js +++ b/front-end/src/components/CustomNode/NodeConfig.js @@ -1,11 +1,13 @@ -import React, { useRef } from 'react'; +import React, { useRef, useState } from 'react'; import { Modal, Button, Form } from 'react-bootstrap'; import propTypes from 'prop-types'; import * as _ from 'lodash'; +import * as API from "../../API"; -function NodeConfig(props) { +export default function NodeConfig(props) { const form = useRef(); + const [disabled, setDisabled] = useState(false); // confirm, fire delete callback, close modal const handleDelete = () => { @@ -34,7 +36,9 @@ function NodeConfig(props) { { _.map(props.node.configParams, (info, key) => + node={props.node} + value={props.node.config[key]} + disableFunc={setDisabled}/> )} Node Description @@ -44,7 +48,7 @@ function NodeConfig(props) { - + @@ -59,16 +63,23 @@ NodeConfig.propTypes = { toggleShow: propTypes.func, onDelete: propTypes.func, onSubmit: propTypes.func -} +}; function OptionInput(props) { + let inputComp; if (props.type === "file") { - inputComp = ; + inputComp = } else if (props.type === "string") { inputComp = ; + } else if (props.type === "integer") { + inputComp = ; } else { return (<>) } @@ -81,4 +92,58 @@ function OptionInput(props) { ) } -export default NodeConfig; + +function FileUpload(props) { + + const input = useRef(null); + const [fileName, setFileName] = useState(props.value || ""); + const [status, setStatus] = useState(props.value ? "ready" : "unconfigured"); + + const uploadFile = async file => { + props.disableFunc(true); + setStatus("loading"); + const fd = new FormData(); + fd.append("file", file); + fd.append("nodeId", props.node.options.id); + API.uploadDataFile(fd) + .then(resp => { + setFileName(resp.filename); + setStatus("ready"); + props.disableFunc(false); + setStatus("ready"); + }).catch(() => { + setStatus("failed"); + }); + input.current.value = null; + }; + const onFileSelect = e => { + e.preventDefault(); + if (!input.current.files) return; + uploadFile(input.current.files[0]); + }; + + if (status === "loading") return (
Uploading file...
); + const btnText = status === "ready" ? "Choose Different File" : "Choose File"; + let content; + if (status === "ready") { + const rxp = new RegExp(props.node.options.id + '-'); + content = ( +
+ File loaded:  + {fileName.replace(rxp, '')} +
+ ) + } else if (status === "failed") { + content = (
Upload failed. Try a new file.
); + } + return ( + <> + + + + {content} + + ) +} + From 21ae095d428d4106aecca535d761bdb296a662d1 Mon Sep 17 00:00:00 2001 From: Samir Reddigari Date: Sun, 5 Apr 2020 14:48:08 -0400 Subject: [PATCH 4/6] feat: replace file option values with full FileStorage path --- vp/node/views.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vp/node/views.py b/vp/node/views.py index a033964..1713474 100644 --- a/vp/node/views.py +++ b/vp/node/views.py @@ -234,6 +234,12 @@ def create_node(payload): """ json_data = json.loads(payload) + # for options with type 'file', replace value with FileStorage path + for field, info in json_data.get("option_types", dict()).items(): + if info["type"] == "file": + opt_value = json_data["options"][field] + if opt_value is not None: + json_data["options"][field] = fs.path(opt_value) try: return node_factory(json_data) From bd274caa7d9de5c064f1e6fb300f516ac1f35a81 Mon Sep 17 00:00:00 2001 From: Samir Reddigari Date: Sun, 5 Apr 2020 14:48:47 -0400 Subject: [PATCH 5/6] fix: don't pass description as kwarg in ReadCsvNode --- pyworkflow/pyworkflow/node.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyworkflow/pyworkflow/node.py b/pyworkflow/pyworkflow/node.py index b5827e9..1071ff9 100644 --- a/pyworkflow/pyworkflow/node.py +++ b/pyworkflow/pyworkflow/node.py @@ -75,7 +75,7 @@ class ReadCsvNode(IONode): 'filepath_or_buffer': { "type": "file", "name": "File", - "desc": "File to read" + "desc": "CSV File" }, 'sep': { "type": "string", @@ -96,8 +96,9 @@ def execute(self, predecessor_data): try: # TODO: FileStorage implemented in Django to store in /tmp # Better filename/path handling should be implemented. - - df = pd.read_csv(**self.options) + kwargs = self.options.copy() # won't copy nested dicts though + del kwargs["description"] + df = pd.read_csv(**kwargs) return df.to_json() except Exception as e: raise NodeException('read csv', str(e)) From f5e3aeccc70220f730436ced77be52ef68371800 Mon Sep 17 00:00:00 2001 From: Samir Reddigari Date: Sun, 5 Apr 2020 14:49:07 -0400 Subject: [PATCH 6/6] fix(ui): do not mutate node menu item data when rendering --- front-end/src/components/NodeMenu.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/front-end/src/components/NodeMenu.js b/front-end/src/components/NodeMenu.js index 52c0edb..b84d9c0 100644 --- a/front-end/src/components/NodeMenu.js +++ b/front-end/src/components/NodeMenu.js @@ -14,10 +14,11 @@ export default function NodeMenu(props) { {section}
    { _.map(items, item => { - const config = item.options; - delete item.options; + const data = {...item}; // copy so we can mutate + const config = data.options; + delete data.options; return ( - + )} )}