File upload support

MirMachineWebapp/settings.py

@@ -39,15 +39,13 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'lookupService.apps.LookupserviceConfig',
-    'corsheaders',
     'manifest_loader',
     'rest_framework',
 ]
 
 MIDDLEWARE = [
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
-    'corsheaders.middleware.CorsMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
@@ -133,5 +131,3 @@
 # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
 
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
-
-CORS_ALLOWED_ORIGINS = ['https://localhost:3000']

lookupService/frontend/package.json

@@ -4,7 +4,11 @@
   "private": true,
   "dependencies": {
     "@babel/runtime": "^7.15.3",
+    "@sb1/ffe-core": "^19.1.0",
+    "@sb1/ffe-core-react": "^4.2.5",
     "@sb1/ffe-icons-react": "^7.2.18",
+    "@sb1/ffe-spinner": "^4.0.0",
+    "@sb1/ffe-spinner-react": "^5.0.3",
     "@testing-library/jest-dom": "^5.14.1",
     "@testing-library/react": "^11.2.7",
     "@testing-library/user-event": "^12.8.3",

lookupService/frontend/src/App.less

@@ -257,4 +257,9 @@
   padding: 10px;
   width: 100%;
   justify-content: center;
+}
+
+.spinner::before{
+  border-color: @main-green !important;
+  border-top-color: transparent !important;
 }

lookupService/frontend/src/components/Form.js

@@ -1,6 +1,7 @@
 import { useState, useEffect } from "react";
 import { ChevronIkon, ForstorrelsesglassIkon } from '@sb1/ffe-icons-react'
-import { fetchTree, submitJob, getFamilies, getFamiliesIncludedInSearch } from '../utils/Repository'
+import Spinner from '@sb1/ffe-spinner-react'
+import { fetchTree, submitJob, getFamilies, getFamiliesIncludedInSearch, validData } from '../utils/Repository'
 import { Redirect } from 'react-router-dom'
 import Tree from './Tree'
 import SearchableDropdown from './SearchableDropdown'
@@ -12,6 +13,7 @@ export const SearchForm = () => {
     const [optionalActive, setOptionalActive] = useState(false)
     const [modal, setModal] = useState(false)
     const [showIncluded, setShowIncluded] = useState(false)
+    const [submitting, setSubmitting] = useState(false)
 
     // Form data
     const [inputMode, setInputMode] = useState("text")
@@ -49,6 +51,13 @@ export const SearchForm = () => {
     },[])
 
     const handleSubmit = async () => {
+        setSubmitting(true)
+        let mode = document.getElementById('mode').value
+        let file
+        if(mode == 'file'){
+            file = document.getElementById('sequence').files[0]
+            document.getElementById('sequence').value = ''
+        }
         const data = {
             data: document.getElementById('sequence').value,
             mode: document.getElementById('mode').value,
@@ -60,8 +69,21 @@ export const SearchForm = () => {
             family: singleFam ? selectedFamily : '',
             mail_address: document.getElementById('email').value
         }
-        const response = await submitJob(data)
-        setRedirect(response.id)
+
+        if(validData(data, file)) {
+            try {
+                const response = await submitJob(data, file)
+                setRedirect(response.id)
+            } catch (e) {
+                if (e.name == 'JobPostError') {
+                    alert(e.message)
+                }
+                setSubmitting(false)
+            }
+        } else {
+            alert('Required fields are missing')
+            setSubmitting(false)
+        }
     }
 
     const handleIncludedFamilyFetching = async (refresh) => {
@@ -93,6 +115,9 @@ export const SearchForm = () => {
         <form className={'flex-column limit-width'}
               name={'query'} id={'query'} onSubmit={event => event.preventDefault()}>
             {modal && <Tree hook={setNode} show={setModal} nodes={nodes} edges={edges}/>}
+            {submitting ? <span className={'default-margins'}>
+                <Spinner className='spinner' large={true}/></span> :
+            <>
             <span className={'input-cell'}>
                     <label className={'label'} htmlFor={'sequence'}>Sequence:</label>
                 { inputMode === 'text' ?
@@ -101,7 +126,7 @@ export const SearchForm = () => {
                     <input
                         type={inputMode === 'file' ? 'file' : 'text'}
                         placeholder={`Input ${inputMode === 'link' ? 'link' : 'accession number'} here`}
-                        name={'sequence'}
+                        name={'sequence'} accept={'.txt,.fa,.fasta'}
                         id={'sequence'}
                     />
                 }
@@ -170,6 +195,7 @@ export const SearchForm = () => {
                 includedFamilies={includedFamilies}
                 showIncluded={showIncluded}
             />
+            </>}
             {redirect && <Redirect to={`/job/${redirect}`}/>}
         </form>
     )

lookupService/frontend/src/styles.less

@@ -1,4 +1,6 @@
+@import '~@sb1/ffe-core/less/ffe';
+@import '~@sb1/ffe-spinner/less/spinner';
 @import 'App';
 @import 'index';
 @import './components/Job';
-@import './components/Tree';
+@import './components/Tree';

lookupService/frontend/src/utils/Repository.js

@@ -1,23 +1,33 @@
 const baseURL = 'http://localhost:8000/api/'
 
-export const submitJob = async (data) => {
+export const submitJob = async (data, file) => {
     const csrftoken = getCookie('csrftoken')
+
+    let formData = new FormData()
     const jsonString = JSON.stringify(data,null, '    ')
-    console.log(jsonString)
+    formData.append('data', jsonString)
+
+    if(data.mode == 'file'){
+        if(!file) throw new JobPostError('Please select a file to upload')
+        formData.append('file', file)
+    }
+
 
     const response = await fetch(baseURL + 'jobs/',{
         method: 'POST',
         mode: 'same-origin',
         cache: 'no-cache',
         credentials: 'same-origin',
         headers: {
-            'Content-Type': 'application/json',
             'Accept': '*/*',
             'X-CSRFToken': csrftoken,
             'Accept-Encoding': 'gzip, deflate, br'
         },
-        body: jsonString
+        body: formData
         });
+    if(response.status === 400){
+        throw new JobPostError('Invalid data, make sure you filled out the necessary fields')
+    }
     return response.json()
 }
 
@@ -127,3 +137,17 @@ export class JobFetchError extends Error {
         this.name = "JobFetchError"
     }
 }
+
+export class JobPostError extends Error {
+    constructor(message) {
+        super(message)
+        this.name = "JobPostError"
+    }
+}
+
+export const validData = (data, file) => {
+    if(data.data === '' && !file) return false
+    else if(data.single_fam_mode && data.family === '') return false
+    else if(!data.single_fam_mode && data.node === '') return false
+    return true
+}

lookupService/job_pre_processor.py

@@ -0,0 +1,27 @@
+import hashlib
+import json
+from lookupService.serializers import JobSerializer
+
+
+def process_form_data(request):
+    serializer = JobSerializer(data=json.loads(request.POST.get('data')))
+    if serializer.initial_data['mode'] == 'file':
+        file = request.FILES.get('file')
+        fasta = file.name.endswith(('.fa', '.fasta'))
+        parsed_header = False
+        _list = []
+        for chunk in file.chunks():
+            decoded = chunk.decode('utf-8')
+            if fasta and not parsed_header:
+                if serializer.initial_data['species'] == '':
+                    lines = decoded.splitlines()
+                    serializer.initial_data['species'] = lines[0][1:]
+                    _list.append(''.join(lines[1:]))
+                parsed_header = True
+                continue
+            _list.append(decoded)
+        serializer.initial_data['data'] = ''.join(_list)
+    serializer.initial_data['hash'] = hashlib.md5(serializer.initial_data['data'].encode()).hexdigest()
+    return serializer
+
+

lookupService/migrations/0015_auto_20210923_1219.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.6 on 2021-09-23 12:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('lookupService', '0014_auto_20210915_1132'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='job',
+            name='data',
+            field=models.CharField(max_length=1000000),
+        ),
+        migrations.AlterField(
+            model_name='job',
+            name='mail_address',
+            field=models.EmailField(blank=True, max_length=254),
+        ),
+        migrations.AlterField(
+            model_name='job',
+            name='node',
+            field=models.CharField(blank=True, max_length=100),
+        ),
+    ]

lookupService/migrations/0016_alter_job_data.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.6 on 2021-09-23 12:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('lookupService', '0015_auto_20210923_1219'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='job',
+            name='data',
+            field=models.CharField(max_length=500000),
+        ),
+    ]

lookupService/migrations/0017_alter_job_data.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.6 on 2021-09-23 12:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('lookupService', '0016_alter_job_data'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='job',
+            name='data',
+            field=models.TextField(),
+        ),
+    ]

lookupService/models.py

@@ -1,5 +1,6 @@
 from django.db import models
 from django.utils import timezone
+from django.template.defaultfilters import truncatechars
 import uuid
 # Create your models here.
 
@@ -39,15 +40,15 @@ class Job(models.Model):
     status = models.CharField(choices=STATUSES, max_length=15, default=status_default)
     hash = models.CharField(max_length=168, blank=True)
     initiated = models.DateTimeField(auto_now_add=True)
-    data = models.CharField(max_length=10000)
+    data = models.TextField()
     mode = models.CharField(choices=MODE_OPTIONS, max_length=10)
     species = models.CharField(blank=True, max_length=60, default=species_default)
-    node = models.CharField(max_length=100)
+    node = models.CharField(max_length=100, blank=True)
     model_type = models.CharField(choices=MODEL_TYPES, max_length=15)
     single_node = models.BooleanField(default=boolean_default)
     single_fam_mode = models.BooleanField(default=boolean_default)
     family = models.CharField(blank=True, max_length=18)
-    mail_address = models.CharField(blank=True, max_length=100)
+    mail_address = models.EmailField(max_length=254, blank=True)
 
 
 class Node(models.Model):

lookupService/serializers.py

@@ -2,14 +2,16 @@
 from .models import Job, Node, Edge, Family, NodeFamilyRelation
 
 
+class StrippedJobSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Job
+        exclude = ['data']
+
+
 class JobSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Job
-        fields = ['id', 'initiated', 'status',
-                  'data', 'hash', 'mode',
-                  'species', 'node',
-                  'model_type', 'single_node', 'family',
-                  'single_fam_mode', 'mail_address']
+        fields = '__all__'
 
 
 class NodeSerializer(serializers.HyperlinkedModelSerializer):

lookupService/urls.py

@@ -4,7 +4,7 @@
 
 urlpatterns = [
     path('jobs/', post_job),
-    path('job/<str:id>', get_job),
+    path('job/<str:_id>', get_job),
     path('tree/', get_tree),
     path('families/', get_families),
     path('relations/', get_included_families)