Feature model sciunit integration

Makefile

@@ -1,8 +1,14 @@
-install: create-db install-frontend install-backend
+install: create-db install-sciunit-neuronunit install-frontend install-backend
 	@echo "==========================="
 	@echo "=        Finished         ="
 	@echo "==========================="
 
+install-sciunit-neuronunit:
+	@echo "==========================="
+	@echo "=Install sciunit/neuronunit="
+	@echo "==========================="
+	@./service/scripts/install-sciunit-neuronunit.sh
+
 install-frontend:
 	@echo "==========================="
 	@echo "=    Install frontend     ="
@@ -74,10 +80,10 @@ isort-format:
 	isort --recursive .
 
 yapf-format:
-	yapf -i -r --style .style.yapf -p -e "*/migrations/*.py" -e "env" -e "*/settings.py" .
+	yapf -i -r --style .style.yapf -p -e "*/migrations/*.py" -e "env" -e "*/settings.py" . -e "neuronunit/**" -e "sciunit/**"
 
 yapf-lint:
-	yapf -d -r --style .style.yapf -e "*/migrations/*.py" -e "env" -e "*/settings.py" .
+	yapf -d -r --style .style.yapf -e "*/migrations/*.py" -e "env" -e "*/settings.py" . -e "neuronunit/**" -e "sciunit/**"
 
 generate-tags:
 	ctags -R --exclude=.git --exclude=node_modules --exclude=dist --exclude=env .

docker-compose.yml

@@ -8,7 +8,7 @@ services:
     ports:
         - 5432:5432
   scidash:
-    image: metacell/scidash:3.1.1
+    image: metacell/scidash:4.0.1
     ports:
         - 80:8000
     depends_on:

scidash/account/serializers.py

@@ -6,4 +6,4 @@
 class ScidashUserSerializer(serializers.ModelSerializer):
     class Meta:
         model = ScidashUser
-        fields = '__all__'
+        exclude = ('password', )

scidash/general/helpers.py

@@ -0,0 +1,13 @@
+import importlib
+
+
+def import_class(import_path):
+    splitted = import_path.split('.')
+
+    class_name = splitted[-1:][0]
+    module_path = ".".join(splitted[:-1])
+
+    imported_module = importlib.import_module(module_path)
+    klass = getattr(imported_module, class_name)
+
+    return klass

scidash/main/settings.py

@@ -185,3 +185,6 @@
 
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 POPULATE_USERS = True
+
+DOWNLOADED_MODEL_DIR = os.path.join(BASE_DIR, os.environ.get('STATIC_DIR'), 'models')
+MODEL_DOLL = os.path.join(BASE_DIR, os.environ.get('STATIC_DIR'), 'model_doll.nml')

scidash/main/urls.py

@@ -64,7 +64,9 @@
     url(r'^admin/', admin.site.urls),
     url(r'^api/login/?$', obtain_jwt_token),
     url(r'^data/', include('scidash.general.urls')),
-    url(r'^api/date-range/?$', DateRangeView.as_view(), name='date-range-view'),
+    url(
+        r'^api/date-range/?$', DateRangeView.as_view(), name='date-range-view'
+    ),
     url(r'^api/', include(router.urls)),
     url(r'^auth/', include('django.contrib.auth.urls')),
     url(
@@ -92,7 +94,9 @@
         name='user-info'
     ),
     url(
-        r'^api/users/is-logged/?$', CheckIsLoggedView.as_view(), name='is-logged'
+        r'^api/users/is-logged/?$',
+        CheckIsLoggedView.as_view(),
+        name='is-logged'
     ),
     url(r'^', include('pygeppetto_server.urls')),
 ]

scidash/sciunitmodels/admin.py

@@ -2,7 +2,9 @@
 from django.contrib.contenttypes.admin import GenericStackedInline
 
 from scidash.general.models import Tag
-from scidash.sciunitmodels.models import Capability, ModelClass, ModelInstance
+from scidash.sciunitmodels.models import (
+    Capability, CapabilityModelThrough, ModelClass, ModelInstance
+)
 
 # Register your models here.
 
@@ -20,3 +22,4 @@ class ModelModelAdmin(admin.ModelAdmin):
 admin.site.register(ModelClass)
 admin.site.register(ModelInstance, ModelModelAdmin)
 admin.site.register(Capability)
+admin.site.register(CapabilityModelThrough)

scidash/sciunitmodels/api/views.py

@@ -1,6 +1,6 @@
 from rest_framework import permissions, viewsets
 
-from scidash.sciunitmodels.filters import ModelInstanceFilter
+from scidash.sciunitmodels.filters import ModelClassFilter, ModelInstanceFilter
 from scidash.sciunitmodels.models import Capability, ModelClass, ModelInstance
 from scidash.sciunitmodels.serializers import (
     CapabilitySerializer, ModelClassSerializer, ModelInstanceSerializer
@@ -17,6 +17,7 @@ class ModelClassViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = ModelClass.objects.all()
     serializer_class = ModelClassSerializer
     permission_classes = (permissions.AllowAny, )
+    filter_class = ModelClassFilter
 
 
 class ModelInstanceViewSet(viewsets.ModelViewSet):

scidash/sciunitmodels/filters.py

@@ -1,5 +1,10 @@
+import os
+from urllib.parse import urlparse
+
+from django.conf import settings
 from django_filters import rest_framework as filters
 
+import scidash.sciunitmodels.helpers as helpers
 import scidash.sciunitmodels.models as models
 
 
@@ -27,3 +32,32 @@ class Meta:
         fields = [
             'name', 'class_name', 'tags', 'timestamp_from', 'timestamp_to'
         ]
+
+
+class ModelClassFilter(filters.FilterSet):
+    model_url = filters.CharFilter(method='by_model_url')
+
+    class Meta:
+        model = models.ModelClass
+        fields = [
+            'model_url',
+        ]
+
+    def by_model_url(self, queryset, name, value):
+        model_name = None
+        url = urlparse(value)
+        model_name = os.path.basename(url.path)
+        model_path = os.path.join(settings.DOWNLOADED_MODEL_DIR, model_name)
+
+        helpers.download_and_save_model(model_path, value)
+
+        model_classes = models.ModelClass.objects.filter(
+            import_path__isnull=False
+        )
+
+        matching_classes = [
+            kls.pk for kls in model_classes
+            if helpers.check_capabilities(model_path, kls.import_path)
+        ]
+
+        return queryset.filter(pk__in=matching_classes)

scidash/sciunitmodels/helpers.py

@@ -0,0 +1,35 @@
+import requests
+from django.conf import settings
+
+from scidash.general.helpers import import_class
+
+
+def download_and_save_model(path, url):
+    model_content = requests.get(url)
+
+    with open(path, 'w') as f:
+        f.write(model_content.text)
+
+
+def check_capabilities(model_file_path, model_class_import_path):
+    klass = import_class(model_class_import_path)
+
+    failed = klass(model_file_path).failed_extra_capabilities
+
+    return len(failed) == 0
+
+
+def get_model_capabilities(model_class_import_path):
+    klass = import_class(model_class_import_path)
+
+    doll = settings.MODEL_DOLL
+
+    return klass(doll).capabilities
+
+
+def get_extra_capabilities(model_class_import_path):
+    klass = import_class(model_class_import_path)
+
+    doll = settings.MODEL_DOLL
+
+    return klass(doll).extra_capability_checks

scidash/sciunitmodels/migrations/0012_modelclass_import_path.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.7 on 2019-02-18 13:52
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sciunitmodels', '0011_auto_20190131_1333'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='modelclass',
+            name='import_path',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

scidash/sciunitmodels/migrations/0013_auto_20190221_1626.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.7 on 2019-02-21 16:26
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sciunitmodels', '0012_modelclass_import_path'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CapabilityModelThrough',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('extra_check', models.CharField(max_length=300)),
+                ('capability', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sciunitmodels.Capability')),
+                ('model_class', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sciunitmodels.ModelClass')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='modelclass',
+            name='extra_capabilities',
+            field=models.ManyToManyField(related_name='extra_capabilities_set', through='sciunitmodels.CapabilityModelThrough', to='sciunitmodels.Capability'),
+        ),
+    ]

scidash/sciunitmodels/migrations/0014_auto_20190222_1021.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.7 on 2019-02-22 10:21
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sciunitmodels', '0013_auto_20190221_1626'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='capabilitymodelthrough',
+            options={'verbose_name': 'Capabilities with extra check', 'verbose_name_plural': 'Capabilities with extra checks'},
+        ),
+        migrations.AlterField(
+            model_name='modelclass',
+            name='capabilities',
+            field=models.ManyToManyField(blank=True, related_name='capabilities', to='sciunitmodels.Capability'),
+        ),
+        migrations.AlterField(
+            model_name='modelclass',
+            name='extra_capabilities',
+            field=models.ManyToManyField(blank=True, related_name='capabilities_with_check', through='sciunitmodels.CapabilityModelThrough', to='sciunitmodels.Capability'),
+        ),
+    ]

scidash/sciunitmodels/models.py

@@ -7,6 +7,9 @@
 from django.db import models
 
 from scidash.general import models as general_models
+from scidash.sciunitmodels.helpers import (
+    get_extra_capabilities, get_model_capabilities
+)
 
 # Models Related
 
@@ -21,10 +24,70 @@ def __str__(self):
         return self.class_name
 
 
+class CapabilityModelThrough(models.Model):
+    capability = models.ForeignKey(Capability, on_delete=models.CASCADE)
+    model_class = models.ForeignKey('ModelClass', on_delete=models.CASCADE)
+    extra_check = models.CharField(max_length=300)
+
+    def __str__(self):
+        return f'{self.capability.class_name} with check in {self.extra_check}'
+
+    class Meta:
+        verbose_name = 'Capabilities with extra check'
+        verbose_name_plural = 'Capabilities with extra checks'
+
+
 class ModelClass(models.Model):
     class_name = models.CharField(max_length=50)
-    capabilities = models.ManyToManyField(Capability)
+    capabilities = models.ManyToManyField(
+        Capability, blank=True, related_name='capabilities'
+    )
+    extra_capabilities = models.ManyToManyField(
+        Capability,
+        through=CapabilityModelThrough,
+        related_name='capabilities_with_check',
+        blank=True
+    )
     url = models.URLField(default='', null=True, blank=True, unique=True)
+    import_path = models.TextField(null=True, blank=True)
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+
+        capabilities = []
+        extra_capabilities = []
+
+        try:
+            capabilities = get_model_capabilities(self.import_path)
+            extra_capabilities = get_extra_capabilities(self.import_path)
+        except ImportError:
+            self.memo = f"Can't import {self.import_path}"
+        except AttributeError:
+            self.memo = \
+                f"Wrong class for import {self.import_path}"
+
+        if capabilities is None:
+            self.memo = \
+                f"Wrong class for import capabilities {self.import_path}"
+        elif extra_capabilities is None:
+            self.memo = \
+                f"No extra capabilities check {self.import_path}"
+        else:
+            self.memo = ""
+
+        for capability in capabilities:
+            capability_model, created = Capability.objects.get_or_create(
+                class_name=capability.__name__
+            )
+            if capability not in extra_capabilities:
+                self.capabilities.add(capability_model)
+            else:
+                extra_capability_model, created = CapabilityModelThrough.objects.get_or_create(
+                    capability=capability_model,
+                    model_class=self,
+                    extra_check=extra_capabilities[capability]
+                )
+                extra_capability_model.save()
 
     class Meta:
         verbose_name = 'Model class'

scidash/sciunittests/admin.py

@@ -27,6 +27,7 @@ class TestInstanceModelAdmin(admin.ModelAdmin):
     inlines = [
         TagInline,
     ]
+    list_filter = ["tags__name"]
 
 
 class TestSuiteAdmin(admin.ModelAdmin):

scidash/sciunittests/filters.py

@@ -13,9 +13,7 @@ class TestInstanceFilter(filters.FilterSet):
 
     owner = filters.CharFilter(name='owner__username', lookup_expr='icontains')
 
-    name = filters.CharFilter(
-        name='name', lookup_expr='icontains'
-    )
+    name = filters.CharFilter(name='name', lookup_expr='icontains')
 
     timestamp_from = filters.IsoDateTimeFilter(
         name='timestamp', lookup_expr='gte'