From ceacd0aee5e32010840960e428bb6533779d7daf Mon Sep 17 00:00:00 2001 From: NeZanyat Date: Thu, 14 Feb 2019 20:43:09 +0200 Subject: [PATCH 1/7] Sciunit integration --- scidash/main/urls.py | 8 +++-- scidash/sciunittests/filters.py | 4 +-- scidash/sciunittests/helpers.py | 33 ++++++++++++++++++ .../migrations/0027_auto_20190212_1443.py | 31 +++++++++++++++++ .../0028_testclass_test_parameters_schema.py | 21 ++++++++++++ scidash/sciunittests/models.py | 34 +++++++++++++++++++ 6 files changed, 126 insertions(+), 5 deletions(-) create mode 100644 scidash/sciunittests/helpers.py create mode 100644 scidash/sciunittests/migrations/0027_auto_20190212_1443.py create mode 100644 scidash/sciunittests/migrations/0028_testclass_test_parameters_schema.py diff --git a/scidash/main/urls.py b/scidash/main/urls.py index 55943018..d9550f9e 100644 --- a/scidash/main/urls.py +++ b/scidash/main/urls.py @@ -64,7 +64,9 @@ url(r'^admin/', admin.site.urls), url(r'^api/login/?$', obtain_jwt_token), url(r'^data/', include('scidash.general.urls')), - url(r'^api/date-range/?$', DateRangeView.as_view(), name='date-range-view'), + url( + r'^api/date-range/?$', DateRangeView.as_view(), name='date-range-view' + ), url(r'^api/', include(router.urls)), url(r'^auth/', include('django.contrib.auth.urls')), url( @@ -92,7 +94,9 @@ name='user-info' ), url( - r'^api/users/is-logged/?$', CheckIsLoggedView.as_view(), name='is-logged' + r'^api/users/is-logged/?$', + CheckIsLoggedView.as_view(), + name='is-logged' ), url(r'^', include('pygeppetto_server.urls')), ] diff --git a/scidash/sciunittests/filters.py b/scidash/sciunittests/filters.py index bc66afdc..0041ca29 100644 --- a/scidash/sciunittests/filters.py +++ b/scidash/sciunittests/filters.py @@ -13,9 +13,7 @@ class TestInstanceFilter(filters.FilterSet): owner = filters.CharFilter(name='owner__username', lookup_expr='icontains') - name = filters.CharFilter( - name='name', lookup_expr='icontains' - ) + name = filters.CharFilter(name='name', lookup_expr='icontains') timestamp_from = filters.IsoDateTimeFilter( name='timestamp', lookup_expr='gte' diff --git a/scidash/sciunittests/helpers.py b/scidash/sciunittests/helpers.py new file mode 100644 index 00000000..4f25f803 --- /dev/null +++ b/scidash/sciunittests/helpers.py @@ -0,0 +1,33 @@ +import importlib + + +def get_observation_schema(import_path): + splitted = import_path.split('.') + + class_name = splitted[-1:][0] + module_path = ".".join(splitted[:-1]) + + imported_module = importlib.import_module(module_path) + klass = getattr(imported_module, class_name) + + observation_schema = klass.observation_schema + result = {} + + for schema in observation_schema: + result = {**result, **schema} + + return result + + +def get_test_parameters_schema(import_path): + splitted = import_path.split('.') + + class_name = splitted[-1:][0] + module_path = ".".join(splitted[:-1]) + + imported_module = importlib.import_module(module_path) + klass = getattr(imported_module, class_name) + + params_schema = klass.params_schema + + return params_schema diff --git a/scidash/sciunittests/migrations/0027_auto_20190212_1443.py b/scidash/sciunittests/migrations/0027_auto_20190212_1443.py new file mode 100644 index 00000000..2658a305 --- /dev/null +++ b/scidash/sciunittests/migrations/0027_auto_20190212_1443.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2019-02-12 14:43 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sciunittests', '0026_auto_20190206_1329'), + ] + + operations = [ + migrations.AddField( + model_name='testclass', + name='import_path', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='testclass', + name='memo', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='testclass', + name='observation_schema', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True), + ), + ] diff --git a/scidash/sciunittests/migrations/0028_testclass_test_parameters_schema.py b/scidash/sciunittests/migrations/0028_testclass_test_parameters_schema.py new file mode 100644 index 00000000..9d558120 --- /dev/null +++ b/scidash/sciunittests/migrations/0028_testclass_test_parameters_schema.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2019-02-13 13:49 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('sciunittests', '0027_auto_20190212_1443'), + ] + + operations = [ + migrations.AddField( + model_name='testclass', + name='test_parameters_schema', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True), + ), + ] diff --git a/scidash/sciunittests/models.py b/scidash/sciunittests/models.py index 1766c8c7..d3c754af 100644 --- a/scidash/sciunittests/models.py +++ b/scidash/sciunittests/models.py @@ -4,9 +4,12 @@ from django.contrib.contenttypes.fields import GenericRelation from django.contrib.postgres.fields import HStoreField, JSONField from django.db import models +from django.db.models.signals import post_save +from django.dispatch import receiver import scidash.sciunitmodels as sciunitmodels from scidash.general import models as general_models +from scidash.sciunittests.helpers import get_observation_schema, get_test_parameters_schema logger = logging.getLogger(__name__) @@ -28,6 +31,10 @@ def __str__(self): class TestClass(models.Model): class_name = models.CharField(max_length=50) url = models.URLField(default='', null=True, blank=True) + import_path = models.TextField(null=True, blank=True) + observation_schema = JSONField(null=True, blank=True) + test_parameters_schema = JSONField(null=True, blank=True) + memo = models.TextField(null=True, blank=True) class Meta: verbose_name = 'Test class' @@ -36,6 +43,33 @@ class Meta: def __str__(self): return self.class_name + def clean_fields(self, exclude=None): + super().clean_fields(exclude=exclude) + + observation_schema = None + params_schema = None + + try: + observation_schema = get_observation_schema(self.import_path) + params_schema = get_test_parameters_schema(self.import_path) + except ImportError: + self.memo = f"Can't import {self.import_path}" + except AttributeError: + self.memo = \ + f"Wrong class for import {self.import_path}" + + if observation_schema is None: + self.memo = \ + f"Wrong class for import observations {self.import_path}" + elif params_schema is None: + self.memo = \ + f"Wrong class for import params {self.import_path}" + else: + self.memo = "" + + self.observation_schema = observation_schema + self.test_parameters_schema = params_schema + class TestInstance(models.Model): name = models.CharField(max_length=200, default='Default Name') From 0a5393090c3fb16231dc9bc9f335c06f4cde7081 Mon Sep 17 00:00:00 2001 From: NeZanyat Date: Fri, 15 Feb 2019 14:21:57 +0200 Subject: [PATCH 2/7] Some updates for sciunit integration --- scidash/account/serializers.py | 2 +- scidash/sciunittests/admin.py | 1 + .../migrations/0029_auto_20190215_1157.py | 26 +++++++++++++++++++ scidash/sciunittests/models.py | 5 ++-- 4 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 scidash/sciunittests/migrations/0029_auto_20190215_1157.py diff --git a/scidash/account/serializers.py b/scidash/account/serializers.py index 5e589335..000e040a 100644 --- a/scidash/account/serializers.py +++ b/scidash/account/serializers.py @@ -6,4 +6,4 @@ class ScidashUserSerializer(serializers.ModelSerializer): class Meta: model = ScidashUser - fields = '__all__' + exclude = ('password',) diff --git a/scidash/sciunittests/admin.py b/scidash/sciunittests/admin.py index f9d3c812..b59b077e 100644 --- a/scidash/sciunittests/admin.py +++ b/scidash/sciunittests/admin.py @@ -27,6 +27,7 @@ class TestInstanceModelAdmin(admin.ModelAdmin): inlines = [ TagInline, ] + list_filter = ["tags__name"] class TestSuiteAdmin(admin.ModelAdmin): diff --git a/scidash/sciunittests/migrations/0029_auto_20190215_1157.py b/scidash/sciunittests/migrations/0029_auto_20190215_1157.py new file mode 100644 index 00000000..824902ef --- /dev/null +++ b/scidash/sciunittests/migrations/0029_auto_20190215_1157.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2019-02-15 11:57 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('sciunittests', '0028_testclass_test_parameters_schema'), + ] + + operations = [ + migrations.AddField( + model_name='testinstance', + name='params', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True), + ), + migrations.AlterField( + model_name='testinstance', + name='observation', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True), + ), + ] diff --git a/scidash/sciunittests/models.py b/scidash/sciunittests/models.py index d3c754af..0b7c8f04 100644 --- a/scidash/sciunittests/models.py +++ b/scidash/sciunittests/models.py @@ -74,7 +74,8 @@ def clean_fields(self, exclude=None): class TestInstance(models.Model): name = models.CharField(max_length=200, default='Default Name') test_class = models.ForeignKey(TestClass) - observation = JSONField() + observation = JSONField(null=True, blank=True) + params = JSONField(null=True, blank=True) test_suites = models.ManyToManyField( TestSuite, related_name='tests', blank=True ) @@ -96,7 +97,7 @@ class Meta: verbose_name_plural = 'Test instances' def __str__(self): - return "{0} instance".format(self.test_class.class_name) + return f"{self.name} - {self.test_class.class_name} instance" class ScoreClass(models.Model): From 9489c8b510c474f0d6a8316088c8d9644c061551 Mon Sep 17 00:00:00 2001 From: NeZanyat Date: Fri, 15 Feb 2019 16:41:23 +0200 Subject: [PATCH 3/7] Updates to deploying --- Makefile | 8 +++++- service/codefresh/codefresh.yml | 1 - service/docker/Dockerfile-scidash | 1 + service/dotenv/env-docker | 2 +- .../scidash/scidash-deployment.yaml | 2 +- service/sciunit/requirements.txt | 12 ++++++++ service/scripts/install-sciunit-neuronunit.sh | 28 +++++++++++++++++++ 7 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 service/sciunit/requirements.txt create mode 100755 service/scripts/install-sciunit-neuronunit.sh diff --git a/Makefile b/Makefile index e98208c7..7d57373e 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,14 @@ -install: create-db install-frontend install-backend +install: create-db install-sciunit-neuronunit install-frontend install-backend @echo "===========================" @echo "= Finished =" @echo "===========================" +install-sciunit-neuronunit: + @echo "===========================" + @echo "=Install sciunit/neuronunit=" + @echo "===========================" + @./service/scripts/install-sciunit-neuronunit.sh + install-frontend: @echo "===========================" @echo "= Install frontend =" diff --git a/service/codefresh/codefresh.yml b/service/codefresh/codefresh.yml index 08d2183e..15b814c7 100644 --- a/service/codefresh/codefresh.yml +++ b/service/codefresh/codefresh.yml @@ -24,7 +24,6 @@ steps: kind: kubernetes cluster: geppetto-cluster@metacellllc namespace: scidash-testing - file_path: ./service/kubernetes/scidash/scidash-deployment.yaml candidate: image: ${{BuildingDockerImage}} registry: cfcr diff --git a/service/docker/Dockerfile-scidash b/service/docker/Dockerfile-scidash index 062a297a..277c35db 100644 --- a/service/docker/Dockerfile-scidash +++ b/service/docker/Dockerfile-scidash @@ -32,6 +32,7 @@ RUN curl -sL https://deb.nodesource.com/setup_8.x | bash RUN apt-get update && apt-get install nodejs RUN make install-backend-with-env +RUN make install-sciunit-neuronunit WORKDIR $GEPPETTO_DIR RUN npm run build-dev-noTest diff --git a/service/dotenv/env-docker b/service/dotenv/env-docker index 1973dc90..004a3e9c 100644 --- a/service/dotenv/env-docker +++ b/service/dotenv/env-docker @@ -10,4 +10,4 @@ DB_PASSWORD=scidash_local_password STATIC_URL=/static/ STATIC_DIR=static -REDIS_URL=redis://localhost:6379 +REDIS_URL=redis://scidash-redis:6379 diff --git a/service/kubernetes/scidash/scidash-deployment.yaml b/service/kubernetes/scidash/scidash-deployment.yaml index 13a23d3f..334595ac 100644 --- a/service/kubernetes/scidash/scidash-deployment.yaml +++ b/service/kubernetes/scidash/scidash-deployment.yaml @@ -21,7 +21,7 @@ spec: app: scidash spec: containers: - - image: r.cfcr.io/tarelli/metacell/scidash + - image: r.cfcr.io/tarelli/metacell/scidash:latest imagePullPolicy: Always name: scidash ports: diff --git a/service/sciunit/requirements.txt b/service/sciunit/requirements.txt new file mode 100644 index 00000000..8f45747a --- /dev/null +++ b/service/sciunit/requirements.txt @@ -0,0 +1,12 @@ +cypy>=0.2 +quantities==0.12.1 +pandas>=0.18 +ipython +matplotlib +bs4 +lxml +nbconvert +ipykernel +nbformat +gitpython +cerberus>=1.2 diff --git a/service/scripts/install-sciunit-neuronunit.sh b/service/scripts/install-sciunit-neuronunit.sh new file mode 100755 index 00000000..8a741b1d --- /dev/null +++ b/service/scripts/install-sciunit-neuronunit.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +root_path=$PWD; + +sciunit_repo="https://github.com/scidash/sciunit.git -b dash" +neuronunit_repo="https://github.com/scidash/neuronunit.git -b dash" + +sciunit_path="$root_path/sciunit" +neuronunit_path="$root_path/neuronunit" +proper_requirements_path="$root_path/service/sciunit/requirements.txt" +sciunit_requirements_path="$sciunit_path/requirements.txt" + +git clone $sciunit_repo $sciunit_path +git clone $neuronunit_repo $neuronunit_path + +cp $proper_requirements_path $sciunit_requirements_path + +cd $sciunit_path + +pip install -e . + +cd $root_path + +cd $neuronunit_path + +pip install -e . + +cd $root_path From e7498a8e696cbe0146f5b467f7211eec03d3c9b6 Mon Sep 17 00:00:00 2001 From: NeZanyat Date: Fri, 15 Feb 2019 16:49:37 +0200 Subject: [PATCH 4/7] Update codefresh.yml --- service/codefresh/codefresh.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/service/codefresh/codefresh.yml b/service/codefresh/codefresh.yml index 15b814c7..f38d7f4c 100644 --- a/service/codefresh/codefresh.yml +++ b/service/codefresh/codefresh.yml @@ -22,6 +22,7 @@ steps: Deploy: type: deploy kind: kubernetes + service: scidash cluster: geppetto-cluster@metacellllc namespace: scidash-testing candidate: From 0a01b2e7a87dc6832403964f6268daef7f4c6c61 Mon Sep 17 00:00:00 2001 From: NeZanyat Date: Sun, 17 Feb 2019 14:39:22 +0200 Subject: [PATCH 5/7] Some fixes for deploying --- docker-compose.yml | 2 +- service/scripts/install-sciunit-neuronunit.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 26e81a83..2c15b601 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: ports: - 5432:5432 scidash: - image: metacell/scidash:3.1.1 + image: metacell/scidash:4.0.1 ports: - 80:8000 depends_on: diff --git a/service/scripts/install-sciunit-neuronunit.sh b/service/scripts/install-sciunit-neuronunit.sh index 8a741b1d..35b24d15 100755 --- a/service/scripts/install-sciunit-neuronunit.sh +++ b/service/scripts/install-sciunit-neuronunit.sh @@ -4,6 +4,7 @@ root_path=$PWD; sciunit_repo="https://github.com/scidash/sciunit.git -b dash" neuronunit_repo="https://github.com/scidash/neuronunit.git -b dash" +venv="./venv/bin/activate" sciunit_path="$root_path/sciunit" neuronunit_path="$root_path/neuronunit" @@ -15,6 +16,8 @@ git clone $neuronunit_repo $neuronunit_path cp $proper_requirements_path $sciunit_requirements_path +source $venv + cd $sciunit_path pip install -e . From 3d701af158466f7a1608814e14bc3e454aa81c3a Mon Sep 17 00:00:00 2001 From: NeZanyat Date: Thu, 21 Feb 2019 17:24:23 +0530 Subject: [PATCH 6/7] Some updates --- scidash/general/helpers.py | 13 +++++++++ scidash/main/settings.py | 2 ++ scidash/main/urls.py | 8 ++++-- scidash/sciunitmodels/filters.py | 28 +++++++++++++++++++ scidash/sciunitmodels/helpers.py | 28 +++++++++++++++++++ .../migrations/0012_modelclass_import_path.py | 20 +++++++++++++ scidash/sciunitmodels/models.py | 8 ++++++ scidash/sciunittests/filters.py | 4 +-- 8 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 scidash/general/helpers.py create mode 100644 scidash/sciunitmodels/helpers.py create mode 100644 scidash/sciunitmodels/migrations/0012_modelclass_import_path.py diff --git a/scidash/general/helpers.py b/scidash/general/helpers.py new file mode 100644 index 00000000..e026e878 --- /dev/null +++ b/scidash/general/helpers.py @@ -0,0 +1,13 @@ +import importlib + + +def import_class(import_path): + splitted = import_path.split('.') + + class_name = splitted[-1:][0] + module_path = ".".join(splitted[:-1]) + + imported_module = importlib.import_module(module_path) + klass = getattr(imported_module, class_name) + + return klass diff --git a/scidash/main/settings.py b/scidash/main/settings.py index d774dfd2..b9f5ffba 100644 --- a/scidash/main/settings.py +++ b/scidash/main/settings.py @@ -185,3 +185,5 @@ EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' POPULATE_USERS = True + +DOWNLOADED_MODEL_DIR = os.path.join(BASE_DIR, os.environ.get('STATIC_DIR'), 'models') diff --git a/scidash/main/urls.py b/scidash/main/urls.py index 55943018..d9550f9e 100644 --- a/scidash/main/urls.py +++ b/scidash/main/urls.py @@ -64,7 +64,9 @@ url(r'^admin/', admin.site.urls), url(r'^api/login/?$', obtain_jwt_token), url(r'^data/', include('scidash.general.urls')), - url(r'^api/date-range/?$', DateRangeView.as_view(), name='date-range-view'), + url( + r'^api/date-range/?$', DateRangeView.as_view(), name='date-range-view' + ), url(r'^api/', include(router.urls)), url(r'^auth/', include('django.contrib.auth.urls')), url( @@ -92,7 +94,9 @@ name='user-info' ), url( - r'^api/users/is-logged/?$', CheckIsLoggedView.as_view(), name='is-logged' + r'^api/users/is-logged/?$', + CheckIsLoggedView.as_view(), + name='is-logged' ), url(r'^', include('pygeppetto_server.urls')), ] diff --git a/scidash/sciunitmodels/filters.py b/scidash/sciunitmodels/filters.py index eb59d620..6553a2c3 100644 --- a/scidash/sciunitmodels/filters.py +++ b/scidash/sciunitmodels/filters.py @@ -1,5 +1,10 @@ +import os +from urllib.parse import urlparse + +from django.conf import settings from django_filters import rest_framework as filters +import scidash.sciunitmodels.helpers as helpers import scidash.sciunitmodels.models as models @@ -27,3 +32,26 @@ class Meta: fields = [ 'name', 'class_name', 'tags', 'timestamp_from', 'timestamp_to' ] + + +class ModelClassFilter(filters.FilterSet): + by_model_url = filters.CharFilter(method='by_model_url') + + def by_model_url(self, queryset, name, value): + model_name = None + url = urlparse.urlparse(value) + model_name = os.path.basename(url) + model_path = os.path.join(settings.DOWNLOADED_MODEL_DIR, model_name) + + helpers.download_and_save_model(model_path, value) + + model_classes = models.ModelClass.objects.filter( + import_path__isnull=False + ) + + matching_classes = [ + kls.pk for kls in model_classes + if helpers.check_capabilities(model_path, kls.import_path) + ] + + return queryset.filter(pk__in=matching_classes) diff --git a/scidash/sciunitmodels/helpers.py b/scidash/sciunitmodels/helpers.py new file mode 100644 index 00000000..efc49e91 --- /dev/null +++ b/scidash/sciunitmodels/helpers.py @@ -0,0 +1,28 @@ +import requests + +from scidash.general.helpers import import_class + + +def download_and_save_model(path, url): + model_content = requests.get(url) + + with open(path, 'w') as f: + f.write(model_content) + + +def check_capabilities(model_file_path, model_class_import_path): + klass = import_class(model_class_import_path) + + return klass.failed_extra_capabilities + + +def get_model_capabilities(model_class_import_path): + klass = import_class(model_class_import_path) + + return klass.capabilities + + +def get_extra_capabilities(model_class_import_path): + klass = import_class(model_class_import_path) + + return klass.extra diff --git a/scidash/sciunitmodels/migrations/0012_modelclass_import_path.py b/scidash/sciunitmodels/migrations/0012_modelclass_import_path.py new file mode 100644 index 00000000..d48212dc --- /dev/null +++ b/scidash/sciunitmodels/migrations/0012_modelclass_import_path.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2019-02-18 13:52 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sciunitmodels', '0011_auto_20190131_1333'), + ] + + operations = [ + migrations.AddField( + model_name='modelclass', + name='import_path', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/scidash/sciunitmodels/models.py b/scidash/sciunitmodels/models.py index 86ebe6c0..c701a684 100644 --- a/scidash/sciunitmodels/models.py +++ b/scidash/sciunitmodels/models.py @@ -24,7 +24,15 @@ def __str__(self): class ModelClass(models.Model): class_name = models.CharField(max_length=50) capabilities = models.ManyToManyField(Capability) + extra_capabilities = models.ManyToManyField() url = models.URLField(default='', null=True, blank=True, unique=True) + import_path = models.TextField(null=True, blank=True) + + def clean_fields(self, exclude=None): + return super().clean_fields(exclude=exclude) + + capabilities = [] + extra_capabilities = [] class Meta: verbose_name = 'Model class' diff --git a/scidash/sciunittests/filters.py b/scidash/sciunittests/filters.py index bc66afdc..0041ca29 100644 --- a/scidash/sciunittests/filters.py +++ b/scidash/sciunittests/filters.py @@ -13,9 +13,7 @@ class TestInstanceFilter(filters.FilterSet): owner = filters.CharFilter(name='owner__username', lookup_expr='icontains') - name = filters.CharFilter( - name='name', lookup_expr='icontains' - ) + name = filters.CharFilter(name='name', lookup_expr='icontains') timestamp_from = filters.IsoDateTimeFilter( name='timestamp', lookup_expr='gte' From 52c16fdce99d715a3901d94cd3076b3f24300cd3 Mon Sep 17 00:00:00 2001 From: NeZanyat Date: Fri, 22 Feb 2019 16:07:52 +0530 Subject: [PATCH 7/7] Integration finished --- Makefile | 4 +- scidash/account/serializers.py | 2 +- scidash/main/settings.py | 1 + scidash/sciunitmodels/admin.py | 5 +- scidash/sciunitmodels/api/views.py | 3 +- scidash/sciunitmodels/filters.py | 12 +++- scidash/sciunitmodels/helpers.py | 15 +++-- .../migrations/0013_auto_20190221_1626.py | 30 +++++++++ .../migrations/0014_auto_20190222_1021.py | 29 +++++++++ scidash/sciunitmodels/models.py | 63 +++++++++++++++++-- scidash/sciunittests/models.py | 4 +- static/model_doll.nml | 20 ++++++ static/models/.gitkeep | 0 13 files changed, 171 insertions(+), 17 deletions(-) create mode 100644 scidash/sciunitmodels/migrations/0013_auto_20190221_1626.py create mode 100644 scidash/sciunitmodels/migrations/0014_auto_20190222_1021.py create mode 100644 static/model_doll.nml create mode 100644 static/models/.gitkeep diff --git a/Makefile b/Makefile index 7d57373e..c0137cd3 100644 --- a/Makefile +++ b/Makefile @@ -80,10 +80,10 @@ isort-format: isort --recursive . yapf-format: - yapf -i -r --style .style.yapf -p -e "*/migrations/*.py" -e "env" -e "*/settings.py" . + yapf -i -r --style .style.yapf -p -e "*/migrations/*.py" -e "env" -e "*/settings.py" . -e "neuronunit/**" -e "sciunit/**" yapf-lint: - yapf -d -r --style .style.yapf -e "*/migrations/*.py" -e "env" -e "*/settings.py" . + yapf -d -r --style .style.yapf -e "*/migrations/*.py" -e "env" -e "*/settings.py" . -e "neuronunit/**" -e "sciunit/**" generate-tags: ctags -R --exclude=.git --exclude=node_modules --exclude=dist --exclude=env . diff --git a/scidash/account/serializers.py b/scidash/account/serializers.py index 000e040a..a74f119e 100644 --- a/scidash/account/serializers.py +++ b/scidash/account/serializers.py @@ -6,4 +6,4 @@ class ScidashUserSerializer(serializers.ModelSerializer): class Meta: model = ScidashUser - exclude = ('password',) + exclude = ('password', ) diff --git a/scidash/main/settings.py b/scidash/main/settings.py index b9f5ffba..a5033588 100644 --- a/scidash/main/settings.py +++ b/scidash/main/settings.py @@ -187,3 +187,4 @@ POPULATE_USERS = True DOWNLOADED_MODEL_DIR = os.path.join(BASE_DIR, os.environ.get('STATIC_DIR'), 'models') +MODEL_DOLL = os.path.join(BASE_DIR, os.environ.get('STATIC_DIR'), 'model_doll.nml') diff --git a/scidash/sciunitmodels/admin.py b/scidash/sciunitmodels/admin.py index 7d7c379c..d6a51511 100644 --- a/scidash/sciunitmodels/admin.py +++ b/scidash/sciunitmodels/admin.py @@ -2,7 +2,9 @@ from django.contrib.contenttypes.admin import GenericStackedInline from scidash.general.models import Tag -from scidash.sciunitmodels.models import Capability, ModelClass, ModelInstance +from scidash.sciunitmodels.models import ( + Capability, CapabilityModelThrough, ModelClass, ModelInstance +) # Register your models here. @@ -20,3 +22,4 @@ class ModelModelAdmin(admin.ModelAdmin): admin.site.register(ModelClass) admin.site.register(ModelInstance, ModelModelAdmin) admin.site.register(Capability) +admin.site.register(CapabilityModelThrough) diff --git a/scidash/sciunitmodels/api/views.py b/scidash/sciunitmodels/api/views.py index 9cfe889b..c46fe8a7 100644 --- a/scidash/sciunitmodels/api/views.py +++ b/scidash/sciunitmodels/api/views.py @@ -1,6 +1,6 @@ from rest_framework import permissions, viewsets -from scidash.sciunitmodels.filters import ModelInstanceFilter +from scidash.sciunitmodels.filters import ModelClassFilter, ModelInstanceFilter from scidash.sciunitmodels.models import Capability, ModelClass, ModelInstance from scidash.sciunitmodels.serializers import ( CapabilitySerializer, ModelClassSerializer, ModelInstanceSerializer @@ -17,6 +17,7 @@ class ModelClassViewSet(viewsets.ReadOnlyModelViewSet): queryset = ModelClass.objects.all() serializer_class = ModelClassSerializer permission_classes = (permissions.AllowAny, ) + filter_class = ModelClassFilter class ModelInstanceViewSet(viewsets.ModelViewSet): diff --git a/scidash/sciunitmodels/filters.py b/scidash/sciunitmodels/filters.py index 6553a2c3..42b97d8c 100644 --- a/scidash/sciunitmodels/filters.py +++ b/scidash/sciunitmodels/filters.py @@ -35,12 +35,18 @@ class Meta: class ModelClassFilter(filters.FilterSet): - by_model_url = filters.CharFilter(method='by_model_url') + model_url = filters.CharFilter(method='by_model_url') + + class Meta: + model = models.ModelClass + fields = [ + 'model_url', + ] def by_model_url(self, queryset, name, value): model_name = None - url = urlparse.urlparse(value) - model_name = os.path.basename(url) + url = urlparse(value) + model_name = os.path.basename(url.path) model_path = os.path.join(settings.DOWNLOADED_MODEL_DIR, model_name) helpers.download_and_save_model(model_path, value) diff --git a/scidash/sciunitmodels/helpers.py b/scidash/sciunitmodels/helpers.py index efc49e91..6b077311 100644 --- a/scidash/sciunitmodels/helpers.py +++ b/scidash/sciunitmodels/helpers.py @@ -1,4 +1,5 @@ import requests +from django.conf import settings from scidash.general.helpers import import_class @@ -7,22 +8,28 @@ def download_and_save_model(path, url): model_content = requests.get(url) with open(path, 'w') as f: - f.write(model_content) + f.write(model_content.text) def check_capabilities(model_file_path, model_class_import_path): klass = import_class(model_class_import_path) - return klass.failed_extra_capabilities + failed = klass(model_file_path).failed_extra_capabilities + + return len(failed) == 0 def get_model_capabilities(model_class_import_path): klass = import_class(model_class_import_path) - return klass.capabilities + doll = settings.MODEL_DOLL + + return klass(doll).capabilities def get_extra_capabilities(model_class_import_path): klass = import_class(model_class_import_path) - return klass.extra + doll = settings.MODEL_DOLL + + return klass(doll).extra_capability_checks diff --git a/scidash/sciunitmodels/migrations/0013_auto_20190221_1626.py b/scidash/sciunitmodels/migrations/0013_auto_20190221_1626.py new file mode 100644 index 00000000..950024e5 --- /dev/null +++ b/scidash/sciunitmodels/migrations/0013_auto_20190221_1626.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2019-02-21 16:26 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('sciunitmodels', '0012_modelclass_import_path'), + ] + + operations = [ + migrations.CreateModel( + name='CapabilityModelThrough', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('extra_check', models.CharField(max_length=300)), + ('capability', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sciunitmodels.Capability')), + ('model_class', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sciunitmodels.ModelClass')), + ], + ), + migrations.AddField( + model_name='modelclass', + name='extra_capabilities', + field=models.ManyToManyField(related_name='extra_capabilities_set', through='sciunitmodels.CapabilityModelThrough', to='sciunitmodels.Capability'), + ), + ] diff --git a/scidash/sciunitmodels/migrations/0014_auto_20190222_1021.py b/scidash/sciunitmodels/migrations/0014_auto_20190222_1021.py new file mode 100644 index 00000000..0125b194 --- /dev/null +++ b/scidash/sciunitmodels/migrations/0014_auto_20190222_1021.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2019-02-22 10:21 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sciunitmodels', '0013_auto_20190221_1626'), + ] + + operations = [ + migrations.AlterModelOptions( + name='capabilitymodelthrough', + options={'verbose_name': 'Capabilities with extra check', 'verbose_name_plural': 'Capabilities with extra checks'}, + ), + migrations.AlterField( + model_name='modelclass', + name='capabilities', + field=models.ManyToManyField(blank=True, related_name='capabilities', to='sciunitmodels.Capability'), + ), + migrations.AlterField( + model_name='modelclass', + name='extra_capabilities', + field=models.ManyToManyField(blank=True, related_name='capabilities_with_check', through='sciunitmodels.CapabilityModelThrough', to='sciunitmodels.Capability'), + ), + ] diff --git a/scidash/sciunitmodels/models.py b/scidash/sciunitmodels/models.py index c701a684..6b76600a 100644 --- a/scidash/sciunitmodels/models.py +++ b/scidash/sciunitmodels/models.py @@ -7,6 +7,9 @@ from django.db import models from scidash.general import models as general_models +from scidash.sciunitmodels.helpers import ( + get_extra_capabilities, get_model_capabilities +) # Models Related @@ -21,19 +24,71 @@ def __str__(self): return self.class_name +class CapabilityModelThrough(models.Model): + capability = models.ForeignKey(Capability, on_delete=models.CASCADE) + model_class = models.ForeignKey('ModelClass', on_delete=models.CASCADE) + extra_check = models.CharField(max_length=300) + + def __str__(self): + return f'{self.capability.class_name} with check in {self.extra_check}' + + class Meta: + verbose_name = 'Capabilities with extra check' + verbose_name_plural = 'Capabilities with extra checks' + + class ModelClass(models.Model): class_name = models.CharField(max_length=50) - capabilities = models.ManyToManyField(Capability) - extra_capabilities = models.ManyToManyField() + capabilities = models.ManyToManyField( + Capability, blank=True, related_name='capabilities' + ) + extra_capabilities = models.ManyToManyField( + Capability, + through=CapabilityModelThrough, + related_name='capabilities_with_check', + blank=True + ) url = models.URLField(default='', null=True, blank=True, unique=True) import_path = models.TextField(null=True, blank=True) - def clean_fields(self, exclude=None): - return super().clean_fields(exclude=exclude) + def save(self, *args, **kwargs): + super().save(*args, **kwargs) capabilities = [] extra_capabilities = [] + try: + capabilities = get_model_capabilities(self.import_path) + extra_capabilities = get_extra_capabilities(self.import_path) + except ImportError: + self.memo = f"Can't import {self.import_path}" + except AttributeError: + self.memo = \ + f"Wrong class for import {self.import_path}" + + if capabilities is None: + self.memo = \ + f"Wrong class for import capabilities {self.import_path}" + elif extra_capabilities is None: + self.memo = \ + f"No extra capabilities check {self.import_path}" + else: + self.memo = "" + + for capability in capabilities: + capability_model, created = Capability.objects.get_or_create( + class_name=capability.__name__ + ) + if capability not in extra_capabilities: + self.capabilities.add(capability_model) + else: + extra_capability_model, created = CapabilityModelThrough.objects.get_or_create( + capability=capability_model, + model_class=self, + extra_check=extra_capabilities[capability] + ) + extra_capability_model.save() + class Meta: verbose_name = 'Model class' verbose_name_plural = 'Model classes' diff --git a/scidash/sciunittests/models.py b/scidash/sciunittests/models.py index 0b7c8f04..5ca236bf 100644 --- a/scidash/sciunittests/models.py +++ b/scidash/sciunittests/models.py @@ -9,7 +9,9 @@ import scidash.sciunitmodels as sciunitmodels from scidash.general import models as general_models -from scidash.sciunittests.helpers import get_observation_schema, get_test_parameters_schema +from scidash.sciunittests.helpers import ( + get_observation_schema, get_test_parameters_schema +) logger = logging.getLogger(__name__) diff --git a/static/model_doll.nml b/static/model_doll.nml new file mode 100644 index 00000000..d3a96990 --- /dev/null +++ b/static/model_doll.nml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + diff --git a/static/models/.gitkeep b/static/models/.gitkeep new file mode 100644 index 00000000..e69de29b