Refactor datasets & experiment versions

weblab/core/models.py

@@ -1,8 +1,12 @@
+import urllib.parse
+
 from django.conf import settings
+from django.core.urlresolvers import reverse
 from django.db import models
 from guardian.shortcuts import assign_perm, get_users_with_perms, remove_perm
 
 from . import visibility
+from .combine import ArchiveReader
 
 
 class VisibilityModelMixin(models.Model):
@@ -95,3 +99,93 @@ def is_managed_by(self, user):
 
     class Meta:
         abstract = True
+
+
+class FileCollectionMixin:
+    """Mixin for DB models that represent collections of files backed by a COMBINE Archive.
+
+    This doesn't provide any DB fields, but does define common properties and methods for
+    such collections, ensuring they present a consistent API.
+    """
+    @property
+    def abs_path(self):
+        """The folder where the backing archive is stored on disk.
+
+        Must be defined by subclasses.
+        """
+        raise NotImplementedError
+
+    @property
+    def archive_name(self):
+        """The name of the backing archive.
+
+        Must be defined by subclasses.
+        """
+        raise NotImplementedError
+
+    @property
+    def archive_path(self):
+        """The full path to the backing archive. A ``pathlib.Path`` instance."""
+        return self.abs_path / self.archive_name
+
+    @property
+    def files(self):
+        """The list of files (``core.combine.ArchiveFile`` instances) contained in this archive."""
+        if self.archive_path.exists():
+            return ArchiveReader(str(self.archive_path)).files
+        else:
+            return []
+
+    def mkdir(self):
+        """Create the folder for the backing archive (and parents if needed)."""
+        self.abs_path.mkdir(exist_ok=True, parents=True)
+
+    def open_file(self, name):
+        """Open the given file in the archive for reading."""
+        return ArchiveReader(str(self.archive_path)).open_file(name)
+
+    def get_file_json(self, file_, ns, url_args):
+        """Get information about a file in JSON format for use by Javascript code.
+
+        :param core.combine.ArchiveFile file_: the file to provide info about
+        :param str ns: the app namespace to use for reversing download URLs
+        :param list url_args: initial argument(s) for reverse to identify the collection the file is in
+        :return: a dictionary of file metadata
+        """
+        return {
+            'id': file_.name,
+            'author': self.author.full_name,
+            'created': self.created_at,
+            'name': file_.name,
+            'filetype': file_.fmt,
+            'masterFile': file_.is_master,
+            'size': file_.size,
+            'url': reverse(
+                ns + ':file_download',
+                args=url_args + [urllib.parse.quote(file_.name)]
+            )
+        }
+
+    def get_json(self, ns, url_args):
+        """Get information about this collection in JSON format for use by Javascript code.
+
+        :param str ns: the app namespace to use for reversing download URLs
+        :param list url_args: initial argument(s) for reverse to identify this collection
+        :return: a dictionary of collection metadata, including info about each file
+        """
+        files = [
+            self.get_file_json(f, ns, url_args)
+            for f in self.files
+            if f.name not in ['manifest.xml', 'metadata.rdf']
+        ]
+        return {
+            'id': self.id,
+            'author': self.author.full_name,
+            'parsedOk': False,
+            'visibility': self.visibility,
+            'created': self.created_at,
+            'name': self.name,
+            'files': files,
+            'numFiles': len(files),
+            'download_url': reverse(ns + ':archive', args=url_args),
+        }

weblab/datasets/models.py

@@ -2,13 +2,11 @@
 from django.db import models
 from django.utils.text import get_valid_filename
 
-from core.combine import ArchiveReader
-from core.models import UserCreatedModelMixin, VisibilityModelMixin
-from core.visibility import visibility_check
+from core.models import FileCollectionMixin, UserCreatedModelMixin, VisibilityModelMixin
 from entities.models import ProtocolEntity
 
 
-class Dataset(UserCreatedModelMixin, VisibilityModelMixin, models.Model):
+class Dataset(UserCreatedModelMixin, VisibilityModelMixin, FileCollectionMixin, models.Model):
     """Prototyping class for experimental datasets
     """
     name = models.CharField(validators=[MinLengthValidator(2)], max_length=255)
@@ -37,30 +35,6 @@ def abs_path(self):
     def archive_name(self):
         return get_valid_filename(self.name + '.zip')
 
-    @property
-    def archive_path(self):
-        return self.abs_path / self.archive_name
-
-    @property
-    def files(self):
-        if self.archive_path.exists():
-            return ArchiveReader(str(self.archive_path)).files
-        else:
-            return []
-
-    def open_file(self, name):
-        return ArchiveReader(str(self.archive_path)).open_file(name)
-
-    def is_visible_to_user(self, user):
-        """
-        Can the user view the dataset?
-
-        :param user: user to test against
-
-        :returns: True if the user is allowed to view the dataset, False otherwise
-        """
-        return visibility_check(self.visibility, self.viewers, user)
-
 
 class DatasetFile(models.Model):
     dataset = models.ForeignKey(Dataset, related_name='file_uploads')

weblab/datasets/tests/test_models.py

@@ -11,11 +11,6 @@ def test_str(self):
         dataset = recipes.dataset.make(name='test dataset')
         assert str(dataset) == 'test dataset'
 
-    def test_is_visible_to_user(self, user, other_user):
-        dataset = recipes.dataset.make(author=user, name='mydataset', visibility="private")
-        assert dataset.is_visible_to_user(user)
-        assert not dataset.is_visible_to_user(other_user)
-
     def test_related_protocol(self, user):
         protocol = recipes.protocol.make(author=user)
         dataset = recipes.dataset.make(author=user, name='mydataset', protocol=protocol)

weblab/datasets/views.py

@@ -1,6 +1,5 @@
 import mimetypes
 import os.path
-import urllib
 from zipfile import ZipFile
 
 from braces.views import UserFormKwargsMixin
@@ -17,7 +16,6 @@
     HttpResponseRedirect,
     JsonResponse,
 )
-from django.utils.text import get_valid_filename
 from django.views import View
 from django.views.generic.detail import DetailView, SingleObjectMixin
 from django.views.generic.edit import CreateView, DeleteView, FormMixin
@@ -46,27 +44,6 @@ def get_success_url(self):
         return reverse('datasets:addfiles', args=[self.object.pk])
 
 
-class DatasetListView(LoginRequiredMixin, ListView):
-    """
-    List all user's datasets
-    """
-    model = Dataset
-    template_name = 'datasets/dataset_list.html'
-
-    def get_queryset(self):
-        return Dataset.objects.filter(author=self.request.user)
-
-
-class DatasetView(VisibilityMixin, DetailView):
-    """
-    View a Dataset
-
-    """
-    model = Dataset
-    context_object_name = 'dataset'
-    template_name = 'datasets/dataset_detail.html'
-
-
 class DatasetAddFilesView(
     LoginRequiredMixin, FormMixin, DetailView
 ):
@@ -151,50 +128,39 @@ def post(self, request, *args, **kwargs):
             return HttpResponseBadRequest(form.errors)
 
 
+class DatasetListView(LoginRequiredMixin, ListView):
+    """
+    List all user's datasets
+    """
+    model = Dataset
+    template_name = 'datasets/dataset_list.html'
+
+    def get_queryset(self):
+        return self.model.objects.filter(author=self.request.user)
+
+
+class DatasetView(VisibilityMixin, DetailView):
+    """
+    View a Dataset
+
+    """
+    model = Dataset
+    context_object_name = 'dataset'
+    template_name = 'datasets/dataset_detail.html'
+
+
 class DatasetJsonView(VisibilityMixin, SingleObjectMixin, View):
     """
     Serve up json view of files in a dataset
     """
     model = Dataset
 
-    def _file_json(self, archive_file):
-        dataset = self.object
-        return {
-            'id': archive_file.name,
-            'author': dataset.author.full_name,
-            'created': dataset.created_at,
-            'name': archive_file.name,
-            'filetype': archive_file.fmt,
-            'masterFile': archive_file.is_master,
-            'size': archive_file.size,
-            'url': reverse(
-                'datasets:file_download',
-                args=[dataset.id, urllib.parse.quote(archive_file.name)]
-            )
-        }
-
     def get(self, request, *args, **kwargs):
-        dataset = self.object = self.get_object()
-        files = [
-            self._file_json(f)
-            for f in dataset.files
-            if f.name not in ['manifest.xml', 'metadata.rdf']
-        ]
-
+        ns = self.request.resolver_match.namespace
+        dataset = self.get_object()
+        url_args = [dataset.id]
         return JsonResponse({
-            'version': {
-                'id': dataset.id,
-                'author': dataset.author.full_name,
-                'parsedOk': False,
-                'visibility': dataset.visibility,
-                'created': dataset.created_at,
-                'name': dataset.name,
-                'files': files,
-                'numFiles': len(files),
-                'download_url': reverse(
-                    'datasets:archive', args=[dataset.id]
-                ),
-            }
+            'version': dataset.get_json(ns, url_args)
         })
 
 
@@ -229,16 +195,17 @@ class DatasetArchiveView(VisibilityMixin, SingleObjectMixin, View):
     """
     model = Dataset
 
+    def get_archive_name(self, dataset):
+        return dataset.archive_name
+
     def get(self, request, *args, **kwargs):
         dataset = self.get_object()
         path = dataset.archive_path
 
         if not path.exists():
             raise Http404
 
-        zipfile_name = os.path.join(
-            get_valid_filename('%s.zip' % dataset.name)
-        )
+        zipfile_name = self.get_archive_name(dataset)
 
         with path.open('rb') as archive:
             response = HttpResponse(content_type='application/zip')
@@ -261,4 +228,5 @@ def test_func(self):
         return self.get_object().is_deletable_by(self.request.user)
 
     def get_success_url(self, *args, **kwargs):
-        return reverse('datasets:list')
+        ns = self.request.resolver_match.namespace
+        return reverse(ns + ':list')

weblab/entities/models.py

@@ -3,10 +3,12 @@
 from pathlib import Path
 
 from django.core.exceptions import ObjectDoesNotExist
+from django.core.urlresolvers import reverse
 from django.core.validators import MinLengthValidator
 from django.db import models
 from guardian.shortcuts import get_objects_for_user
 
+from core.filetypes import get_file_type
 from core.models import UserCreatedModelMixin
 from core.visibility import HELP_TEXT as VIS_HELP_TEXT
 from core.visibility import Visibility, visibility_check
@@ -274,6 +276,60 @@ def is_parsed_ok(self, version):
             ok = self.repocache.get_version(version).parsed_ok
         return ok
 
+    def get_version_json(self, commit, ns):
+        """Get metadata for a particular version of this entity suitable for sending as JSON.
+
+        :param commit: a `Commit` instance (TODO #191 a `CachedEntityVersion` instance)
+        :param str ns: the app namespace to use for reversing download URLs
+        :return: a dictionary of version metadata, including file info
+        """
+        files = [
+            self.get_file_json(commit, f, ns)
+            for f in commit.files
+            if f.name not in ['manifest.xml', 'metadata.rdf']
+        ]
+        return {
+            'id': commit.sha,
+            'entityId': self.id,
+            'author': commit.author.name,
+            'parsedOk': self.is_parsed_ok(commit.sha),
+            'visibility': self.get_version_visibility(commit.sha, default=self.DEFAULT_VISIBILITY),
+            'created': commit.timestamp,
+            'name': self.name,
+            'version': self.repo.get_name_for_commit(commit.sha),  # TODO #191 use repocache instead
+            'files': files,
+            'commitMessage': commit.message,
+            'numFiles': len(files),
+            'url': reverse(
+                ns + ':version',
+                args=[self.url_type, self.id, commit.sha]
+            ),
+        }
+
+    def get_file_json(self, commit, file_, ns):
+        """Get metadata for a single file within a commit suitable for sending as JSON.
+
+        TODO #191 consider how to replace Commit with CachedEntityVersion here. We'd need
+        to cache the list of file names and sizes.
+
+        :param commit: a `Commit` instance
+        :param git.Blob file_: the file to get metadata for
+        :param str ns: the app namespace to use for reversing download URLs
+        :return: a dictionary of file metadata
+        """
+        return {
+            'id': file_.name,
+            'name': file_.name,
+            'author': commit.author.name,
+            'created': commit.timestamp,
+            'filetype': get_file_type(file_.name),
+            'size': file_.size,
+            'url': reverse(
+                ns + ':file_download',
+                args=[self.url_type, self.id, commit.sha, file_.name]
+            ),
+        }
+
 
 class EntityManager(models.Manager):
     def get_queryset(self):