diff --git a/weblab/conftest.py b/weblab/conftest.py index 1836685a2..39ba2f6d8 100644 --- a/weblab/conftest.py +++ b/weblab/conftest.py @@ -173,7 +173,9 @@ def queued_experiment(model_with_version, protocol_with_version): version = recipes.experiment_version.make( status='QUEUED', experiment__model=model_with_version, + experiment__model_version=model_with_version.repocache.latest_version, experiment__protocol=protocol_with_version, + experiment__protocol_version=protocol_with_version.repocache.latest_version, ) recipes.running_experiment.make(experiment_version=version) return version @@ -184,7 +186,9 @@ def experiment_with_result(model_with_version, protocol_with_version): version = recipes.experiment_version.make( status='SUCCESS', experiment__model=model_with_version, + experiment__model_version=model_with_version.repocache.latest_version, experiment__protocol=protocol_with_version, + experiment__protocol_version=protocol_with_version.repocache.latest_version, ) version.mkdir() with (version.abs_path / 'result.txt').open('w') as f: @@ -197,9 +201,9 @@ def experiment_version(public_model, public_protocol): return recipes.experiment_version.make( status='SUCCESS', experiment__model=public_model, - experiment__model_version=public_model.repo.latest_commit.sha, + experiment__model_version=public_model.repocache.latest_version, experiment__protocol=public_protocol, - experiment__protocol_version=public_protocol.repo.latest_commit.sha, + experiment__protocol_version=public_protocol.repocache.latest_version, ) @@ -207,9 +211,9 @@ def experiment_version(public_model, public_protocol): def quick_experiment_version(helpers): """An experiment version that exists only in the DB - no model/proto repos, no results.""" model = recipes.model.make() - model_version = str(helpers.add_fake_version(model, 'public').sha) + model_version = helpers.add_fake_version(model, 'public') protocol = recipes.protocol.make() - protocol_version = str(helpers.add_fake_version(protocol, 'public').sha) + protocol_version = helpers.add_fake_version(protocol, 'public') return recipes.experiment_version.make( status='SUCCESS', experiment__model=model, @@ -224,9 +228,9 @@ def moderated_experiment_version(moderated_model, moderated_protocol): return recipes.experiment_version.make( status='SUCCESS', experiment__model=moderated_model, - experiment__model_version=moderated_model.repo.latest_commit.sha, + experiment__model_version=moderated_model.repocache.latest_version, experiment__protocol=moderated_protocol, - experiment__protocol_version=moderated_protocol.repo.latest_commit.sha, + experiment__protocol_version=moderated_protocol.repocache.latest_version, ) diff --git a/weblab/core/recipes.py b/weblab/core/recipes.py index 729fe02a5..dc8715f20 100644 --- a/weblab/core/recipes.py +++ b/weblab/core/recipes.py @@ -22,23 +22,26 @@ analysis_task = Recipe('AnalysisTask', entity=foreign_key(protocol)) +cached_model = Recipe('CachedModel') +cached_model_version = Recipe('CachedModelVersion') +cached_model_tag = Recipe('CachedModelTag') + +cached_protocol = Recipe('CachedProtocol') +cached_protocol_version = Recipe('CachedProtocolVersion') +cached_protocol_tag = Recipe('CachedProtocolTag') + experiment = Recipe( 'Experiment', model=foreign_key(model), - protocol=foreign_key(protocol) + model_version=foreign_key(cached_model_version), + protocol=foreign_key(protocol), + protocol_version=foreign_key(cached_protocol_version), ) -running_experiment = Recipe('RunningExperiment') - experiment_version = Recipe('ExperimentVersion', experiment=foreign_key(experiment)) -cached_model = Recipe('CachedModel') -cached_model_version = Recipe('CachedModelVersion') -cached_model_tag = Recipe('CachedModelTag') +running_experiment = Recipe('RunningExperiment', experiment_version=foreign_key(experiment_version)) -cached_protocol = Recipe('CachedProtocol') -cached_protocol_version = Recipe('CachedProtocolVersion') -cached_protocol_tag = Recipe('CachedProtocolTag') dataset = Recipe('Dataset', name=seq('mydataset'), diff --git a/weblab/entities/models.py b/weblab/entities/models.py index 0b5daaeef..11dc98597 100644 --- a/weblab/entities/models.py +++ b/weblab/entities/models.py @@ -94,18 +94,6 @@ def repo_abs_path(self): self.author.get_storage_dir('repo'), '%ss' % self.entity_type, str(self.id) ) - def nice_version(self, sha_or_tag): - """ - Returns tag/sha with ellipses - - :param sha_or_tag: version sha or tag string - :return version_name: string with the sha_or_tag formatted - """ - version_name = self.repocache.get_name_for_version(sha_or_tag) - if len(version_name) > 20: - version_name = version_name[:8] + '...' - return version_name - def get_visibility_from_repo(self, commit): """ Get the visibility of the given entity version from the repository diff --git a/weblab/entities/processing.py b/weblab/entities/processing.py index fe032be97..a40404148 100644 --- a/weblab/entities/processing.py +++ b/weblab/entities/processing.py @@ -159,7 +159,7 @@ def record_experiments_to_run(user, entity, commit): # Find visible experiments involving this parent parent_kwargs = { entity.entity_type: entity, - entity.entity_type + '_version': parent.sha, + entity.entity_type + '_version': entity.repocache.get_version(parent.sha), } for expt in Experiment.objects.filter(**parent_kwargs): if expt.is_visible_to_user(user): @@ -167,9 +167,9 @@ def record_experiments_to_run(user, entity, commit): kwargs = { 'submitter': user, 'model_id': expt.model_id, - 'model_version': expt.model_version, + 'model_version': expt.model_version.sha, 'protocol_id': expt.protocol_id, - 'protocol_version': expt.protocol_version, + 'protocol_version': expt.protocol_version.sha, } kwargs.update(new_version_kwargs) PlannedExperiment.objects.get_or_create(**kwargs) diff --git a/weblab/entities/templatetags/entities.py b/weblab/entities/templatetags/entities.py index 40b168d81..e42cc8a15 100644 --- a/weblab/entities/templatetags/entities.py +++ b/weblab/entities/templatetags/entities.py @@ -115,16 +115,12 @@ def url_entity_diff_base(context, entity_type): @register.filter def name_of_model(experiment): - model = experiment.model - model_version = model.repocache.get_name_for_version(experiment.model_version) - return '%s @ %s' % (model.name, model_version) + return '%s @ %s' % (experiment.model.name, experiment.model_version.get_name()) @register.filter def name_of_protocol(experiment): - protocol = experiment.protocol - protocol_version = protocol.repocache.get_name_for_version(experiment.protocol_version) - return '%s @ %s' % (protocol.name, protocol_version) + return '%s @ %s' % (experiment.protocol.name, experiment.protocol_version.get_name()) def _url_friendly_label(entity, version): diff --git a/weblab/entities/tests/test_models.py b/weblab/entities/tests/test_models.py index 58d57a929..52e8a0330 100644 --- a/weblab/entities/tests/test_models.py +++ b/weblab/entities/tests/test_models.py @@ -92,14 +92,6 @@ def test_repo_abs_path(self, fake_repo_path): assert model.repo._root == path assert str(model.repo_abs_path) == path - def test_nice_version(self, model_with_version): - commit = model_with_version.repo.latest_commit.sha - assert model_with_version.nice_version(commit) == '%s...' % commit[:8] - - model_with_version.repo.tag('v1') - populate_entity_cache(model_with_version) - assert model_with_version.nice_version(commit) == 'v1' - def test_set_and_get_version_visibility(self, model_with_version): commit = model_with_version.repo.latest_commit assert model_with_version.get_version_visibility(commit.sha) == 'private' diff --git a/weblab/entities/tests/test_templatetags.py b/weblab/entities/tests/test_templatetags.py index 405a8c4c5..674726029 100644 --- a/weblab/entities/tests/test_templatetags.py +++ b/weblab/entities/tests/test_templatetags.py @@ -105,9 +105,9 @@ def test_name_of_entity_linked_to_experiment(model_with_version, protocol_with_v exp = recipes.experiment_version.make( status='SUCCESS', experiment__model=model_with_version, - experiment__model_version=model_with_version.repo.latest_commit.sha, + experiment__model_version=model_with_version.cachedentity.latest_version, experiment__protocol=protocol_with_version, - experiment__protocol_version=protocol_with_version.repo.latest_commit.sha, + experiment__protocol_version=protocol_with_version.cachedentity.latest_version, ).experiment assert entity_tags.name_of_model(exp) == '%s @ v1' % model_with_version.name diff --git a/weblab/entities/tests/test_views.py b/weblab/entities/tests/test_views.py index 751a09e74..0177e8120 100644 --- a/weblab/entities/tests/test_views.py +++ b/weblab/entities/tests/test_views.py @@ -183,6 +183,7 @@ def test_version_with_two_tags(self, client, helpers): version = model.repocache.latest_version model.add_tag('tag1', version.sha) model.add_tag('tag2', version.sha) + self.check(client, '/entities/models/%d/versions/%s' % (model.pk, version.sha), version, ['tag1', 'tag2']) self.check(client, '/entities/models/%d/versions/%s' % (model.pk, 'tag1'), version, ['tag1', 'tag2']) self.check(client, '/entities/models/%d/versions/%s' % (model.pk, 'tag2'), @@ -445,31 +446,32 @@ def test_complex_visibilities(self, client, logged_in_user, other_user, helpers) class TestModelEntityCompareExperimentsView: def test_shows_related_experiments(self, client, helpers, experiment_version): exp = experiment_version.experiment - sha = exp.model.repo.latest_commit.sha + model_version = exp.model.repocache.latest_version recipes.experiment_version.make() # another experiment which should not be included exp.model.set_version_visibility('latest', 'public') exp.protocol.set_version_visibility('latest', 'public') - + helpers.add_version(exp.protocol, visibility='public') # Add an experiment with a newer version of the protocol but that was created earlier exp2 = recipes.experiment_version.make( experiment__protocol=exp.protocol, - experiment__protocol_version=helpers.add_version(exp.protocol, visibility='public').sha, + experiment__protocol_version=exp.protocol.repocache.latest_version, experiment__model=exp.model, - experiment__model_version=sha, + experiment__model_version=model_version, ).experiment exp2.created_at = exp.created_at - timedelta(seconds=10) exp2.save() # Add an experiment with a newer version of the protocol and that was created later + helpers.add_version(exp.protocol, visibility='public') exp3 = recipes.experiment_version.make( experiment__protocol=exp.protocol, - experiment__protocol_version=helpers.add_version(exp.protocol, visibility='public').sha, + experiment__protocol_version=exp.protocol.repocache.latest_version, experiment__model=exp.model, - experiment__model_version=sha, + experiment__model_version=model_version, ).experiment response = client.get( - '/entities/models/%d/versions/%s/compare' % (exp.model.pk, sha) + '/entities/models/%d/versions/%s/compare' % (exp.model.pk, model_version.sha) ) assert response.status_code == 200 @@ -477,20 +479,19 @@ def test_shows_related_experiments(self, client, helpers, experiment_version): def test_applies_visibility(self, client, helpers, experiment_version): exp = experiment_version.experiment - sha = exp.model_version protocol = recipes.protocol.make() exp.model.set_version_visibility('latest', 'public') exp.protocol.set_version_visibility('latest', 'public') - + protocol_version = helpers.add_version(protocol, visibility='private') recipes.experiment_version.make( experiment__protocol=protocol, - experiment__protocol_version=helpers.add_version(protocol, visibility='private').sha, + experiment__protocol_version=protocol.repocache.get_version(protocol_version.sha), experiment__model=exp.model, - experiment__model_version=sha, + experiment__model_version=exp.model_version, ) # should not be included for visibility reasons response = client.get( - '/entities/models/%d/versions/%s/compare' % (exp.model.pk, sha) + '/entities/models/%d/versions/%s/compare' % (exp.model.pk, exp.model_version.sha) ) assert response.status_code == 200 @@ -527,16 +528,16 @@ def test_shows_related_experiments(self, client, logged_in_user, experiment_vers def test_applies_visibility(self, client, helpers, experiment_version): exp = experiment_version.experiment - sha = exp.protocol_version + sha = exp.protocol_version.sha model = recipes.model.make() exp.protocol.set_version_visibility('latest', 'public') exp.model.set_version_visibility('latest', 'public') - + model_version = helpers.add_version(model, visibility='private') recipes.experiment_version.make( experiment__protocol=exp.protocol, - experiment__protocol_version=sha, + experiment__protocol_version=exp.protocol.repocache.get_version(sha), experiment__model=model, - experiment__model_version=helpers.add_version(model, visibility='private').sha, + experiment__model_version=model.repocache.get_version(model_version.sha), ) # should not be included for visibility reasons response = client.get( @@ -1182,9 +1183,9 @@ def _add_experiment(proto_author, proto_vis, shared=False, recipes.experiment_version.make( status='SUCCESS', experiment__model=model, - experiment__model_version=model_commit.sha, + experiment__model_version=model.repocache.get_version(model_commit.sha), experiment__protocol=proto, - experiment__protocol_version=proto_commit.sha, + experiment__protocol_version=proto.repocache.get_version(proto_commit.sha), ) if shared: assign_perm('edit_entity', logged_in_user, proto) @@ -2218,9 +2219,9 @@ def test_view_run_experiment_model_post_exclude_existing(self, client, helpers, recipes.experiment_version.make( status='SUCCESS', experiment__model=model, - experiment__model_version=commit_model.sha, + experiment__model_version=model.repocache.get_version(commit_model.sha), experiment__protocol=protocol, - experiment__protocol_version=commit1.sha) + experiment__protocol_version=protocol.repocache.get_version(commit1.sha),) # Test context has correct information response = client.get( @@ -2482,15 +2483,15 @@ def test_view_run_experiment_protocol_post_exclude_existing(self, client, helper recipes.experiment_version.make( status='SUCCESS', experiment__model=model, - experiment__model_version=commit1.sha, + experiment__model_version=model.repocache.get_version(commit1.sha), experiment__protocol=protocol, - experiment__protocol_version=commit_protocol.sha) + experiment__protocol_version=protocol.repocache.get_version(commit_protocol.sha)) # This experiment has no versions so should not be excluded recipes.experiment.make( model=model, - model_version=commit2.sha, + model_version=model.repocache.get_version(commit2.sha), protocol=protocol, - protocol_version=commit_protocol.sha) + protocol_version=protocol.repocache.get_version(commit_protocol.sha)) version1 = model.repocache.get_version(commit1.sha) version2 = model.repocache.get_version(commit2.sha) diff --git a/weblab/entities/views.py b/weblab/entities/views.py index 3b0012cde..765b3738e 100644 --- a/weblab/entities/views.py +++ b/weblab/entities/views.py @@ -17,13 +17,7 @@ ) from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.core.urlresolvers import reverse -from django.db.models import ( - Count, - F, - OuterRef, - Q, - Subquery, -) +from django.db.models import Count, F, Q from django.http import ( Http404, HttpResponse, @@ -47,7 +41,7 @@ from experiments.models import Experiment, ExperimentVersion, PlannedExperiment from fitting.models import FittingSpec from repocache.exceptions import RepoCacheMiss -from repocache.models import CACHED_VERSION_TYPE_MAP, CachedProtocolVersion +from repocache.models import CachedProtocolVersion from .forms import ( EntityChangeVisibilityForm, @@ -266,22 +260,17 @@ class EntityCompareExperimentsView(EntityTypeMixin, EntityVersionMixin, DetailVi def get_context_data(self, **kwargs): entity = self._get_object() - commit = self.get_version() + version = self.get_version() entity_type = entity.entity_type other_type = entity.other_type - q_other_type_versions = CACHED_VERSION_TYPE_MAP[other_type].objects.filter( - entity__entity=OuterRef(other_type), - sha=OuterRef(other_type + '_version'), - ) - experiments = Experiment.objects.filter(**{ entity_type: entity.pk, - entity_type + '_version': commit.sha, + entity_type + '_version': version.pk, }).annotate( version_count=Count('versions'), - other_version_timestamp=Subquery(q_other_type_versions.values('timestamp')[:1]), + other_version_timestamp=F(other_type + '_version__timestamp'), ).filter( version_count__gt=0, ).select_related(other_type).order_by(other_type, '-other_version_timestamp') @@ -1040,26 +1029,29 @@ def post(self, request, *args, **kwargs): # in get context self.object was the entity being worked with # here we have to retrieve it this_entity = self.get_object() - this_version = self.get_version().sha - is_latest = (this_version == this_entity.repocache.latest_version.sha) + this_version = self.get_version() + is_latest = (this_version == this_entity.repocache.latest_version) exclude_existing = 'rerun_expts' not in request.POST experiments_to_run = request.POST.getlist('model_protocol_list[]') for version in experiments_to_run: ident, sha = version.split(':') - exper_kwargs = { - this_entity.other_type + '_id': ident, - this_entity.other_type + '_version': sha, - this_entity.entity_type + '_id': this_entity.id, - this_entity.entity_type + '_version': this_version, - } + other_version = Entity.objects.get(pk=ident).cachedentity.versions.get(sha=sha) if exclude_existing: filter_kwargs = { - 'experiment__' + name: value - for (name, value) in exper_kwargs.items() + 'experiment__' + this_entity.other_type + '_id': ident, + 'experiment__' + this_entity.other_type + '_version_id': other_version.id, + 'experiment__' + this_entity.entity_type + '_id': this_entity.id, + 'experiment__' + this_entity.entity_type + '_version_id': this_version.id, } if ExperimentVersion.objects.filter(**filter_kwargs).exists(): continue - exper_kwargs['submitter'] = request.user + exper_kwargs = { + 'submitter': request.user, + this_entity.other_type + '_id': ident, + this_entity.other_type + '_version': other_version.sha, + this_entity.entity_type + '_id': this_entity.id, + this_entity.entity_type + '_version': this_version.sha, + } PlannedExperiment.objects.get_or_create(**exper_kwargs) # return to entity page version_to_use = 'latest' diff --git a/weblab/experiments/migrations/0019_auto_20200204_1039.py b/weblab/experiments/migrations/0019_auto_20200204_1039.py new file mode 100644 index 000000000..933aec6cc --- /dev/null +++ b/weblab/experiments/migrations/0019_auto_20200204_1039.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.23 on 2020-02-04 10:39 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('repocache', '0021_auto_20200116_0913'), + ('experiments', '0018_auto_20190218_1513'), + ] + + operations = [ + migrations.AddField( + model_name='experiment', + name='model_version_fk', + field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='model_ver_exps', to='repocache.CachedModelVersion'), + ), + migrations.AddField( + model_name='experiment', + name='protocol_version_fk', + field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='pro_ver_exps', to='repocache.CachedProtocolVersion'), + ), + ] diff --git a/weblab/experiments/migrations/0020_link_exps_cachedentity.py b/weblab/experiments/migrations/0020_link_exps_cachedentity.py new file mode 100644 index 000000000..dbb9259d2 --- /dev/null +++ b/weblab/experiments/migrations/0020_link_exps_cachedentity.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.23 on 2020-02-04 10:56 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + def populate_fks(apps, schema_editor): + Experiment = apps.get_model('experiments', 'Experiment') + for experiment in Experiment.objects.all(): + experiment.model_version_fk = experiment.model.cachedmodel.versions.get(sha=experiment.model_version) + experiment.protocol_version_fk = experiment.protocol.cachedprotocol.versions.get(sha=experiment.protocol_version) + experiment.save() + + def remove_fks(apps, schema_editor): + Experiment = apps.get_model('experiments', 'Experiment') + for experiment in Experiment.objects.all(): + experiment.model_version_fk = None + experiment.protocol_version_fk = None + experiment.save() + + dependencies = [ + ('experiments', '0019_auto_20200204_1039'), + ] + + operations = [ + migrations.RunPython(populate_fks, remove_fks), + ] diff --git a/weblab/experiments/migrations/0021_auto_20200204_1502.py b/weblab/experiments/migrations/0021_auto_20200204_1502.py new file mode 100644 index 000000000..528d9c43a --- /dev/null +++ b/weblab/experiments/migrations/0021_auto_20200204_1502.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.23 on 2020-02-04 15:02 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('repocache', '0021_auto_20200116_0913'), + ('entities', '0015_auto_20191128_1601'), + ('experiments', '0020_link_exps_cachedentity'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='experiment', + unique_together=set([('model', 'protocol', 'model_version_fk', 'protocol_version_fk')]), + ), + migrations.RemoveField( + model_name='experiment', + name='model_version', + ), + migrations.RemoveField( + model_name='experiment', + name='protocol_version', + ), + ] diff --git a/weblab/experiments/migrations/0022_auto_20200204_1532.py b/weblab/experiments/migrations/0022_auto_20200204_1532.py new file mode 100644 index 000000000..526636b69 --- /dev/null +++ b/weblab/experiments/migrations/0022_auto_20200204_1532.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.23 on 2020-02-04 15:32 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('repocache', '0021_auto_20200116_0913'), + ('entities', '0015_auto_20191128_1601'), + ('experiments', '0021_auto_20200204_1502'), + ] + + operations = [ + migrations.RenameField( + model_name='experiment', + old_name='model_version_fk', + new_name='model_version', + ), + migrations.RenameField( + model_name='experiment', + old_name='protocol_version_fk', + new_name='protocol_version', + ), + migrations.AlterUniqueTogether( + name='experiment', + unique_together=set([('model', 'protocol', 'model_version', 'protocol_version')]), + ), + ] diff --git a/weblab/experiments/migrations/0023_auto_20200204_1533.py b/weblab/experiments/migrations/0023_auto_20200204_1533.py new file mode 100644 index 000000000..4519e2168 --- /dev/null +++ b/weblab/experiments/migrations/0023_auto_20200204_1533.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.23 on 2020-02-04 15:33 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('experiments', '0022_auto_20200204_1532'), + ] + + operations = [ + migrations.AlterField( + model_name='experiment', + name='model_version', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='model_ver_exps', to='repocache.CachedModelVersion'), + ), + migrations.AlterField( + model_name='experiment', + name='protocol_version', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='pro_ver_exps', to='repocache.CachedProtocolVersion'), + ), + ] diff --git a/weblab/experiments/migrations/0024_merge_20200211_0929.py b/weblab/experiments/migrations/0024_merge_20200211_0929.py new file mode 100644 index 000000000..abe0fe6e8 --- /dev/null +++ b/weblab/experiments/migrations/0024_merge_20200211_0929.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2020-02-11 09:29 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('experiments', '0023_auto_20200204_1533'), + ('experiments', '0019_auto_20200131_1612'), + ] + + operations = [ + ] diff --git a/weblab/experiments/models.py b/weblab/experiments/models.py index c9bc72696..589d81677 100644 --- a/weblab/experiments/models.py +++ b/weblab/experiments/models.py @@ -7,6 +7,7 @@ from core.models import FileCollectionMixin, UserCreatedModelMixin from core.visibility import Visibility, get_joint_visibility, visibility_check from entities.models import ModelEntity, ProtocolEntity +from repocache.models import CachedModelVersion, CachedProtocolVersion class Experiment(UserCreatedModelMixin, models.Model): @@ -22,11 +23,8 @@ class Experiment(UserCreatedModelMixin, models.Model): model = models.ForeignKey(ModelEntity, related_name='model_experiments') protocol = models.ForeignKey(ProtocolEntity, related_name='protocol_experiments') - # Note that we can't use a ForeignKey here, because versions of models and protocols - # are not stored in the DB - they are just commits in the associated git repo. - # So instead we store the full git SHA as a string. - model_version = models.CharField(max_length=50) - protocol_version = models.CharField(max_length=50) + model_version = models.ForeignKey(CachedModelVersion, default=None, null=False, related_name='model_ver_exps') + protocol_version = models.ForeignKey(CachedProtocolVersion, default=None, null=False, related_name='pro_ver_exps') class Meta: unique_together = ('model', 'protocol', 'model_version', 'protocol_version') @@ -52,22 +50,15 @@ def get_name(self, model_version=False, proto_version=False): """ model_part = self.model.name if model_version: - model_part += '@' + self.nice_model_version + model_part += '@' + self.model_version.nice_version() proto_part = self.protocol.name if proto_version: - proto_part += '@' + self.nice_protocol_version + proto_part += '@' + self.protocol_version.nice_version() return '{0} / {1}'.format(model_part, proto_part) @property def visibility(self): - return get_joint_visibility( - self.model.get_version_visibility( - self.model_version, - default=self.model.DEFAULT_VISIBILITY), - self.protocol.get_version_visibility( - self.protocol_version, - default=self.protocol.DEFAULT_VISIBILITY), - ) + return get_joint_visibility(self.model_version.visibility, self.protocol_version.visibility) @property def viewers(self): @@ -104,12 +95,12 @@ def latest_version(self): @property def nice_model_version(self): """Use tags to give a nicer representation of the commit id""" - return self.model.nice_version(self.model_version) + return self.model_version.nice_version() @property def nice_protocol_version(self): """Use tags to give a nicer representation of the commit id""" - return self.protocol.nice_version(self.protocol_version) + return self.protocol_version.nice_version() @property def latest_result(self): diff --git a/weblab/experiments/processing.py b/weblab/experiments/processing.py index c7a17d901..92b2f8607 100644 --- a/weblab/experiments/processing.py +++ b/weblab/experiments/processing.py @@ -49,8 +49,8 @@ def submit_experiment(model, model_version, protocol, protocol_version, user, re experiment, _ = Experiment.objects.get_or_create( model=model, protocol=protocol, - model_version=model_version, - protocol_version=protocol_version, + model_version=model.repocache.get_version(model_version), + protocol_version=protocol.repocache.get_version(protocol_version), defaults={ 'author': user, } diff --git a/weblab/experiments/tests/test_models.py b/weblab/experiments/tests/test_models.py index 4cf2734bb..e83ff92fd 100644 --- a/weblab/experiments/tests/test_models.py +++ b/weblab/experiments/tests/test_models.py @@ -20,11 +20,13 @@ class TestExperiment: def test_name(self, helpers): model = recipes.model.make(name='my model') protocol = recipes.protocol.make(name='my protocol') + model_version = helpers.add_version(model, tag_name='v1') + protocol_version = helpers.add_version(protocol, tag_name='v2') experiment = recipes.experiment.make( model=model, - model_version=helpers.add_version(model, tag_name='v1').sha, + model_version=model.repocache.get_version(model_version.sha), protocol=protocol, - protocol_version=helpers.add_version(protocol, tag_name='v2').sha, + protocol_version=protocol.repocache.get_version(protocol_version.sha), ) assert str(experiment) == experiment.name == 'my model / my protocol' @@ -53,8 +55,8 @@ def test_latest_result_empty_if_no_versions(self): def test_nice_versions(self, experiment_version): exp = experiment_version.experiment - assert exp.nice_model_version == exp.model.repo.latest_commit.sha[:8] + '...' - assert exp.nice_protocol_version == exp.protocol.repo.latest_commit.sha[:8] + '...' + assert exp.nice_model_version == exp.model.repocache.latest_version.sha[:8] + '...' + assert exp.nice_protocol_version == exp.protocol.repocache.latest_version.sha[:8] + '...' exp.model.repo.tag('v1') populate_entity_cache(exp.model) @@ -68,24 +70,24 @@ def test_nice_versions(self, experiment_version): def test_visibility(self, helpers): model = recipes.model.make() protocol = recipes.protocol.make() - mv1 = helpers.add_version(model, visibility='private').sha - mv2 = helpers.add_version(model, visibility='public').sha - pv1 = helpers.add_version(protocol, visibility='private').sha - pv2 = helpers.add_version(protocol, visibility='public').sha + mv1 = helpers.add_version(model, visibility='private') + mv2 = helpers.add_version(model, visibility='public') + pv1 = helpers.add_version(protocol, visibility='private') + pv2 = helpers.add_version(protocol, visibility='public') assert recipes.experiment.make( - model=model, model_version=mv2, - protocol=protocol, protocol_version=pv2 + model=model, model_version=model.repocache.get_version(mv2.sha), + protocol=protocol, protocol_version=protocol.repocache.get_version(pv2.sha) ).visibility == 'public' assert recipes.experiment.make( - model=model, model_version=mv1, - protocol=protocol, protocol_version=pv1 + model=model, model_version=model.repocache.get_version(mv1.sha), + protocol=protocol, protocol_version=protocol.repocache.get_version(pv1.sha) ).visibility == 'private' assert recipes.experiment.make( - model=model, model_version=mv1, - protocol=protocol, protocol_version=pv2 + model=model, model_version=model.repocache.get_version(mv1.sha), + protocol=protocol, protocol_version=protocol.repocache.get_version(pv2.sha) ).visibility == 'private' def test_viewers(self, helpers, user): @@ -100,8 +102,8 @@ def test_viewers(self, helpers, user): exp = recipes.experiment_version.make( experiment__model=model, experiment__protocol=protocol, - experiment__model_version=mv.sha, - experiment__protocol_version=pv.sha, + experiment__model_version=model.repocache.get_version(mv.sha), + experiment__protocol_version=protocol.repocache.get_version(pv.sha), ).experiment assert user not in exp.viewers diff --git a/weblab/experiments/tests/test_processing.py b/weblab/experiments/tests/test_processing.py index 1dba8da28..2820a9df6 100644 --- a/weblab/experiments/tests/test_processing.py +++ b/weblab/experiments/tests/test_processing.py @@ -53,8 +53,8 @@ def test_creates_new_experiment_and_side_effects( assert version.experiment.model == model assert version.experiment.protocol == protocol assert version.author == user - assert version.experiment.model_version == model_version - assert version.experiment.protocol_version == protocol_version + assert version.experiment.model_version.sha == model_version + assert version.experiment.protocol_version.sha == protocol_version assert version.experiment.author == user assert version.status == ExperimentVersion.STATUS_QUEUED @@ -99,13 +99,15 @@ def test_uses_existing_experiment(self, mock_post, user, model_with_version, protocol_with_version): model = model_with_version protocol = protocol_with_version - model_version = model.repo.latest_commit.sha - protocol_version = protocol.repo.latest_commit.sha + model_version = model.repocache.latest_version + protocol_version = protocol.repocache.latest_version experiment = recipes.experiment.make(model=model, model_version=model_version, - protocol=protocol, protocol_version=protocol_version) + protocol=protocol, + protocol_version=protocol_version) - version, is_new = submit_experiment(model, model_version, protocol, protocol_version, user, False) + version, is_new = submit_experiment(model, model_version.sha, + protocol, protocol_version.sha, user, False) assert is_new assert version.experiment == experiment @@ -114,12 +116,12 @@ def test_raises_exception_on_webservice_error(self, mock_post, user, model_with_version, protocol_with_version): model = model_with_version protocol = protocol_with_version - model_version = model.repo.latest_commit.sha - protocol_version = protocol.repo.latest_commit.sha + model_version = model.repocache.latest_version + protocol_version = protocol.repocache.latest_version mock_post.side_effect = generate_response('something %s') with pytest.raises(ProcessingException): - submit_experiment(model, model_version, protocol, protocol_version, user, False) + submit_experiment(model, model_version.sha, protocol, protocol_version.sha, user, False) # There should be no running experiment assert RunningExperiment.objects.count() == 0 @@ -137,12 +139,12 @@ def test_records_submission_error(self, mock_post, user, model_with_version, protocol_with_version): model = model_with_version protocol = protocol_with_version - model_version = model.repo.latest_commit.sha - protocol_version = protocol.repo.latest_commit.sha + model_version = model.repocache.latest_version + protocol_version = protocol.repocache.latest_version mock_post.side_effect = generate_response('%s an error occurred') - version, is_new = submit_experiment(model, model_version, protocol, protocol_version, user, False) + version, is_new = submit_experiment(model, model_version.sha, protocol, protocol_version.sha, user, False) assert is_new assert version.status == ExperimentVersion.STATUS_FAILED @@ -153,12 +155,12 @@ def test_records_inapplicable_result(self, mock_post, user, model_with_version, protocol_with_version): model = model_with_version protocol = protocol_with_version - model_version = model.repo.latest_commit.sha - protocol_version = protocol.repo.latest_commit.sha + model_version = model.repocache.latest_version + protocol_version = protocol.repocache.latest_version mock_post.side_effect = generate_response('%s inapplicable') - version, is_new = submit_experiment(model, model_version, protocol, protocol_version, user, False) + version, is_new = submit_experiment(model, model_version.sha, protocol, protocol_version.sha, user, False) assert is_new assert version.status == ExperimentVersion.STATUS_INAPPLICABLE diff --git a/weblab/experiments/tests/test_views.py b/weblab/experiments/tests/test_views.py index 0e8eec778..a60c0d1d4 100644 --- a/weblab/experiments/tests/test_views.py +++ b/weblab/experiments/tests/test_views.py @@ -42,8 +42,8 @@ def add_permission(user, perm): def make_experiment(model, model_version, protocol, protocol_version): """Create an experiment in the DB with a single version.""" exp = recipes.experiment.make( - model=model, model_version=str(model_version), - protocol=protocol, protocol_version=str(protocol_version)) + model=model, model_version=model_version, + protocol=protocol, protocol_version=protocol_version) recipes.experiment_version.make(experiment=exp) return exp @@ -137,9 +137,9 @@ def test_matrix_json(self, client, quick_experiment_version): assert 'getMatrix' in data assert len(data['getMatrix']['models']) == 1 - assert str(exp.model_version) in data['getMatrix']['models'] + assert str(exp.model_version.sha) in data['getMatrix']['models'] assert len(data['getMatrix']['protocols']) == 1 - assert str(exp.protocol_version) in data['getMatrix']['protocols'] + assert str(exp.protocol_version.sha) in data['getMatrix']['protocols'] assert len(data['getMatrix']['experiments']) == 1 assert str(exp.pk) in data['getMatrix']['experiments'] @@ -182,22 +182,22 @@ def test_view_my_experiments_with_moderated_flags( my_moderated_protocol = recipes.protocol.make(author=logged_in_user) helpers.add_fake_version(my_moderated_protocol, visibility='moderated') - my_version = make_experiment(my_model, my_model_version.sha, my_protocol, my_protocol_version.sha) + my_version = make_experiment(my_model, my_model_version, my_protocol, my_protocol_version) with_moderated_model = make_experiment( - moderated_model, moderated_model.repo.latest_commit.sha, - my_protocol, my_protocol_version.sha, + moderated_model, moderated_model.repocache.latest_version, + my_protocol, my_protocol_version, ) with_moderated_protocol = make_experiment( - my_model, my_model_version.sha, - moderated_protocol, moderated_protocol.repo.latest_commit.sha, + my_model, my_model_version, + moderated_protocol, moderated_protocol.repocache.latest_version, ) with_my_moderated_model = make_experiment( - my_moderated_model, my_moderated_model.repocache.latest_version.sha, - my_protocol, my_protocol_version.sha, + my_moderated_model, my_moderated_model.repocache.latest_version, + my_protocol, my_protocol_version, ) with_my_moderated_protocol = make_experiment( - my_model, my_model_version.sha, - my_moderated_protocol, my_moderated_protocol.repocache.latest_version.sha, + my_model, my_model_version, + my_moderated_protocol, my_moderated_protocol.repocache.latest_version, ) # All my experiments plus ones involving moderated entities @@ -256,8 +256,8 @@ def test_view_public_experiments(self, client, logged_in_user, other_user, helpe my_protocol_private_version = helpers.add_fake_version(my_protocol_private, visibility='private') exp1 = make_experiment( - my_model_moderated, my_model_moderated_version.sha, - my_protocol_private, my_protocol_private_version.sha, + my_model_moderated, my_model_moderated_version, + my_protocol_private, my_protocol_private_version, ) # Someone else's public model with my public protocol: should be visible @@ -271,16 +271,16 @@ def test_view_public_experiments(self, client, logged_in_user, other_user, helpe my_protocol_second_private_version = helpers.add_fake_version(my_protocol_public, visibility='private') exp2 = make_experiment( - other_model_public, other_model_public_version.sha, - my_protocol_public, my_protocol_public_version.sha, + other_model_public, other_model_public_version, + my_protocol_public, my_protocol_public_version, ) exp2_model_private = make_experiment( # noqa: F841 - other_model_public, other_model_second_private_version.sha, - my_protocol_public, my_protocol_public_version.sha, + other_model_public, other_model_second_private_version, + my_protocol_public, my_protocol_public_version, ) exp2_protocol_private = make_experiment( - other_model_public, other_model_public_version.sha, - my_protocol_public, my_protocol_second_private_version.sha, + other_model_public, other_model_public_version, + my_protocol_public, my_protocol_second_private_version, ) # Someone else's public model and moderated protocol: should be visible @@ -289,12 +289,12 @@ def test_view_public_experiments(self, client, logged_in_user, other_user, helpe other_protocol_private_version = helpers.add_fake_version(other_protocol_moderated, visibility='private') exp3 = make_experiment( - other_model_public, other_model_public_version.sha, - other_protocol_moderated, other_protocol_moderated_version.sha, + other_model_public, other_model_public_version, + other_protocol_moderated, other_protocol_moderated_version, ) exp3_protocol_private = make_experiment( - other_model_public, other_model_second_private_version.sha, - other_protocol_moderated, other_protocol_private_version.sha, + other_model_public, other_model_second_private_version, + other_protocol_moderated, other_protocol_private_version, ) # Other's private model, my public protocol: should not be visible @@ -302,8 +302,8 @@ def test_view_public_experiments(self, client, logged_in_user, other_user, helpe other_model_private_version = helpers.add_fake_version(other_model_private, visibility='private') exp4 = make_experiment( # noqa: F841 - other_model_private, other_model_private_version.sha, - my_protocol_public, my_protocol_public_version.sha, + other_model_private, other_model_private_version, + my_protocol_public, my_protocol_public_version, ) response = client.get('/experiments/matrix?subset=public') @@ -347,8 +347,8 @@ def test_view_moderated_experiments(self, client, logged_in_user, other_user, he other_protocol_public_version = helpers.add_fake_version(other_protocol_public, visibility='public') exp1 = make_experiment( - my_model_public, my_model_public_version.sha, - other_protocol_public, other_protocol_public_version.sha, + my_model_public, my_model_public_version, + other_protocol_public, other_protocol_public_version, ) # My public model with somebody else's moderated protocol: should not be visible @@ -356,8 +356,8 @@ def test_view_moderated_experiments(self, client, logged_in_user, other_user, he other_protocol_moderated_version = helpers.add_fake_version(other_protocol_moderated, visibility='moderated') exp2 = make_experiment( - my_model_public, my_model_public_version.sha, - other_protocol_moderated, other_protocol_moderated_version.sha, + my_model_public, my_model_public_version, + other_protocol_moderated, other_protocol_moderated_version, ) # Someone else's moderated model and public protocol: should not be visible @@ -365,21 +365,21 @@ def test_view_moderated_experiments(self, client, logged_in_user, other_user, he other_model_moderated_version = helpers.add_fake_version(other_model_moderated, visibility='moderated') exp3 = make_experiment( # noqa: F841 - other_model_moderated, other_model_moderated_version.sha, - other_protocol_public, other_protocol_public_version.sha, + other_model_moderated, other_model_moderated_version, + other_protocol_public, other_protocol_public_version, ) # Someone else's moderated model and moderated protocol: should be visible exp4 = make_experiment( - other_model_moderated, other_model_moderated_version.sha, - other_protocol_moderated, other_protocol_moderated_version.sha, + other_model_moderated, other_model_moderated_version, + other_protocol_moderated, other_protocol_moderated_version, ) # A later public version shouldn't show up other_model_second_public_version = helpers.add_fake_version(other_model_moderated, visibility='public') exp4_public = make_experiment( - other_model_moderated, other_model_second_public_version.sha, - other_protocol_moderated, other_protocol_moderated_version.sha, + other_model_moderated, other_model_second_public_version, + other_protocol_moderated, other_protocol_moderated_version, ) response = client.get('/experiments/matrix') @@ -408,8 +408,8 @@ def test_submatrix(self, client, helpers, quick_experiment_version): other_protocol = recipes.protocol.make() other_protocol_version = helpers.add_fake_version(other_protocol) make_experiment( - other_model, other_model_version.sha, - other_protocol, other_protocol_version.sha, + other_model, other_model_version, + other_protocol, other_protocol_version, ) # Throw in a non-existent protocol so we can make sure it gets ignored @@ -429,15 +429,15 @@ def test_submatrix(self, client, helpers, quick_experiment_version): models = data['getMatrix']['models'] assert len(models) == 1 - assert exp.model_version in models - assert models[exp.model_version]['id'] == exp.model_version - assert models[exp.model_version]['entityId'] == exp.model.pk + assert str(exp.model_version.sha) in models + assert models[str(exp.model_version.sha)]['id'] == str(exp.model_version.sha) + assert models[str(exp.model_version.sha)]['entityId'] == exp.model.pk protocols = data['getMatrix']['protocols'] assert len(protocols) == 1 - assert exp.protocol_version in protocols - assert protocols[exp.protocol_version]['id'] == exp.protocol_version - assert protocols[exp.protocol_version]['entityId'] == exp.protocol.pk + assert str(exp.protocol_version.sha) in protocols + assert protocols[str(exp.protocol_version.sha)]['id'] == str(exp.protocol_version.sha) + assert protocols[str(exp.protocol_version.sha)]['entityId'] == exp.protocol.pk experiments = data['getMatrix']['experiments'] assert len(experiments) == 1 @@ -445,21 +445,21 @@ def test_submatrix(self, client, helpers, quick_experiment_version): def test_submatrix_with_model_versions(self, client, helpers, quick_experiment_version): exp = quick_experiment_version.experiment - v1 = str(exp.model_version) - v2 = str(helpers.add_fake_version(exp.model).sha) + v1 = exp.model_version + v2 = helpers.add_fake_version(exp.model) helpers.add_fake_version(exp.model) # v3, not used # Add an experiment with a different model, which shouldn't appear other_model = recipes.model.make() other_model_version = helpers.add_fake_version(other_model, 'public') - make_experiment(other_model, other_model_version.sha, exp.protocol, exp.protocol_version) + make_experiment(other_model, other_model_version, exp.protocol, exp.protocol_version) response = client.get( '/experiments/matrix', { 'subset': 'all', 'modelIds[]': [exp.model.pk], - 'modelVersions[]': [v1, v2], + 'modelVersions[]': [str(v1.sha), str(v2.sha)], } ) @@ -467,14 +467,14 @@ def test_submatrix_with_model_versions(self, client, helpers, quick_experiment_v data = json.loads(response.content.decode()) assert 'getMatrix' in data - assert set(data['getMatrix']['models'].keys()) == {v1, v2} + assert set(data['getMatrix']['models'].keys()) == {str(v1.sha), str(v2.sha)} assert set(data['getMatrix']['experiments'].keys()) == {str(exp.pk)} def test_submatrix_with_all_model_versions(self, client, helpers, quick_experiment_version): exp = quick_experiment_version.experiment - v1 = str(exp.model_version) - v2 = str(helpers.add_fake_version(exp.model).sha) - v3 = str(helpers.add_fake_version(exp.model).sha) + v1 = exp.model_version + v2 = helpers.add_fake_version(exp.model) + v3 = helpers.add_fake_version(exp.model) exp2 = make_experiment( exp.model, v2, @@ -484,7 +484,7 @@ def test_submatrix_with_all_model_versions(self, client, helpers, quick_experime # Add an experiment with a different model, which shouldn't appear other_model = recipes.model.make() other_model_version = helpers.add_fake_version(other_model, 'public') - make_experiment(other_model, other_model_version.sha, exp.protocol, exp.protocol_version) + make_experiment(other_model, other_model_version, exp.protocol, exp.protocol_version) response = client.get( '/experiments/matrix', @@ -499,7 +499,7 @@ def test_submatrix_with_all_model_versions(self, client, helpers, quick_experime data = json.loads(response.content.decode()) assert 'getMatrix' in data - assert set(data['getMatrix']['models'].keys()) == {v1, v2, v3} + assert set(data['getMatrix']['models'].keys()) == {str(v1.sha), str(v2.sha), str(v3.sha)} assert set(data['getMatrix']['experiments'].keys()) == {str(exp.pk), str(exp2.pk)} def test_submatrix_with_too_many_model_ids(self, client, helpers, quick_experiment_version): @@ -520,8 +520,8 @@ def test_submatrix_with_too_many_model_ids(self, client, helpers, quick_experime def test_submatrix_with_protocol_versions(self, client, helpers, quick_experiment_version): exp = quick_experiment_version.experiment - v1 = str(exp.protocol_version) - v2 = str(helpers.add_fake_version(exp.protocol).sha) + v1 = exp.protocol_version + v2 = helpers.add_fake_version(exp.protocol) helpers.add_fake_version(exp.protocol) # v3, not used exp2 = make_experiment( @@ -530,16 +530,17 @@ def test_submatrix_with_protocol_versions(self, client, helpers, quick_experimen ) # Add an experiment with a different protocol, which shouldn't appear - other_protocol = recipes.model.make() + other_protocol = recipes.protocol.make() other_protocol_version = helpers.add_fake_version(other_protocol, 'public') - make_experiment(exp.model, exp.model_version, other_protocol, other_protocol_version.sha) + make_experiment(exp.model, exp.model_version, + other_protocol, other_protocol_version) response = client.get( '/experiments/matrix', { 'subset': 'all', 'protoIds[]': [exp.protocol.pk], - 'protoVersions[]': [v1, v2], + 'protoVersions[]': [str(v1.sha), str(v2.sha)], } ) @@ -547,14 +548,14 @@ def test_submatrix_with_protocol_versions(self, client, helpers, quick_experimen data = json.loads(response.content.decode()) assert 'getMatrix' in data - assert set(data['getMatrix']['protocols'].keys()) == {v1, v2} + assert set(data['getMatrix']['protocols'].keys()) == {str(v1.sha), str(v2.sha)} assert set(data['getMatrix']['experiments'].keys()) == {str(exp.pk), str(exp2.pk)} def test_submatrix_with_all_protocol_versions(self, client, helpers, quick_experiment_version): exp = quick_experiment_version.experiment - v1 = str(exp.protocol_version) - v2 = str(helpers.add_fake_version(exp.protocol).sha) - v3 = str(helpers.add_fake_version(exp.protocol).sha) + v1 = exp.protocol_version + v2 = helpers.add_fake_version(exp.protocol) + v3 = helpers.add_fake_version(exp.protocol) exp2 = make_experiment( exp.model, exp.model_version, @@ -564,7 +565,8 @@ def test_submatrix_with_all_protocol_versions(self, client, helpers, quick_exper # Add an experiment with a different protocol, which shouldn't appear other_protocol = recipes.protocol.make() other_protocol_version = helpers.add_fake_version(other_protocol, 'public') - make_experiment(exp.model, exp.model_version, other_protocol, other_protocol_version.sha) + make_experiment(exp.model, exp.model_version, + other_protocol, other_protocol_version) response = client.get( '/experiments/matrix', @@ -579,7 +581,7 @@ def test_submatrix_with_all_protocol_versions(self, client, helpers, quick_exper data = json.loads(response.content.decode()) assert 'getMatrix' in data - assert set(data['getMatrix']['protocols'].keys()) == {v1, v2, v3} + assert set(data['getMatrix']['protocols'].keys()) == {str(v1.sha), str(v2.sha), str(v3.sha)} assert set(data['getMatrix']['experiments'].keys()) == {str(exp.pk), str(exp2.pk)} def test_submatrix_with_too_many_protocol_ids(self, client, helpers, quick_experiment_version): @@ -600,13 +602,13 @@ def test_submatrix_with_too_many_protocol_ids(self, client, helpers, quick_exper def test_submatrix_with_models_and_protocols_given(self, client, helpers): m1, m2 = recipes.model.make(_quantity=2) - m1v1 = helpers.add_fake_version(m1, 'public').sha - m1v2 = helpers.add_fake_version(m1, 'public').sha - m2v1 = helpers.add_fake_version(m2, 'public').sha + m1v1 = helpers.add_fake_version(m1, 'public') + m1v2 = helpers.add_fake_version(m1, 'public') + m2v1 = helpers.add_fake_version(m2, 'public') p1, p2 = recipes.protocol.make(_quantity=2) - p1v1 = helpers.add_fake_version(p1, 'public').sha - p1v2 = helpers.add_fake_version(p1, 'public').sha - p2v1 = helpers.add_fake_version(p2, 'public').sha + p1v1 = helpers.add_fake_version(p1, 'public') + p1v2 = helpers.add_fake_version(p1, 'public') + p2v1 = helpers.add_fake_version(p2, 'public') exp1 = make_experiment(m1, m1v1, p1, p1v1) exp2 = make_experiment(m1, m1v2, p1, p1v2) @@ -628,8 +630,8 @@ def test_submatrix_with_models_and_protocols_given(self, client, helpers): assert response.status_code == 200 data = json.loads(response.content.decode()) assert 'getMatrix' in data - assert set(data['getMatrix']['models'].keys()) == {str(m1v1), str(m1v2)} - assert set(data['getMatrix']['protocols'].keys()) == {str(p1v1), str(p1v2)} + assert set(data['getMatrix']['models'].keys()) == {str(m1v1.sha), str(m1v2.sha)} + assert set(data['getMatrix']['protocols'].keys()) == {str(p1v1.sha), str(p1v2.sha)} assert set(data['getMatrix']['experiments'].keys()) == {str(exp1.pk), str(exp2.pk)} # Now select only some versions @@ -638,17 +640,17 @@ def test_submatrix_with_models_and_protocols_given(self, client, helpers): { 'subset': 'all', 'modelIds[]': [m1.pk], - 'modelVersions[]': [m1v1], + 'modelVersions[]': [str(m1v1.sha)], 'protoIds[]': [p1.pk], - 'protoVersions[]': [p1v1], + 'protoVersions[]': [str(p1v1.sha)], } ) assert response.status_code == 200 data = json.loads(response.content.decode()) assert 'getMatrix' in data - assert set(data['getMatrix']['models'].keys()) == {str(m1v1)} - assert set(data['getMatrix']['protocols'].keys()) == {str(p1v1)} + assert set(data['getMatrix']['models'].keys()) == {str(m1v1.sha)} + assert set(data['getMatrix']['protocols'].keys()) == {str(p1v1.sha)} assert set(data['getMatrix']['experiments'].keys()) == {str(exp1.pk)} def test_experiment_without_version_is_ignored( @@ -656,9 +658,9 @@ def test_experiment_without_version_is_ignored( ): recipes.experiment.make( model=public_model, - model_version=public_model.repo.latest_commit.sha, + model_version=public_model.repocache.latest_version, protocol=public_protocol, - protocol_version=public_protocol.repo.latest_commit.sha, + protocol_version=public_protocol.repocache.latest_version, ) response = client.get('/experiments/matrix?subset=all') @@ -676,7 +678,7 @@ def test_old_version_is_hidden(self, client, public_model, experiment_version, h data = json.loads(response.content.decode()) assert 'getMatrix' in data assert str(new_version.sha) in data['getMatrix']['models'] - assert str(experiment_version.experiment.protocol_version) in data['getMatrix']['protocols'] + assert str(experiment_version.experiment.protocol_version.sha) in data['getMatrix']['protocols'] assert len(data['getMatrix']['experiments']) == 0 @@ -718,8 +720,8 @@ def test_submits_experiment( # Check this has been removed from the list of planned experiments assert PlannedExperiment.objects.count() == planned_experiments - 1 assert PlannedExperiment.objects.filter( - model=model, model_version=model_version, - protocol=protocol, protocol_version=protocol_version + model=model, model_version=model.repocache.get_version(model_version), + protocol=protocol, protocol_version=protocol.repocache.get_version(protocol_version) ).count() == 0 # Check a subsequent submit gives the same experiment version back @@ -986,18 +988,18 @@ def test_get_queryset(self, logged_in_user, client, helpers): recipes.experiment_version.make( status=ExperimentVersion.STATUS_SUCCESS, experiment__model=model_1, - experiment__model_version=model_1_version.sha, + experiment__model_version=model_1.repocache.get_version(model_1_version.sha), experiment__protocol=protocol_1, - experiment__protocol_version=protocol_1_version.sha, + experiment__protocol_version=protocol_1.repocache.get_version(protocol_1_version.sha), author=logged_in_user, ) exp_version_2 = recipes.experiment_version.make( status=ExperimentVersion.STATUS_QUEUED, experiment__model=model_1, - experiment__model_version=model_1_version.sha, + experiment__model_version=model_1.repocache.get_version(model_1_version.sha), experiment__protocol=protocol_1, - experiment__protocol_version=protocol_1_version2.sha, + experiment__protocol_version=protocol_1.repocache.get_version(protocol_1_version2.sha), author=logged_in_user, ) running_exp_version2 = recipes.running_experiment.make(experiment_version=exp_version_2) @@ -1005,9 +1007,9 @@ def test_get_queryset(self, logged_in_user, client, helpers): exp_version_3 = recipes.experiment_version.make( status=ExperimentVersion.STATUS_RUNNING, experiment__model=model_1, - experiment__model_version=model_1_version.sha, + experiment__model_version=model_1.repocache.get_version(model_1_version.sha), experiment__protocol=protocol_2, - experiment__protocol_version=protocol_2_version.sha, + experiment__protocol_version=protocol_2.repocache.get_version(protocol_2_version.sha), author=logged_in_user, ) running_exp_version3 = recipes.running_experiment.make(experiment_version=exp_version_3) @@ -1116,7 +1118,7 @@ def test_compare_experiments(self, client, experiment_version, helpers): experiment__model=exp.model, experiment__model_version=exp.model_version, experiment__protocol=protocol, - experiment__protocol_version=protocol_commit.sha, + experiment__protocol_version=protocol.repocache.get_version(protocol_commit.sha), ) response = client.get( @@ -1139,7 +1141,7 @@ def test_only_compare_visible_experiments(self, client, experiment_version, help experiment__model=exp.model, experiment__model_version=exp.model_version, experiment__protocol=proto, - experiment__protocol_version=proto_commit.sha, + experiment__protocol_version=proto.repocache.get_version(proto_commit.sha), ) response = client.get( @@ -1154,6 +1156,7 @@ def test_only_compare_visible_experiments(self, client, experiment_version, help def test_no_visible_experiments(self, client, experiment_version): proto = experiment_version.experiment.protocol proto.set_version_visibility('latest', 'private') + experiment_version.experiment.protocol_version.refresh_from_db() assert experiment_version.visibility == 'private' response = client.get('/experiments/compare/%d' % (experiment_version.id)) @@ -1176,7 +1179,7 @@ def test_compare_experiments(self, client, experiment_version, helpers): experiment__model=exp.model, experiment__model_version=exp.model_version, experiment__protocol=protocol, - experiment__protocol_version=protocol_commit.sha, + experiment__protocol_version=protocol.repocache.get_version(protocol_commit.sha), ) response = client.get( @@ -1189,7 +1192,7 @@ def test_compare_experiments(self, client, experiment_version, helpers): assert versions[0]['versionId'] == experiment_version.id assert versions[1]['versionId'] == version2.id assert versions[0]['modelName'] == exp.model.name - assert versions[0]['modelVersion'] == exp.model_version + assert versions[0]['modelVersion'] == exp.model_version.sha assert versions[0]['protoName'] == exp.protocol.name assert versions[0]['protoVersion'] == 'v1' assert versions[0]['name'] == exp.name @@ -1206,7 +1209,7 @@ def test_only_compare_visible_experiments(self, client, experiment_version, help experiment__model=exp.model, experiment__model_version=exp.model_version, experiment__protocol=proto, - experiment__protocol_version=proto_commit.sha, + experiment__protocol_version=proto.repocache.get_version(proto_commit.sha), ) response = client.get( @@ -1233,9 +1236,9 @@ def test_file_json(self, client, archive_file_path, helpers, experiment_version) version2 = recipes.experiment_version.make( status='SUCCESS', experiment__model=exp.model, - experiment__model_version=exp.model_version, + experiment__model_version=exp.model.repocache.get_version(exp.model_version.sha), experiment__protocol=protocol, - experiment__protocol_version=protocol_commit.sha, + experiment__protocol_version=protocol.repocache.get_version(protocol_commit.sha), ) version2.mkdir() shutil.copyfile(archive_file_path, str(version2.archive_path)) @@ -1420,9 +1423,9 @@ def test_private_expt_visible_to_self( protocol_version = helpers.add_version(protocol, visibility='public') experiment_version = recipes.experiment_version.make( experiment__model=model, - experiment__model_version=model_version.sha, + experiment__model_version=model.repocache.get_version(model_version.sha), experiment__protocol=protocol, - experiment__protocol_version=protocol_version.sha, + experiment__protocol_version=protocol.repocache.get_version(protocol_version.sha), ) experiment_version.mkdir() diff --git a/weblab/experiments/views.py b/weblab/experiments/views.py index 2c9f7cab4..047aa7929 100644 --- a/weblab/experiments/views.py +++ b/weblab/experiments/views.py @@ -4,12 +4,7 @@ from django.contrib.admin.views.decorators import staff_member_required from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin from django.core.urlresolvers import reverse -from django.db.models import ( - F, - OuterRef, - Q, - Subquery, -) +from django.db.models import F, Q from django.db.models.functions import Coalesce from django.http import Http404, JsonResponse from django.shortcuts import get_object_or_404, redirect @@ -25,7 +20,7 @@ from core.visibility import VisibilityMixin from datasets import views as dataset_views from entities.models import ModelEntity, ProtocolEntity -from repocache.models import CACHED_VERSION_TYPE_MAP, CachedModelVersion, CachedProtocolVersion +from repocache.models import CACHED_VERSION_TYPE_MAP from .forms import ExperimentSimulateCallbackForm from .models import ( @@ -104,11 +99,11 @@ def experiment_version_json(cls, version): 'entity_id': version.experiment.id, 'latestResult': version.status, 'protocol': cls.entity_json( - version.experiment.protocol, version.experiment.protocol_version, + version.experiment.protocol, version.experiment.protocol_version.sha, extend_name=True, visibility=version.protocol_visibility, author=version.protocol_author, ), 'model': cls.entity_json( - version.experiment.model, version.experiment.model_version, + version.experiment.model, version.experiment.model_version.sha, extend_name=True, visibility=version.model_visibility, author=version.model_author, ), 'url': reverse( @@ -268,17 +263,9 @@ def get(self, request, *args, **kwargs): experiments = {} q_experiments = Experiment.objects.filter( model__in=q_models, - model_version__in=model_versions.keys(), + model_version__in=q_model_versions, protocol__in=q_protocols, - protocol_version__in=protocol_versions.keys(), - ) - q_cached_protocol = CachedProtocolVersion.objects.filter( - entity__entity=OuterRef('experiment__protocol'), - sha=OuterRef('experiment__protocol_version'), - ) - q_cached_model = CachedModelVersion.objects.filter( - entity__entity=OuterRef('experiment__model'), - sha=OuterRef('experiment__model_version'), + protocol_version__in=q_protocol_versions, ) q_experiment_versions = ExperimentVersion.objects.filter( experiment__in=q_experiments, @@ -292,8 +279,8 @@ def get(self, request, *args, **kwargs): 'experiment__protocol', 'experiment__model', 'experiment__protocol__cachedprotocol', 'experiment__model__cachedmodel', ).annotate( - protocol_visibility=Subquery(q_cached_protocol.values('visibility')[:1]), - model_visibility=Subquery(q_cached_model.values('visibility')[:1]), + protocol_visibility=F('experiment__protocol_version__visibility'), + model_visibility=F('experiment__model_version__visibility'), protocol_author=F('experiment__protocol__author__full_name'), model_author=F('experiment__model__author__full_name'), ) @@ -326,8 +313,8 @@ def post(self, request, *args, **kwargs): exp = exp_ver.experiment model = exp.model protocol = exp.protocol - model_version = exp.model_version - protocol_version = exp.protocol_version + model_version = exp.model_version.sha + protocol_version = exp.protocol_version.sha else: model = get_object_or_404(ModelEntity, pk=request.POST['model']) protocol = get_object_or_404(ProtocolEntity, pk=request.POST['protocol']) @@ -450,8 +437,8 @@ def _version_json(self, version, model_version_in_name, protocol_version_in_name 'versionId': version.id, 'modelName': exp.model.name, 'protoName': exp.protocol.name, - 'modelVersion': exp.model.repocache.get_name_for_version(exp.model_version), - 'protoVersion': exp.protocol.repocache.get_name_for_version(exp.protocol_version), + 'modelVersion': exp.model_version.get_name(), + 'protoVersion': exp.protocol_version.get_name(), 'runNumber': version.run_number, }) return details diff --git a/weblab/repocache/models.py b/weblab/repocache/models.py index c2e1e20ca..6549b0a75 100644 --- a/weblab/repocache/models.py +++ b/weblab/repocache/models.py @@ -65,7 +65,10 @@ def get_version(self, sha): else: return self.versions.get(sha=sha) except ObjectDoesNotExist: - raise RepoCacheMiss("Entity version not found") + try: + return self.tags.get(tag=sha).version + except ObjectDoesNotExist: + raise RepoCacheMiss("Entity version not found") def get_name_for_version(self, sha): """Get a human-friendly display name for the given version @@ -74,11 +77,9 @@ def get_name_for_version(self, sha): :return: first cached tag for this version, if any, or sha if not """ version = self.get_version(sha) - - first_tag = version.tags.first() - if first_tag is not None: - return first_tag.tag - return sha + if sha == 'latest' and version.tags.count() == 0: + return 'latest' + return version.get_name() def add_version(self, sha): """ @@ -154,6 +155,26 @@ def tag(self, tagname): tag=tagname, ) + def get_name(self): + """ + Get name for this version + """ + first_tag = self.tags.first() + if first_tag is not None: + return first_tag.tag + return self.sha + + def nice_version(self): + """ + Returns tag/sha with ellipses + + :return version_name: string with the sha_or_tag formatted + """ + version_name = self.get_name() + if len(version_name) > 20: + version_name = version_name[:8] + '...' + return version_name + class CachedEntityTag(models.Model): """ diff --git a/weblab/repocache/tests/test_models.py b/weblab/repocache/tests/test_models.py index f9ea415c4..436391063 100644 --- a/weblab/repocache/tests/test_models.py +++ b/weblab/repocache/tests/test_models.py @@ -126,7 +126,14 @@ def test_get_name_for_version(self, helpers): populate_entity_cache(model) assert model.repocache.get_name_for_version(commit.sha) == 'v1' assert model.repocache.get_name_for_version('latest') == 'v1' - - # get_name_for_version must be sha or latest + assert model.repocache.get_name_for_version('v1') == 'v1' with pytest.raises(RepoCacheMiss): - model.repocache.get_name_for_version('v1') + model.repocache.get_name_for_version('random value') + + def test_nice_version(self, model_with_version): + version = model_with_version.cachedentity.latest_version + assert version.nice_version() == '%s...' % version.sha[:8] + + model_with_version.repo.tag('v1') + populate_entity_cache(model_with_version) + assert version.nice_version() == 'v1'