Skip to content

Rerun experiments enhancement #302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sroderick-g5sro merged 6 commits into from
Oct 20, 2020
+52 −46
Merged

Rerun experiments enhancement #302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a81d700
Initial write of rerun experiments enhancement
sroderick-g5sro Oct 14, 2020
bef37ee
Create test for rerun experiments enhancement
sroderick-g5sro Oct 16, 2020
5c03af0
Update weblab/entities/tests/test_views.py
sroderick-g5sro Oct 16, 2020
6b75514
Update weblab/entities/tests/test_views.py
sroderick-g5sro Oct 16, 2020
567a970
Changed documentation
sroderick-g5sro Oct 16, 2020
53d2482
Merge remote-tracking branch 'origin/230-rerun-experiments' into 230-…
sroderick-g5sro Oct 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 21 additions & 16 deletions weblab/entities/processing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@
from django.core.urlresolvers import reverse
from django.db import IntegrityError

from experiments.models import Experiment, PlannedExperiment
from experiments.models import PlannedExperiment
from repocache.models import ProtocolInterface, ProtocolIoputs

from .models import AnalysisTask
from .models import AnalysisTask, ModelEntity, ProtocolEntity


logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -160,8 +160,10 @@ def process_check_protocol_callback(data):
def record_experiments_to_run(user, entity, commit):
"""Record what experiments to run automatically on a new entity version.

Any experiments that were run with the parent version(s) (that are visible to the user)
should be repeated with the new version.
Find all experiments run with the entity for which we're adding a new version (consider case of a model)
Get the list of corresponding protocol IDs. We now know both our model ID, and all protocols that have
previously had any version run on any version of this model.
Run our new model version under the latest (visible) version of all those protocols.

:param user: the user that created the new version
:param entity: the entity that has had a new version created
Expand All @@ -171,21 +173,24 @@ def record_experiments_to_run(user, entity, commit):
entity.entity_type: entity,
entity.entity_type + '_version': commit.sha,
}
for parent in commit.parents:
# Find visible experiments involving this parent
parent_kwargs = {
entity.entity_type: entity,
entity.entity_type + '_version': entity.repocache.get_version(parent.sha),
}
for expt in Experiment.objects.filter(**parent_kwargs):
if expt.is_visible_to_user(user):
if entity.other_type == entity.ENTITY_TYPE_MODEL:
other_model = ModelEntity
else:
other_model = ProtocolEntity
search_args = {
entity.other_type + '_experiments__' + entity.entity_type: entity,
}
other_entities = other_model.objects.visible_to_user(user).filter(**search_args)
for other_entity in other_entities:
# Look for latest visible version
for other_version in other_entity.cachedentity.versions.all():
if other_entity.is_version_visible_to_user(other_version.sha, user):
# Record the new experiment to run
kwargs = {
'submitter': user,
'model_id': expt.model_id,
'model_version': expt.model_version.sha,
'protocol_id': expt.protocol_id,
'protocol_version': expt.protocol_version.sha,
entity.other_type: other_entity,
entity.other_type + '_version': other_version.sha,
}
kwargs.update(new_version_kwargs)
PlannedExperiment.objects.get_or_create(**kwargs)
break
61 changes: 31 additions & 30 deletions weblab/entities/tests/test_views.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1347,15 +1347,16 @@ def test_rolls_back_if_tag_exists(self, logged_in_user, client, helpers):
@pytest.mark.parametrize("route", ['main_form', 'edit_file'])
def test_rerun_experiments(self, logged_in_user, other_user, client, helpers, route):
# Set up previous experiments
model = recipes.model.make(author=logged_in_user)
model_first_commit = helpers.add_version(model, visibility='private')
model_commit = helpers.add_version(model, visibility='private')
other_model = recipes.model.make(author=other_user)
other_model_commit = helpers.add_version(other_model, visibility='public')
m1 = recipes.model.make(author=logged_in_user)
m1v1 = helpers.add_version(m1, visibility='private')
m1v2 = helpers.add_version(m1, visibility='private')

m2 = recipes.model.make(author=other_user)
m2v1 = helpers.add_version(m2, visibility='public')

def _add_experiment(proto_author, proto_vis, shared=False,
proto=None, proto_commit=None,
model=model, model_commit=model_commit):
model=m1, model_commit=m1v1):
if proto is None:
proto = recipes.protocol.make(author=proto_author)
proto_commit = helpers.add_version(proto, visibility=proto_vis)
Expand All @@ -1370,29 +1371,29 @@ def _add_experiment(proto_author, proto_vis, shared=False,
assign_perm('edit_entity', logged_in_user, proto)
return proto

my_private_protocol = _add_experiment(logged_in_user, 'private') # Re-run case 1
_add_experiment(logged_in_user, 'private', # Shouldn't be re-run
model=model, model_commit=model_first_commit)
_add_experiment(logged_in_user, 'private', # Does get re-run because same proto version as case 1
proto=my_private_protocol, proto_commit=my_private_protocol.repo.latest_commit,
model=model, model_commit=model_first_commit)
public_protocol = _add_experiment(other_user, 'public') # Re-run case 2
_add_experiment(other_user, 'public', # Not re-run as other model
proto=public_protocol, proto_commit=public_protocol.repo.latest_commit,
model=other_model, model_commit=other_model_commit)
_add_experiment(other_user, 'private') # Not re-run as can't see protocol
visible_protocol = _add_experiment(other_user, 'private', shared=True) # Re-run case 3
p1 = _add_experiment(logged_in_user, 'private', model=m1, model_commit=m1v2)
p1v2 = helpers.add_version(p1, visibility='private')
_add_experiment(logged_in_user, 'private', model=m1, model_commit=m1v1, proto=p1, proto_commit=p1v2)
_add_experiment(logged_in_user, 'private', model=m2, model_commit=m2v1, proto=p1, proto_commit=p1v2)
_add_experiment(logged_in_user, 'public', model=m2, model_commit=m2v1)
p3 = _add_experiment(other_user, 'public', model=m1, model_commit=m1v2)
p3v1 = p3.repocache.latest_version
p3v2 = helpers.add_version(p3, visibility='private')
_add_experiment(logged_in_user, 'private', model=m1, model_commit=m1v2, proto=p3, proto_commit=p3v2)
p4 = _add_experiment(other_user, 'public', model=m1, model_commit=m1v2, shared=True)
p4v2 = helpers.add_version(p4, visibility='private')
_add_experiment(other_user, 'private', model=m1, model_commit=m1v2)

# Create a new version of our model, re-running experiments
helpers.add_permission(logged_in_user, 'create_model')
if route == 'main_form':
recipes.model_file.make(
entity=model,
entity=m1,
upload=SimpleUploadedFile('file2.txt', b'file 2'),
original_name='file2.txt',
)
response = client.post(
'/entities/models/%d/versions/new' % model.pk,
'/entities/models/%d/versions/new' % m1.pk,
data={
'filename[]': ['uploads/file2.txt'],
'mainEntry': ['file2.txt'],
Expand All @@ -1403,12 +1404,12 @@ def _add_experiment(proto_author, proto_vis, shared=False,
},
)
assert response.status_code == 302
new_commit = model.repo.latest_commit
assert response.url == '/entities/models/%d/versions/%s' % (model.id, new_commit.sha)
new_commit = m1.repo.latest_commit
assert response.url == '/entities/models/%d/versions/%s' % (m1.id, new_commit.sha)
else:
assert route == 'edit_file'
response = client.post('/entities/models/%d/versions/edit' % model.id, json.dumps({
'parent_hexsha': model_commit.sha,
response = client.post('/entities/models/%d/versions/edit' % m1.id, json.dumps({
'parent_hexsha': m1v2.sha,
'file_name': 'file1.txt',
'file_contents': 'new file 1',
'visibility': 'private',
Expand All @@ -1420,20 +1421,20 @@ def _add_experiment(proto_author, proto_vis, shared=False,
detail = json.loads(response.content.decode())
assert 'updateEntityFile' in detail
assert detail['updateEntityFile']['response']
new_commit = model.repo.latest_commit
new_commit = m1.repo.latest_commit
assert detail['updateEntityFile']['url'] == '/entities/models/%d/versions/%s' % (
model.id, new_commit.sha)
m1.id, new_commit.sha)

# Test that planned experiments have been added correctly
expected_proto_versions = set([
(my_private_protocol, my_private_protocol.repo.latest_commit.sha),
(public_protocol, public_protocol.repo.latest_commit.sha),
(visible_protocol, visible_protocol.repo.latest_commit.sha),
(p1, p1v2.sha),
(p3, p3v1.sha),
(p4, p4v2.sha),
])
assert len(expected_proto_versions) == PlannedExperiment.objects.count()
for planned_experiment in PlannedExperiment.objects.all():
assert planned_experiment.submitter == logged_in_user
assert planned_experiment.model == model
assert planned_experiment.model == m1
assert planned_experiment.model_version == new_commit.sha
assert (planned_experiment.protocol, planned_experiment.protocol_version) in expected_proto_versions

Expand Down