diff --git a/.env_sample b/.env_sample index b39a83de8..51c6b60b8 100644 --- a/.env_sample +++ b/.env_sample @@ -41,6 +41,8 @@ SELENIUM_HOSTNAME=selenium #EMAIL_HOST_PASSWORD=pass #EMAIL_PORT=587 #EMAIL_USE_TLS=True +#DEFAULT_FROM_EMAIL="Codabench " +#SERVER_EMAIL=noreply@example.com # ----------------------------------------------------------------------------- # Storage diff --git a/src/apps/api/serializers/competitions.py b/src/apps/api/serializers/competitions.py index e3bc20fb7..9b622b3dc 100644 --- a/src/apps/api/serializers/competitions.py +++ b/src/apps/api/serializers/competitions.py @@ -24,6 +24,7 @@ class PhaseSerializer(WritableNestedModelSerializer): tasks = serializers.SlugRelatedField(queryset=Task.objects.all(), required=True, allow_null=False, slug_field='key', many=True) status = serializers.SerializerMethodField() + is_final_phase = serializers.SerializerMethodField() class Meta: model = Phase @@ -48,6 +49,14 @@ class Meta: 'is_final_phase', ) + def get_is_final_phase(self, obj): + if len(obj.competition.phases.all()) > 1: + return obj.is_final_phase + elif len(obj.competition.phases.all()) == 1: + obj.is_final_phase = True + obj.save() + return obj.is_final_phase + def get_status(self, obj): now = datetime.now().replace(tzinfo=None) @@ -245,6 +254,7 @@ class Meta: 'registration_auto_approve', 'queue', 'enable_detailed_results', + 'auto_run_submissions', 'make_programs_available', 'make_input_data_available', 'docker_image', @@ -327,6 +337,7 @@ class CompetitionCreateSerializer(CompetitionSerializer): class CompetitionDetailSerializer(serializers.ModelSerializer): created_by = serializers.CharField(source='created_by.username', read_only=True) + owner_display_name = serializers.SerializerMethodField() logo_icon = NamedBase64ImageField(allow_null=True) pages = PageSerializer(many=True) phases = PhaseDetailSerializer(many=True) @@ -346,6 +357,7 @@ class Meta: 'published', 'secret_key', 'created_by', + 'owner_display_name', 'created_when', 'logo', 'logo_icon', @@ -361,6 +373,7 @@ class Meta: 'submission_count', 'queue', 'enable_detailed_results', + 'auto_run_submissions', 'make_programs_available', 'make_input_data_available', 'docker_image', @@ -371,7 +384,7 @@ class Meta: 'reward', 'contact_email', 'report', - 'whitelist_emails' + 'whitelist_emails', ) def get_leaderboards(self, instance): @@ -389,9 +402,14 @@ def get_whitelist_emails(self, instance): whitelist_emails_list = [entry.email for entry in whitelist_emails_query] return whitelist_emails_list + def get_owner_display_name(self, obj): + # Get the user's display name if not None, otherwise return username + return obj.created_by.display_name if obj.created_by.display_name else obj.created_by.username + class CompetitionSerializerSimple(serializers.ModelSerializer): - created_by = serializers.CharField(source='created_by.username') + created_by = serializers.CharField(source='created_by.username', read_only=True) + owner_display_name = serializers.SerializerMethodField() participant_count = serializers.IntegerField(read_only=True) class Meta: @@ -400,10 +418,12 @@ class Meta: 'id', 'title', 'created_by', + 'owner_display_name', 'created_when', 'published', 'participant_count', 'logo', + 'logo_icon', 'description', 'competition_type', 'reward', @@ -411,6 +431,14 @@ class Meta: 'report', ) + def get_created_by(self, obj): + # Get the user's display name if not None, otherwise return username + return obj.created_by.display_name if obj.created_by.display_name else obj.created_by.username + + def get_owner_display_name(self, obj): + # Get the user's display name if not None, otherwise return username + return obj.created_by.display_name if obj.created_by.display_name else obj.created_by.username + PageSerializer.competition = CompetitionSerializer(many=True, source='competition') diff --git a/src/apps/api/serializers/datasets.py b/src/apps/api/serializers/datasets.py index 25e069afc..7543afe54 100644 --- a/src/apps/api/serializers/datasets.py +++ b/src/apps/api/serializers/datasets.py @@ -74,7 +74,8 @@ class Meta: class DataDetailSerializer(serializers.ModelSerializer): - created_by = serializers.CharField(source='created_by.username') + created_by = serializers.CharField(source='created_by.username', read_only=True) + owner_display_name = serializers.SerializerMethodField() competition = serializers.SerializerMethodField() value = serializers.CharField(source='key', required=False) @@ -83,6 +84,7 @@ class Meta: fields = ( 'id', 'created_by', + 'owner_display_name', 'created_when', 'name', 'type', @@ -108,6 +110,9 @@ def get_competition(self, obj): } return None + def get_owner_display_name(self, instance): + return instance.created_by.display_name if instance.created_by.display_name else instance.created_by.username + class DataGroupSerializer(serializers.ModelSerializer): class Meta: diff --git a/src/apps/api/serializers/submissions.py b/src/apps/api/serializers/submissions.py index 81e8de21c..33fe9241e 100644 --- a/src/apps/api/serializers/submissions.py +++ b/src/apps/api/serializers/submissions.py @@ -41,6 +41,7 @@ class SubmissionSerializer(serializers.ModelSerializer): on_leaderboard = serializers.BooleanField(read_only=True) task = TaskSerializer() created_when = serializers.DateTimeField(format="%Y-%m-%d %H:%M") + auto_run = serializers.SerializerMethodField(read_only=True) class Meta: model = Submission @@ -66,6 +67,7 @@ class Meta: 'leaderboard', 'on_leaderboard', 'task', + 'auto_run' ) read_only_fields = ( 'pk', @@ -79,6 +81,10 @@ class Meta: def get_filename(self, instance): return basename(instance.data.data_file.name) + def get_auto_run(self, instance): + # returns this submission's competition auto_run_submissions Flag + return instance.phase.competition.auto_run_submissions + class SubmissionLeaderBoardSerializer(serializers.ModelSerializer): scores = SubmissionScoreSerializer(many=True) @@ -151,9 +157,12 @@ def get_filename(self, instance): def create(self, validated_data): tasks = validated_data.pop('tasks', None) - sub = super().create(validated_data) - sub.start(tasks=tasks) + + # Check if auto_run_submissions is enabled then run the submission + # Otherwise organizer will run manually + if sub.phase.competition.auto_run_submissions: + sub.start(tasks=tasks) return sub diff --git a/src/apps/api/serializers/tasks.py b/src/apps/api/serializers/tasks.py index 1526a1980..62080890d 100644 --- a/src/apps/api/serializers/tasks.py +++ b/src/apps/api/serializers/tasks.py @@ -90,7 +90,8 @@ def get_validated(self, instance): class TaskDetailSerializer(WritableNestedModelSerializer): - created_by = serializers.CharField(source='created_by.username', read_only=True, required=False) + created_by = serializers.CharField(source='created_by.username', read_only=True) + owner_display_name = serializers.SerializerMethodField() input_data = DataSimpleSerializer(read_only=True) ingestion_program = DataSimpleSerializer(read_only=True) reference_data = DataSimpleSerializer(read_only=True) @@ -107,6 +108,7 @@ class Meta: 'description', 'key', 'created_by', + 'owner_display_name', 'created_when', 'is_public', 'validated', @@ -126,12 +128,18 @@ def get_validated(self, task): def get_shared_with(self, instance): return self.context['shared_with'][instance.pk] + def get_owner_display_name(self, instance): + # Get the user's display name if not None, otherwise return username + return instance.created_by.display_name if instance.created_by.display_name else instance.created_by.username + class TaskListSerializer(serializers.ModelSerializer): solutions = SolutionListSerializer(many=True, required=False, read_only=True) value = serializers.CharField(source='key', required=False) competitions = serializers.SerializerMethodField() shared_with = serializers.SerializerMethodField() + created_by = serializers.CharField(source='created_by.username', read_only=True) + owner_display_name = serializers.SerializerMethodField() class Meta: model = Task @@ -139,6 +147,7 @@ class Meta: 'id', 'created_when', 'created_by', + 'owner_display_name', 'key', 'name', 'solutions', @@ -160,6 +169,10 @@ def get_competitions(self, instance): def get_shared_with(self, instance): return self.context['shared_with'][instance.pk] + def get_owner_display_name(self, instance): + # Get the user's display name if not None, otherwise return username + return instance.created_by.display_name if instance.created_by.display_name else instance.created_by.username + class PhaseTaskInstanceSerializer(serializers.HyperlinkedModelSerializer): task = serializers.SlugRelatedField(queryset=Task.objects.all(), required=True, allow_null=False, slug_field='key', diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index 3dc2f0bba..774fd0df6 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -18,8 +18,6 @@ from rest_framework.response import Response from rest_framework.renderers import JSONRenderer from rest_framework_csv.renderers import CSVRenderer -from rest_framework_extensions.cache.decorators import cache_response -from rest_framework_extensions.key_constructor.constructors import DefaultListKeyConstructor from api.pagination import LargePagination from api.renderers import ZipRenderer from rest_framework.viewsets import ModelViewSet @@ -533,7 +531,6 @@ def create_dump(self, request, pk=None): serializer = CompetitionCreationTaskStatusSerializer({"status": "Success. Competition dump is being created."}) return Response(serializer.data, status=201) - @cache_response(key_func=DefaultListKeyConstructor()) @action(detail=False, methods=('GET',), pagination_class=LargePagination) def public(self, request): qs = self.get_queryset() @@ -634,8 +631,8 @@ def rerun_submissions(self, request, pk): phase = self.get_object() comp = phase.competition - # Get submissions - submissions = phase.submissions.all() + # Get submissions with no parent + submissions = phase.submissions.filter(parent__isnull=True) can_re_run_submissions = False error_message = "" @@ -704,12 +701,18 @@ def get_leaderboard(self, request, pk): submission_detailed_results = {} for submission in query['submissions']: # count number of entries/number of submissions for the owner of this submission for this phase - # count all submissions with no parent and count all parents without counting the children + # count all submissions except: + # - child submissions (submissions who has a parent i.e. parent field is not null) + # - Failed submissions + # - Cancelled submissions num_entries = Submission.objects.filter( - Q(owner__username=submission['owner']) | Q(parent__owner__username=submission['owner']), + Q(owner__username=submission['owner']) | + Q(parent__owner__username=submission['owner']), phase=phase, ).exclude( - parent__isnull=False + Q(status=Submission.FAILED) | + Q(status=Submission.CANCELLED) | + Q(parent__isnull=False) ).count() submission_key = f"{submission['owner']}{submission['parent'] or submission['id']}" diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index f9582488d..d0c0defb7 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -6,7 +6,6 @@ from django_filters.rest_framework import DjangoFilterBackend from rest_framework import status from rest_framework.decorators import api_view, permission_classes, action -from django.http import Http404 from rest_framework.exceptions import PermissionDenied, ValidationError from rest_framework.filters import SearchFilter from rest_framework.generics import get_object_or_404 @@ -93,7 +92,7 @@ def check_object_permissions(self, request, obj): not_bot_user = self.request.user.is_authenticated and not self.request.user.is_bot - if self.action in ['update_fact_sheet', 're_run_submission']: + if self.action in ['update_fact_sheet', 'run_submission', 're_run_submission']: # get_queryset will stop us from re-running something we're not supposed to pass elif not self.request.user.is_authenticated or not_bot_user: @@ -206,14 +205,24 @@ def has_admin_permission(self, user, submission): @action(detail=True, methods=('POST', 'DELETE')) def submission_leaderboard_connection(self, request, pk): + + # get submission submission = self.get_object() + + # get submission phase phase = submission.phase - if not (request.user.is_superuser or request.user == submission.owner): - if not phase.competition.collaborators.filter(pk=request.user.pk).exists(): - raise Http404 + # only super user, owner of submission and competition organizer can proceed + if not ( + request.user.is_superuser or + request.user == submission.owner or + request.user in phase.competition.all_organizers + ): + raise PermissionDenied("You cannot perform this action, contact the competition organizer!") + + # only super user and with these leaderboard rules (FORCE_LAST, FORCE_BEST, FORCE_LATEST_MULTIPLE) can proceed if submission.phase.leaderboard.submission_rule in Leaderboard.AUTO_SUBMISSION_RULES and not request.user.is_superuser: - raise ValidationError("Users are not allowed to edit the leaderboard on this Competition") + raise PermissionDenied("Users are not allowed to edit the leaderboard on this Competition") if request.method == 'POST': # Removing any existing submissions on leaderboard unless multiples are allowed @@ -228,7 +237,7 @@ def submission_leaderboard_connection(self, request, pk): if request.method == 'DELETE': if submission.phase.leaderboard.submission_rule not in [Leaderboard.ADD_DELETE, Leaderboard.ADD_DELETE_MULTIPLE]: - raise ValidationError("You are not allowed to remove a submission on this phase") + raise PermissionDenied("You are not allowed to remove a submission on this phase") submission.leaderboard = None submission.save() Submission.objects.filter(parent=submission).update(leaderboard=None) @@ -246,6 +255,21 @@ def cancel_submission(self, request, pk): canceled = submission.cancel() return Response({'canceled': canceled}) + @action(detail=True, methods=('POST',)) + def run_submission(self, request, pk): + submission = self.get_object() + + # Only organizer of the competition can run the submission + if not self.has_admin_permission(request.user, submission): + raise PermissionDenied('You do not have permission to run this submission') + + # Allow only to run a submission with status `Submitting` + if submission.status != Submission.SUBMITTING: + raise PermissionDenied('Cannot run a submission which is not in submitting status') + + new_sub = submission.run() + return Response({'id': new_sub.id}) + @action(detail=True, methods=('POST',)) def re_run_submission(self, request, pk): submission = self.get_object() diff --git a/src/apps/competitions/migrations/0045_competition_auto_run_submissions.py b/src/apps/competitions/migrations/0045_competition_auto_run_submissions.py new file mode 100644 index 000000000..86161e98c --- /dev/null +++ b/src/apps/competitions/migrations/0045_competition_auto_run_submissions.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.17 on 2024-01-22 10:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('competitions', '0044_merge_20231221_1416'), + ] + + operations = [ + migrations.AddField( + model_name='competition', + name='auto_run_submissions', + field=models.BooleanField(default=True), + ), + ] diff --git a/src/apps/competitions/migrations/0046_merge_20240222_1916.py b/src/apps/competitions/migrations/0046_merge_20240222_1916.py new file mode 100644 index 000000000..347e64bc9 --- /dev/null +++ b/src/apps/competitions/migrations/0046_merge_20240222_1916.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.17 on 2024-02-22 19:16 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('competitions', '0045_competition_auto_run_submissions'), + ('competitions', '0045_auto_20240129_2314'), + ] + + operations = [ + ] diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index 1f96cc84e..03d8b65fc 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -65,6 +65,10 @@ class Competition(ChaHubSaveMixin, models.Model): reward = models.CharField(max_length=256, null=True, blank=True) report = models.CharField(max_length=256, null=True, blank=True) + # if true, submissions are auto-run when submitted + # if false, submissions run will be intiiated by organizer + auto_run_submissions = models.BooleanField(default=True) + def __str__(self): return f"competition-{self.title}-{self.pk}-{self.competition_type}" @@ -579,6 +583,13 @@ def start(self, tasks=None): from .tasks import run_submission run_submission(self.pk, tasks=tasks) + def run(self): + # get tasks from the phase + tasks = self.phase.tasks.all() + # start submission providing the tasks + self.start(tasks=tasks) + return self + def re_run(self, task=None): # task to use in the new submission diff --git a/src/apps/competitions/tests/test_submissions.py b/src/apps/competitions/tests/test_submissions.py index 4e58ebd93..a7ae024f2 100644 --- a/src/apps/competitions/tests/test_submissions.py +++ b/src/apps/competitions/tests/test_submissions.py @@ -155,7 +155,21 @@ def test_only_owner_can_add_submission_to_leaderboard(self): self.client.force_login(different_user) url = reverse('submission-submission-leaderboard-connection', kwargs={'pk': parent_sub.pk}) resp = self.client.post(url) - assert resp.status_code == 404 + assert resp.status_code == 403 + assert resp.data["detail"] == "You cannot perform this action, contact the competition organizer!" + + def test_only_owner_can_remove_submission_from_leaderboard(self): + parent_sub = SubmissionFactory(has_children=True) + leaderboard = LeaderboardFactory() + parent_sub.phase.leaderboard = leaderboard + parent_sub.phase.save() + + different_user = UserFactory() + self.client.force_login(different_user) + url = reverse('submission-submission-leaderboard-connection', kwargs={'pk': parent_sub.pk}) + resp = self.client.delete(url) + assert resp.status_code == 403 + assert resp.data["detail"] == "You cannot perform this action, contact the competition organizer!" def test_adding_submission_removes_other_submissions_from_owner(self): leaderboard = LeaderboardFactory() diff --git a/src/apps/competitions/unpackers/v1.py b/src/apps/competitions/unpackers/v1.py index 6802002ac..91496ddca 100644 --- a/src/apps/competitions/unpackers/v1.py +++ b/src/apps/competitions/unpackers/v1.py @@ -23,6 +23,7 @@ def __init__(self, *args, **kwargs): "description": self.competition_yaml.get("description", ""), "docker_image": docker_image, "enable_detailed_results": self.competition_yaml.get('enable_detailed_results', False), + "auto_run_submissions": self.competition_yaml.get('auto_run_submissions', True), "make_programs_available": self.competition_yaml.get('make_programs_available', False), "make_input_data_available": self.competition_yaml.get('make_input_data_available', False), "end_date": self.competition_yaml.get('end_date', None), diff --git a/src/apps/competitions/unpackers/v2.py b/src/apps/competitions/unpackers/v2.py index b3c87b2f4..8de2a42ad 100644 --- a/src/apps/competitions/unpackers/v2.py +++ b/src/apps/competitions/unpackers/v2.py @@ -14,6 +14,7 @@ def __init__(self, *args, **kwargs): "registration_auto_approve": self.competition_yaml.get('registration_auto_approve', False), "docker_image": self.competition_yaml.get('docker_image', 'codalab/codalab-legacy:py37'), "enable_detailed_results": self.competition_yaml.get('enable_detailed_results', False), + "auto_run_submissions": self.competition_yaml.get('auto_run_submissions', True), "make_programs_available": self.competition_yaml.get('make_programs_available', False), "make_input_data_available": self.competition_yaml.get('make_input_data_available', False), "description": self.competition_yaml.get("description", ""), diff --git a/src/apps/pages/views.py b/src/apps/pages/views.py index c3ade0ae4..fad6d33cd 100644 --- a/src/apps/pages/views.py +++ b/src/apps/pages/views.py @@ -100,6 +100,9 @@ def get_context_data(self, *args, **kwargs): queue_name = "*" if submission.queue is None else submission.queue.name submission.competition_queue = queue_name + # Add submission owner display name + submission.owner_display_name = submission.owner.display_name if submission.owner.display_name else submission.owner.username + return context def format_file_size(self, file_size): diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index dc789958e..9a8801a6e 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -102,6 +102,9 @@ CODALAB.api = { cancel_submission: function (id) { return CODALAB.api.request('GET', `${URLS.API}submissions/${id}/cancel_submission/`) }, + run_submission: function (id) { + return CODALAB.api.request('POST', `${URLS.API}submissions/${id}/run_submission/`) + }, re_run_submission: function (id) { return CODALAB.api.request('POST', `${URLS.API}submissions/${id}/re_run_submission/`) }, diff --git a/src/static/riot/competitions/detail/_header.tag b/src/static/riot/competitions/detail/_header.tag index d6cda97d4..9d826abb8 100644 --- a/src/static/riot/competitions/detail/_header.tag +++ b/src/static/riot/competitions/detail/_header.tag @@ -41,7 +41,7 @@
Organized by: - {competition.created_by} + {competition.owner_display_name} ({competition.contact_email})
diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index e7a6f17ed..5cfa0c7f5 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -94,49 +94,57 @@ + + + {get_score(submission)} - - - + + + + + + - + - + - + + +
+
+
+ + +
+ + + + + +
@@ -213,6 +226,7 @@ self.data["description"] = self.markdown_editor.value() self.data["queue"] = self.refs.queue.value self.data["enable_detailed_results"] = self.refs.detailed_results.checked + self.data["auto_run_submissions"] = self.refs.auto_run_submissions.checked self.data["make_programs_available"] = self.refs.make_programs_available.checked self.data["make_input_data_available"] = self.refs.make_input_data_available.checked self.data["docker_image"] = $(self.refs.docker_image).val() @@ -346,6 +360,7 @@ .dropdown('set value', competition.queue.id) } self.refs.detailed_results.checked = competition.enable_detailed_results + self.refs.auto_run_submissions.checked = competition.auto_run_submissions self.refs.make_programs_available.checked = competition.make_programs_available self.refs.make_input_data_available.checked = competition.make_input_data_available $(self.refs.docker_image).val(competition.docker_image) diff --git a/src/static/riot/competitions/public-list.tag b/src/static/riot/competitions/public-list.tag index c9c8e4680..c2eef1f43 100644 --- a/src/static/riot/competitions/public-list.tag +++ b/src/static/riot/competitions/public-list.tag @@ -49,7 +49,6 @@