Merge develop into master

compute_worker/compute_worker.py

@@ -22,7 +22,7 @@
 import websockets
 import yaml
 from billiard.exceptions import SoftTimeLimitExceeded
-from celery import Celery, task
+from celery import Celery, task, utils
 from kombu import Queue, Exchange
 from urllib3 import Retry
 
@@ -309,6 +309,7 @@ def _update_submission(self, data):
     def _update_status(self, status, extra_information=None):
         if status not in AVAILABLE_STATUSES:
             raise SubmissionException(f"Status '{status}' is not in available statuses: {AVAILABLE_STATUSES}")
+
         data = {
             "status": status,
             "status_details": extra_information,
@@ -663,8 +664,9 @@ def prepare(self):
         self._get_container_image(self.container_image)
 
     def start(self):
+        hostname = utils.nodenames.gethostname()
         if not self.is_scoring:
-            self._update_status(STATUS_RUNNING)
+            self._update_status(STATUS_RUNNING, extra_information=f"hostname-{hostname}")
 
         program_dir = os.path.join(self.root_dir, "program")
         ingestion_program_dir = os.path.join(self.root_dir, "ingestion_program")

src/apps/api/views/competitions.py

@@ -35,6 +35,8 @@
 from leaderboards.models import Leaderboard
 from utils.data import make_url_sassy
 from api.permissions import IsOrganizerOrCollaborator
+import logging
+logger = logging.getLogger()
 
 
 class CompetitionViewSet(ModelViewSet):
@@ -56,7 +58,14 @@ def get_queryset(self):
             mine = self.request.query_params.get('mine', None)
 
             if mine:
-                qs = qs.filter(created_by=self.request.user)
+                # either competition is mine
+                # or
+                # I am one of the collaborator
+                qs = Competition.objects.filter(
+                    (Q(created_by=self.request.user)) |
+                    (Q(collaborators__in=[self.request.user]))
+
+                )
 
             participating_in = self.request.query_params.get('participating_in', None)
 
@@ -69,19 +78,25 @@ def get_queryset(self):
             ).values_list('status')[:1]
             qs = qs.annotate(participant_status=Subquery(participant_status_query))
 
-            # new condition for search bar
             # `mine` is true when this is called from "Benchmarks I'm Running"
             # `participating_in` is true when this is called from "Benchmarks I'm in"
-            # `mine` and `participating_in` are none when this is called from Search bar
+            # `mine` and `participating_in` are none when this is called either from Search bar
+            # or from competition detail page
             if (not mine) and (not participating_in):
-                # User is logged in
-                # filter his own competitions
+                # User is logged in then filter
+                # competitions which this user owns
+                # or
+                # competitions in which this user is collaborator
                 # or
-                # filter published competitions by other users
+                # competitions is published and belongs to someone else
+                # or
+                # competitions in which this user is participant and status is approved
                 qs = qs.filter(
                     (Q(created_by=self.request.user)) |
-                    (Q(published=True) & ~Q(created_by=self.request.user))
-                )
+                    (Q(collaborators__in=[self.request.user])) |
+                    (Q(published=True) & ~Q(created_by=self.request.user)) |
+                    (Q(participants__user=self.request.user) & Q(participants__status="approved"))
+                ).distinct()
         else:
             # if user is not authenticated only filter published/public competitions
             qs = qs.filter(Q(published=True))
@@ -111,6 +126,7 @@ def get_queryset(self):
                 )
 
         search_query = self.request.query_params.get('search')
+        # search_query is true when called from searchbar
         if search_query:
             qs = qs.filter(Q(title__icontains=search_query) | Q(description__icontains=search_query))
 
@@ -375,7 +391,11 @@ def create_dump(self, request, pk=None):
         competition = self.get_object()
         if not competition.user_has_admin_permission(request.user):
             raise PermissionDenied("You don't have access")
-        create_competition_dump.delay(pk)
+
+        # arg 1: pk: competition primary key
+        # arg 2: False: keys_instead_of_files (if false: files will be dowloaded in dumps, if true: only keys)
+        create_competition_dump.delay(pk, False)
+
         serializer = CompetitionCreationTaskStatusSerializer({"status": "Success. Competition dump is being created."})
         return Response(serializer.data, status=201)
 

src/apps/api/views/submissions.py

@@ -37,6 +37,13 @@ def check_object_permissions(self, request, obj):
                 raise PermissionDenied("Cannot add task-specific submission re-runs to leaderboards.")
             return
         if self.request and self.request.method in ('POST', 'PUT', 'PATCH'):
+            dir(self.request)
+            # Set hostname of submission
+            if "status_details" in self.request.data.keys():
+                if request.data['status_details'].find('hostname') != -1:
+                    hostname = request.data['status_details'].replace('hostname-', '')
+                    obj.worker_hostname = hostname
+                    obj.save()
             not_bot_user = self.request.user.is_authenticated and not self.request.user.is_bot
 
             if self.action in ['update_fact_sheet', 're_run_submission']:

src/apps/competitions/migrations/0032_submission_worker_hostname.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.17 on 2023-05-09 04:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('competitions', '0031_auto_20230504_1016'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='submission',
+            name='worker_hostname',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

src/apps/competitions/models.py

@@ -452,7 +452,7 @@ class Submission(ChaHubSaveMixin, models.Model):
     started_when = models.DateTimeField(null=True)
     is_public = models.BooleanField(default=False)
     is_specific_task_re_run = models.BooleanField(default=False)
-
+    worker_hostname = models.CharField(max_length=255, blank=True, null=True)
     is_migrated = models.BooleanField(default=False)
     created_by_migration = models.ForeignKey(Phase, related_name='migrated_submissions', on_delete=models.CASCADE,
                                              null=True,

src/apps/competitions/tasks.py

@@ -4,7 +4,8 @@
 import re
 import traceback
 import zipfile
-from datetime import timedelta
+from datetime import timedelta, datetime
+
 from io import BytesIO
 from tempfile import TemporaryDirectory, NamedTemporaryFile
 
@@ -423,15 +424,16 @@ def _get_error_string(error_dict):
 
 
 @app.task(queue='site-worker', soft_time_limit=60 * 10)
-def create_competition_dump(competition_pk, keys_instead_of_files=True):
+def create_competition_dump(competition_pk, keys_instead_of_files=False):
     yaml_data = {"version": "2"}
     try:
         # -------- SetUp -------
 
         logger.info(f"Finding competition {competition_pk}")
         comp = Competition.objects.get(pk=competition_pk)
         zip_buffer = BytesIO()
-        zip_name = f"{comp.title}-{comp.created_when.isoformat()}.zip"
+        current_date_time = datetime.today().strftime('%Y-%m-%d %H:%M:%S')
+        zip_name = f"{comp.title}-{current_date_time}.zip"
         zip_file = zipfile.ZipFile(zip_buffer, "w")
 
         # -------- Main Competition Details -------
@@ -626,7 +628,7 @@ def create_competition_dump(competition_pk, keys_instead_of_files=True):
         bundle_count = CompetitionDump.objects.count() + 1
         temp_dataset_bundle = Data.objects.create(
             created_by=comp.created_by,
-            name=f"{comp.title} Dump #{bundle_count} Created {comp.created_when.date()}",
+            name=f"{comp.title} Dump #{bundle_count} Created {current_date_time}",
             type='competition_bundle',
             description='Automatically created competition dump',
             # 'data_file'=,

src/apps/competitions/tests/unpacker_test_data.py

@@ -149,7 +149,7 @@
     "label": "RESULTS",
     "columns": [
         {
-            "title": "prediction_score",
+            "title": "Prediction score",
             "key": "prediction_score",
             "index": 0,
             "sorting": "desc",

src/apps/competitions/unpackers/base_unpacker.py

@@ -209,7 +209,16 @@ def _unpack_queue(self):
                     all_queue_organizer_names = queue.organizers.all().values_list('username', flat=True)
                     if queue.owner != self.creator and self.creator.username not in all_queue_organizer_names:
                         raise CompetitionUnpackingException("You do not have access to the specified queue!")
-                self.competition['queue'] = queue.id
+                self.competition['queue'] = {
+                    'name': queue.name,
+                    'vhost': queue.vhost,
+                    'is_public': queue.is_public,
+                    'owner': queue.owner,
+                    'organizers': queue.organizers,
+                    'broker_url': queue.broker_url,
+                    'created_when': queue.broker_url,
+                    'id': queue.id,
+                }
             except Queue.DoesNotExist:
                 raise CompetitionUnpackingException("The specified Queue does not exist!")
 

src/apps/competitions/unpackers/v1.py

@@ -158,7 +158,8 @@ def _unpack_leaderboards(self):
 
         for index, column in enumerate(columns):
             new_col_data = {
-                'title': column['title'],
+                # get label as title, if not found, use title by default
+                'title': column.get('label', column['title']),
                 'key': column['title'],
                 'index': index,
                 'sorting': column.get('sort') or 'desc',

src/apps/competitions/views.py

@@ -28,13 +28,16 @@ class CompetitionDetail(DetailView):
     def get_object(self, *args, **kwargs):
         competition = super().get_object(*args, **kwargs)
 
-        is_creator, is_collaborator, is_participant = False, False, False
+        is_admin, is_creator, is_collaborator, is_participant = False, False, False, False
 
         # check if user is loggedin
         if self.request.user.is_authenticated:
 
+            # check if user is admin
+            is_admin = self.request.user.is_superuser
+
             # check if user is the creator of this competition
-            is_creator = self.request.user.is_superuser or self.request.user == competition.created_by
+            is_creator = self.request.user == competition.created_by
 
             # check if user is collaborator of this competition
             is_collaborator = self.request.user in competition.collaborators.all()
@@ -46,7 +49,14 @@ def get_object(self, *args, **kwargs):
         # check if secret key provided is valid
         valid_secret_key = self.request.GET.get('secret_key') == str(competition.secret_key)
 
-        if is_creator or is_collaborator or competition.published or valid_secret_key or is_participant:
+        if (
+            is_admin or
+            is_creator or
+            is_collaborator or
+            competition.published or
+            valid_secret_key or
+            is_participant
+        ):
             return competition
         raise Http404()
 

src/templates/pages/server_status.html

@@ -12,6 +12,7 @@ <h1>Recent submissions (up to 250 or 2 days old)</h1>
             <th>Competition</th>
             <th>Submission PK</th>
             <th>Submitter</th>
+            <th>Hostname</th>
             <th>Submitted at</th>
             <th>Status</th>
             </thead>
@@ -26,6 +27,7 @@ <h1>Recent submissions (up to 250 or 2 days old)</h1>
                     <td>{{ submission.phase.competition }}</td>
                     <td>{{ submission.pk }}</td>
                     <td>{{ submission.owner.username }}</td>
+                    <td>{{ submission.worker_hostname }}</td>
                     <td>{{ submission.created_when|timesince }} ago</td>
                     <td>{{ submission.status }}</td>
                 </tr>