WIP Downloading submissions v1

src/apps/api/views/submissions.py

@@ -15,6 +15,7 @@
 from rest_framework.viewsets import ModelViewSet
 from rest_framework_csv import renderers
 from django.core.files.base import ContentFile
+from django.http import StreamingHttpResponse
 
 from profiles.models import Organization, Membership
 from tasks.models import Task
@@ -23,6 +24,7 @@
 from leaderboards.strategies import put_on_leaderboard_by_submission_rule
 from leaderboards.models import SubmissionScore, Column, Leaderboard
 
+
 logger = logging.getLogger()
 
 
@@ -310,6 +312,27 @@ def re_run_many_submissions(self, request):
             submission.re_run()
         return Response({})
 
+
+    # # New methods impleted !! 
+    @action(detail=False, methods=['get'])
+    def download_many(self, request):
+        pks = request.query_params.get('pks')
+        if pks:
+            pks = json.loads(pks)  # Convert JSON string to list
+
+        # Doing a local import here to avoid circular imports
+        from competitions.tasks import stream_batch_download
+
+        # Call the task and get the result (stream)
+        # in_memory_zip = stream_batch_download.apply_async((pks,)).get()
+        in_memory_zip = stream_batch_download(pks)
+
+        # Stream the response
+        response = StreamingHttpResponse(in_memory_zip, content_type='application/zip')
+        response['Content-Disposition'] = 'attachment; filename="bulk_submissions.zip"'
+
+        return response
+
     @action(detail=True, methods=('GET',))
     def get_details(self, request, pk):
         submission = super().get_object()

src/apps/competitions/tasks.py

@@ -20,6 +20,10 @@
 from django.utils.timezone import now
 from rest_framework.exceptions import ValidationError
 
+from urllib.request import urlopen
+from contextlib import closing
+from urllib.error import ContentTooShortError
+
 from celery_config import app
 from competitions.models import Submission, CompetitionCreationTaskStatus, SubmissionDetails, Competition, \
     CompetitionDump, Phase
@@ -263,6 +267,55 @@ def send_child_id(submission, child_id):
     })
 
 
+def retrieve_data(url,data=None):
+    with closing(urlopen(url, data)) as fp:
+        headers = fp.info()
+
+        bs = 1024*8
+        size = -1
+        read = 0
+        blocknum = 0
+        if "content-length" in headers:
+            size = int(headers["Content-Length"])
+
+        while True:
+            block = fp.read(bs)
+            if not block:
+                break
+            read += len(block)
+            yield(block)
+
+    if size >= 0 and read < size:
+        raise ContentTooShortError(
+            "retrieval incomplete: got only %i out of %i bytes"
+            % (read, size))
+
+
+def zip_generator(submission_pks):
+    in_memory_zip = BytesIO()
+    # logger.info("IN zip generator")
+    with zipfile.ZipFile(in_memory_zip, 'w', zipfile.ZIP_DEFLATED) as zip_file:
+        for submission_id in submission_pks:
+            submission = Submission.objects.get(id=submission_id)
+            # logger.info(submission.data.data_file)
+
+            short_name = submission.data.data_file.name.split('/')[-1]
+            url = make_url_sassy(path=submission.data.data_file.name)
+            for block in retrieve_data(url):
+                zip_file.writestr(short_name, block)
+
+    in_memory_zip.seek(0)
+
+    return in_memory_zip
+
+
+@app.task(queue='site-worker', soft_time_limit=60*60)
+def stream_batch_download(submission_pks):
+    # logger.info("In stream_batch_download")
+    # logger.info(submission_pks)
+    return zip_generator(submission_pks)
+
+
 @app.task(queue='site-worker', soft_time_limit=60)
 def _run_submission(submission_pk, task_pks=None, is_scoring=False):
     """This function is wrapped so that when we run tests we can run this function not

src/static/js/ours/client.js

@@ -124,6 +124,31 @@ CODALAB.api = {
     get_submission_detail_result: function (id) {
         return CODALAB.api.request('GET', `${URLS.API}submissions/${id}/get_detail_result/`)
     },
+    download_many_submissions: function (pks) {
+        console.log('Request bulk');
+        const params = new URLSearchParams({ pks: JSON.stringify(pks) });
+        const url = `${URLS.API}submissions/download_many/?${params}`;
+        return fetch(url, {
+            method: 'GET',
+            headers: {
+                'Content-Type': 'application/json'
+            }
+        }).then(response => {
+            if (!response.ok) {
+                throw new Error('Network response was not ok ' + response.statusText);
+            }
+            return response.blob();
+        }).then(blob => {
+            const link = document.createElement('a');
+            link.href = window.URL.createObjectURL(blob);
+            link.download = 'bulk_submissions.zip';
+            document.body.appendChild(link);
+            link.click();
+            document.body.removeChild(link);
+        }).catch(error => {
+            console.error('Error downloading submissions:', error);
+        });
+    },
 
     /*---------------------------------------------------------------------
          Leaderboards