diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index fccaeefd4..bd6667385 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -15,6 +15,7 @@ from rest_framework.viewsets import ModelViewSet from rest_framework_csv import renderers from django.core.files.base import ContentFile +from django.http import StreamingHttpResponse from profiles.models import Organization, Membership from tasks.models import Task @@ -23,6 +24,7 @@ from leaderboards.strategies import put_on_leaderboard_by_submission_rule from leaderboards.models import SubmissionScore, Column, Leaderboard + logger = logging.getLogger() @@ -310,6 +312,27 @@ def re_run_many_submissions(self, request): submission.re_run() return Response({}) + + # # New methods impleted !! + @action(detail=False, methods=['get']) + def download_many(self, request): + pks = request.query_params.get('pks') + if pks: + pks = json.loads(pks) # Convert JSON string to list + + # Doing a local import here to avoid circular imports + from competitions.tasks import stream_batch_download + + # Call the task and get the result (stream) + # in_memory_zip = stream_batch_download.apply_async((pks,)).get() + in_memory_zip = stream_batch_download(pks) + + # Stream the response + response = StreamingHttpResponse(in_memory_zip, content_type='application/zip') + response['Content-Disposition'] = 'attachment; filename="bulk_submissions.zip"' + + return response + @action(detail=True, methods=('GET',)) def get_details(self, request, pk): submission = super().get_object() diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 8a95a9e20..a2373db7d 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -20,6 +20,10 @@ from django.utils.timezone import now from rest_framework.exceptions import ValidationError +from urllib.request import urlopen +from contextlib import closing +from urllib.error import ContentTooShortError + from celery_config import app from competitions.models import Submission, CompetitionCreationTaskStatus, SubmissionDetails, Competition, \ CompetitionDump, Phase @@ -263,6 +267,55 @@ def send_child_id(submission, child_id): }) +def retrieve_data(url,data=None): + with closing(urlopen(url, data)) as fp: + headers = fp.info() + + bs = 1024*8 + size = -1 + read = 0 + blocknum = 0 + if "content-length" in headers: + size = int(headers["Content-Length"]) + + while True: + block = fp.read(bs) + if not block: + break + read += len(block) + yield(block) + + if size >= 0 and read < size: + raise ContentTooShortError( + "retrieval incomplete: got only %i out of %i bytes" + % (read, size)) + + +def zip_generator(submission_pks): + in_memory_zip = BytesIO() + # logger.info("IN zip generator") + with zipfile.ZipFile(in_memory_zip, 'w', zipfile.ZIP_DEFLATED) as zip_file: + for submission_id in submission_pks: + submission = Submission.objects.get(id=submission_id) + # logger.info(submission.data.data_file) + + short_name = submission.data.data_file.name.split('/')[-1] + url = make_url_sassy(path=submission.data.data_file.name) + for block in retrieve_data(url): + zip_file.writestr(short_name, block) + + in_memory_zip.seek(0) + + return in_memory_zip + + +@app.task(queue='site-worker', soft_time_limit=60*60) +def stream_batch_download(submission_pks): + # logger.info("In stream_batch_download") + # logger.info(submission_pks) + return zip_generator(submission_pks) + + @app.task(queue='site-worker', soft_time_limit=60) def _run_submission(submission_pk, task_pks=None, is_scoring=False): """This function is wrapped so that when we run tests we can run this function not diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index 9a8801a6e..19a0f9949 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -124,6 +124,31 @@ CODALAB.api = { get_submission_detail_result: function (id) { return CODALAB.api.request('GET', `${URLS.API}submissions/${id}/get_detail_result/`) }, + download_many_submissions: function (pks) { + console.log('Request bulk'); + const params = new URLSearchParams({ pks: JSON.stringify(pks) }); + const url = `${URLS.API}submissions/download_many/?${params}`; + return fetch(url, { + method: 'GET', + headers: { + 'Content-Type': 'application/json' + } + }).then(response => { + if (!response.ok) { + throw new Error('Network response was not ok ' + response.statusText); + } + return response.blob(); + }).then(blob => { + const link = document.createElement('a'); + link.href = window.URL.createObjectURL(blob); + link.download = 'bulk_submissions.zip'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + }).catch(error => { + console.error('Error downloading submissions:', error); + }); + }, /*--------------------------------------------------------------------- Leaderboards diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index f99b70046..0f76b320b 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -5,10 +5,8 @@
Rerun all submissions per phase
Select a phase
-
+
{ phase.name }
@@ -16,15 +14,15 @@ Download as CSV - -
@@ -50,115 +48,117 @@ - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + +
-
- - -
- 		ID #File nameOwnerPhaseDateStatusScoreDetailed ResultsActions
+
+ + +
+ 		ID #File nameOwnerPhaseDateStatusScore + Detailed ResultsActions
No submissions found! Please make a submission
- Loading Submissions... -
-
- - -
- 		{ submission.id }{ submission.filename }{ submission.owner }{ submission.phase.name }{ submission.created_when} - { submission.status } - - - - - - - {get_score(submission)} - - - - - - - - - - - - - - - - - - + No submissions found! Please make a submission
+ Loading Submissions... +
+
+ + +
+ 		{ submission.id }{ submission.filename }{ submission.owner }{ submission.phase.name }{ submission.created_when} + { submission.status } + + + + + + + {get_score(submission)} + + + + + + + + + + + + + + + + + + - - - - + + + - - - - - - - - - - - - - - -
@@ -166,14 +166,13 @@
+ show_visualization="{opts.competition.enable_detailed_results}" + submission="{selected_submission}">
-
+
Task {i + 1}
@@ -181,16 +180,16 @@
- ERROR: Submission is a parent, but has no children. There was an error during creation. + ERROR: Submission is a parent, but has no children. There was an error + during creation.
-
+
+ show_visualization="{opts.competition.enable_detailed_results}" + submission="{child}">
@@ -214,6 +213,7 @@ $(self.refs.status).dropdown() $(self.refs.phase).dropdown() $(self.refs.rerun_button).dropdown() + $(self.refs.submission_handling_operation).dropdown() $(self.refs.submission_table).tablesort() }) @@ -233,11 +233,11 @@ self.loading = true self.update() if (opts.admin) { - filters = filters || {phase__competition: opts.competition.id} + filters = filters || { phase__competition: opts.competition.id } } else { - filters = filters || {phase: self.selected_phase.id} + filters = filters || { phase: self.selected_phase.id } } - filters = filters || {phase: self.selected_phase.id} + filters = filters || { phase: self.selected_phase.id } CODALAB.api.get_submissions(filters) .done(function (submissions) { // TODO: should be able to do this with a serializer? @@ -292,7 +292,7 @@ event.stopPropagation() } self.rerun_phase = function (phase) { - if(confirm("Are you sure? This could take hours .. you are re-running all of the submissions in a phase.")) { + if (confirm("Are you sure? This could take hours .. you are re-running all of the submissions in a phase.")) { CODALAB.api.re_run_phase_submissions(phase.id) .done(function (response) { toastr.success(`Rerunning ${response.count} submissions`) @@ -335,14 +335,14 @@ self.update_submissions() }) .fail(function (response) { - if(response.responseJSON.detail){ + if (response.responseJSON.detail) { toastr.error(response.responseJSON.detail) } else { toastr.error(response.responseText) } }) event.stopPropagation() - + } self.rerun_submission = function (submission) { @@ -352,10 +352,10 @@ self.update_submissions() }) .fail(function (response) { - if(response.responseJSON.detail){ + if (response.responseJSON.detail) { toastr.error(response.responseJSON.detail) - } - else if(response.responseJSON.error_msg){ + } + else if (response.responseJSON.error_msg) { toastr.error(response.responseJSON.error_msg) } else { @@ -404,11 +404,11 @@ toastr.success('Submissions deleted') self.update_submissions() }) - .fail(function (response){ + .fail(function (response) { toastr.error('Something went wrong') }) } - } + } self.get_score_details = function (submission, column) { try { @@ -422,9 +422,9 @@ } self.get_score = function (submission) { - try{ + try { return parseFloat(submission.scores[0].score).toFixed(2) - + } catch { return "" } @@ -447,7 +447,7 @@ } } - self.on_submission_checked = function(event){ + self.on_submission_checked = function (event) { event.stopPropagation() self.submission_checked() } @@ -457,23 +457,30 @@ let checked_boxes = inputs.not(':first').filter('input:checked') let unchecked_boxes = inputs.not(':first').filter('input:not(:checked)') inputs.first().prop('checked', unchecked_boxes.length === 0) - self.checked_submissions = checked_boxes.serializeArray().map((x) => {return x.name}) + self.checked_submissions = checked_boxes.serializeArray().map((x) => { return x.name }) } self.select_all_pressed = function () { let check_boxes = $(self.refs.submission_table).find('input') // Set checkboxes to be equal to Select_All checkbox check_boxes.prop('checked', check_boxes.first().is(':checked')) + + + let inputs = $(self.refs.submission_table).find('input') + let checked_boxes = inputs.not(':first').filter('input:checked') + self.checked_submissions = checked_boxes.serializeArray().map((x) => { return x.name }) } + + self.submission_clicked = function (submission) { // stupid workaround to not modify the original submission object submission = _.defaultsDeep({}, submission) if (submission.has_children) { submission.children = _.map(_.sortBy(submission.children), child => { - return {id: child} + return { id: child } }) CODALAB.api.get_submission_details(submission.id) .done(function (data) { @@ -509,6 +516,41 @@ CODALAB.events.trigger('submission_clicked') } + + self.bulk_download = function () { + CODALAB.api.download_many_submissions(self.checked_submissions) + .catch(function (error) { + console.error('Error:', error); + }); + } + + + + self.submission_handling = function () { + // console.log(self.checked_submissions) + if (self.checked_submissions.length === 0) { + console.log("no submission is selected"); + } else { + var submission_operation = self.refs.submission_handling_operation.value + switch (submission_operation) { + case "delete": + // console.log("delete") + self.delete_selected_submissions() + break; + case "download": + // console.log("download") + self.bulk_download() + break; + case "rerun": + // console.log("rerun") + self.rerun_selected_submissions() + break + default: + console.log("should never be in this state of default..") + } + } + } + CODALAB.events.on('phase_selected', function (selected_phase) { self.selected_phase = selected_phase self.selected_phase.hide_output = selected_phase.hide_output && !CODALAB.state.user.has_competition_admin_privileges(self.opts.competition) @@ -565,4 +607,4 @@ .failed.question.circle.icon color #2c3f4c - + \ No newline at end of file