From e67088a47ea6dc13aaecce12dc12892fdd9aa53d Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Wed, 18 Jun 2025 11:43:48 +0200 Subject: [PATCH 01/37] Changed base image from python3.9 to Fedora 42, reducing size from 1.6GB to 1GB; Made Dockerfile.compute_worker_gpu base image the build CPU image, adding only the necessary things to build it faster --- Dockerfile.compute_worker | 15 +++++++++---- Dockerfile.compute_worker_gpu | 40 ++++++----------------------------- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/Dockerfile.compute_worker b/Dockerfile.compute_worker index 6e6b626bf..ca70672d3 100644 --- a/Dockerfile.compute_worker +++ b/Dockerfile.compute_worker @@ -1,13 +1,19 @@ -FROM --platform=linux/amd64 python:3.9 +FROM --platform=linux/amd64 fedora:42 # This makes output not buffer and return immediately, nice for seeing results in stdout ENV PYTHONUNBUFFERED 1 # Install Docker -RUN apt-get update && curl -fsSL https://get.docker.com | sh +RUN dnf -y install dnf-plugins-core && \ + dnf-3 config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo && \ + dnf -y update && \ + dnf install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin && \ + dnf install -y python3.9 && \ + dnf clean all && \ + rm -rf /var/cache /var/log/dnf* /var/log/yum.* -RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.8.3 +RUN curl -sSL https://install.python-poetry.org | python3.9 - --version 1.8.3 # Poetry location so future commands (below) work ENV PATH $PATH:/root/.local/bin # Want poetry to use system python of docker container @@ -15,7 +21,8 @@ RUN poetry config virtualenvs.create false RUN poetry config virtualenvs.in-project false COPY ./compute_worker/pyproject.toml ./ COPY ./compute_worker/poetry.lock ./ -RUN poetry install +# To use python3.9 instead of system python +RUN poetry config virtualenvs.prefer-active-python true && poetry install ADD compute_worker . diff --git a/Dockerfile.compute_worker_gpu b/Dockerfile.compute_worker_gpu index 6bf96f3c5..ba6d42680 100644 --- a/Dockerfile.compute_worker_gpu +++ b/Dockerfile.compute_worker_gpu @@ -1,38 +1,12 @@ -FROM --platform=linux/amd64 python:3.9 - -# This makes output not buffer and return immediately, nice for seeing results in stdout -ENV PYTHONUNBUFFERED 1 - -# Install Docker -RUN apt-get update && curl -fsSL https://get.docker.com | sh - - +FROM --platform=linux/amd64 codalab/competitions-v2-compute-worker:latest # Nvidia Container Toolkit for cuda use with docker # [source](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) -RUN curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \ - && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \ - sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \ - tee /etc/apt/sources.list.d/nvidia-container-toolkit.list -RUN apt-get update -y; -RUN apt-get install -y nvidia-container-toolkit +# Include deps +RUN dnf -y config-manager addrepo --from-repofile=https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo && \ + dnf -y update && \ + dnf -y install nvidia-container-runtime nvidia-container-toolkit --exclude container-selinux && \ + dnf clean all && \ + rm -rf /var/cache /var/log/dnf* /var/log/yum.* # Make it explicit that we're using GPUs # BB - not convinced we need this ENV USE_GPU 1 - -RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.8.3 -# Poetry location so future commands (below) work -ENV PATH $PATH:/root/.local/bin -# Want poetry to use system python of docker container -RUN poetry config virtualenvs.create false -RUN poetry config virtualenvs.in-project false -COPY ./compute_worker/pyproject.toml ./ -COPY ./compute_worker/poetry.lock ./ -RUN poetry install - -ADD compute_worker . - -CMD celery -A compute_worker worker \ - -l info \ - -Q compute-worker \ - -n compute-worker@%n \ - --concurrency=1 From 0ba2994061afbffc2ed5fa9c1f09190bf852a3dc Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Thu, 3 Jul 2025 17:14:51 +0500 Subject: [PATCH 02/37] option added to download all participants --- .../detail/participant_manager.tag | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/static/riot/competitions/detail/participant_manager.tag b/src/static/riot/competitions/detail/participant_manager.tag index b8ebeca13..5394c847c 100644 --- a/src/static/riot/competitions/detail/participant_manager.tag +++ b/src/static/riot/competitions/detail/participant_manager.tag @@ -16,7 +16,16 @@ -
Email all participants
+
+ +
+ Email all participants +
+ +
+ Download all participants +
+
@@ -204,5 +213,35 @@ $(self.refs.email_modal).modal('hide') } + // Download participants in csv file + self.download_participants_csv = () => { + if (!self.participants || self.participants.length === 0) { + toastr.warning('No participants to download') + return + } + + const headers = ['ID', 'Username', 'Email', 'Is Bot', 'Status']; + const rows = self.participants.map(p => [ + p.id, + p.username, + p.email, + p.is_bot ? 'Yes' : 'No', + p.status + ]); + + const csvContent = [headers, ...rows] + .map(e => e.map(v => `"${(v ?? '').toString().replace(/"/g, '""')}"`).join(',')) + .join('\n') + + const blob = new Blob([csvContent], { type: 'text/csv;charset=utf-8;' }) + const url = URL.createObjectURL(blob) + const link = document.createElement("a") + link.setAttribute("href", url) + link.setAttribute("download", "participants.csv") + document.body.appendChild(link) + link.click() + document.body.removeChild(link) + } + From a5d52300ad9e016b727f8655a08b52c0b1d69971 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Thu, 3 Jul 2025 18:42:16 +0500 Subject: [PATCH 03/37] code commented for clarity --- .../riot/competitions/detail/participant_manager.tag | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/static/riot/competitions/detail/participant_manager.tag b/src/static/riot/competitions/detail/participant_manager.tag index 5394c847c..f3f93b823 100644 --- a/src/static/riot/competitions/detail/participant_manager.tag +++ b/src/static/riot/competitions/detail/participant_manager.tag @@ -213,14 +213,17 @@ $(self.refs.email_modal).modal('hide') } - // Download participants in csv file + // Function to download participants in csv file self.download_participants_csv = () => { + // Show warning when there is no participant if (!self.participants || self.participants.length === 0) { toastr.warning('No participants to download') return } + // prepare csv header const headers = ['ID', 'Username', 'Email', 'Is Bot', 'Status']; + // prepare csv rows const rows = self.participants.map(p => [ p.id, p.username, @@ -229,10 +232,12 @@ p.status ]); + // prepare csv content using header and rows const csvContent = [headers, ...rows] .map(e => e.map(v => `"${(v ?? '').toString().replace(/"/g, '""')}"`).join(',')) .join('\n') + // Download prepared csv as `participants.csv` const blob = new Blob([csvContent], { type: 'text/csv;charset=utf-8;' }) const url = URL.createObjectURL(blob) const link = document.createElement("a") From e163cf5304fdde848c537a2d487985dd4282a914 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Fri, 4 Jul 2025 19:40:06 +0500 Subject: [PATCH 04/37] latex rendering problems fixed --- src/static/js/ours/latex_markdown_html.js | 120 +++++++++++++++++----- 1 file changed, 97 insertions(+), 23 deletions(-) diff --git a/src/static/js/ours/latex_markdown_html.js b/src/static/js/ours/latex_markdown_html.js index 6ee169f08..eba1b874d 100644 --- a/src/static/js/ours/latex_markdown_html.js +++ b/src/static/js/ours/latex_markdown_html.js @@ -1,29 +1,103 @@ -// Function to render Markdown, HTML and Latex and return updated content +// Function to render Markdown content that may include: +// - Code blocks (```...```) +// - Inline and block LaTeX ($...$ or $$...$$) +// - HTML +// The function returns an array of DOM nodes generated from the fully rendered and processed content. function renderMarkdownWithLatex(content) { - if(content === null){ - return [] - } - const parsedHtml = new DOMParser().parseFromString(marked(content), "text/html") - - const traverseAndRenderLatex = (node) => { - if (node.nodeType === Node.ELEMENT_NODE) { - const latexPattern = /\$\$([\s\S]*?)\$\$|\$([^\$\n]*?)\$/g - const hasLatex = latexPattern.test(node.textContent) - if (hasLatex) { - const tempDiv = document.createElement('div') - tempDiv.innerHTML = node.innerHTML.replace(latexPattern, (_, formula1, formula2) => { - const formula = formula1 || formula2 - const decodedFormula = formula.replace(/</g, '<').replace(/>/g, '>') - return katex.renderToString(decodedFormula, { throwOnError: false }) - }); - node.innerHTML = tempDiv.innerHTML - } + if (!content) return [] // Return empty if content is null or empty + + // --------------------------------------------------------- + // Step 1: Extract and temporarily replace all code blocks + // --------------------------------------------------------- + + // Regex to match code blocks in Markdown: ```[language]\n[code]``` + const codeBlockPattern = /```(\w*)\n([\s\S]*?)```/g + const codeBlocks = [] // Store original code blocks and their tokens + let codeIndex = 0 // Counter to generate unique tokens for code blocks + + // Replace each code block with a unique token and store the original + const contentWithoutCode = content.replace(codeBlockPattern, (_, lang, code) => { + const token = `%%CODE_BLOCK_${codeIndex++}%%` + codeBlocks.push({ token, lang, code }) // Store the token and the original code + return token // Replace the block with its token in the text + }) + + // --------------------------------------------------------- + // Step 2: Extract and replace LaTeX expressions with placeholders + // --------------------------------------------------------- + + // Regex to match inline ($...$) and block ($$...$$) LaTeX formulas + const latexPattern = /\$\$([\s\S]+?)\$\$|\$([^\$\n]+?)\$/g + const latexBlocks = [] // Store rendered LaTeX HTML and their tokens + let latexIndex = 0 // Counter for LaTeX token IDs + + // Replace LaTeX expressions with unique tokens and store rendered HTML + const contentWithLatexPlaceholders = contentWithoutCode.replace(latexPattern, (_, block, inline) => { + const formula = block || inline // Pick block or inline formula content + const displayMode = !!block // Use displayMode for block ($$...$$) + let rendered // Store the rendered HTML from KaTeX + + try { + // Render LaTeX to HTML using KaTeX + rendered = katex.renderToString(formula, { + throwOnError: false, + displayMode, + }) + } catch (e) { + console.error("KaTeX error:", e) + rendered = `${formula}` // If render fails, fallback to raw code } - node.childNodes.forEach(traverseAndRenderLatex) - }; + const token = `%%LATEX_BLOCK_${latexIndex++}%%` + latexBlocks.push({ token, rendered }) // Store the token and rendered HTML + return token // Replace formula with its token in the text + }) + + // --------------------------------------------------------- + // Step 3: Convert Markdown to HTML + // --------------------------------------------------------- + + // Run the Markdown parser on the content (now safe with all code and LaTeX replaced by tokens) + let html = marked(contentWithLatexPlaceholders) - traverseAndRenderLatex(parsedHtml.body) + // --------------------------------------------------------- + // Step 4: Restore rendered LaTeX blocks into the HTML + // --------------------------------------------------------- + // Replace each LaTeX token with the rendered KaTeX HTML + for (const { token, rendered } of latexBlocks) { + html = html.replace(token, rendered) + } + + // --------------------------------------------------------- + // Step 5: Restore escaped code blocks back into the HTML + // --------------------------------------------------------- + + // Replace each code block token with HTML-safe 
 block
+  for (const { token, code, lang } of codeBlocks) {
+    const safeCode = escapeHtml(code) // Escape HTML-sensitive characters inside code
+    html = html.replace(token, `
${safeCode}
`)
+  }
+
+
+  // ---------------------------------------------------------
+  // Step 6: Convert final HTML string into DOM nodes and return
+  // ---------------------------------------------------------
+
+  // Parse the final HTML string into actual DOM nodes
+  const parsedHtml = new DOMParser().parseFromString(html, "text/html")
+
+  // Return child nodes from the parsed HTML body
   return parsedHtml.body.childNodes
-}
\ No newline at end of file
+}
+
+// Utility function to escape HTML special characters inside code blocks
+function escapeHtml(text) {
+  return text
+    .replace(/&/g, "&")   // escape ampersands
+    .replace(//g, ">")    // escape >
+    .replace(/"/g, """)  // escape double quotes
+    .replace(/'/g, "'")   // escape single quotes
+}
+

From b1cdb571fdb6c2de47ba8f7bd5d1fb05cafc37ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adrien=20Pav=C3=A3o?= 
Date: Fri, 11 Jul 2025 13:32:00 +0200
Subject: [PATCH 05/37] Put back copy of submission files

---
 compute_worker/compute_worker.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 85a2cbe06..37c14c911 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -609,6 +609,9 @@ async def _run_program_directory(self, program_dir, kind):
                 logger.info(
                     "Program directory missing metadata, assuming it's going to be handled by ingestion"
                 )
+                # Copy submission files into prediction output
+                # This is useful for results submissions but wrongly uses storage
+                shutil.copytree(program_dir, self.output_dir)
                 return
             else:
                 raise SubmissionException("Program directory missing 'metadata.yaml/metadata'")

From 026c8beba06ddc909d5c0ebf07ee0f84ad68fcc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adrien=20Pav=C3=A3o?= 
Date: Mon, 14 Jul 2025 11:44:52 +0200
Subject: [PATCH 06/37] Update mc command from "config host add" to "alias set"

---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 09165a9d9..83762f6c7 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -78,7 +78,7 @@ services:
       /bin/sh -c "
       set -x;
       if [ -n \"$MINIO_ACCESS_KEY\" ] && [ -n \"$MINIO_SECRET_KEY\" ] && [ -n \"$MINIO_PORT\" ]; then
-        until /usr/bin/mc config host add minio_docker http://minio:$MINIO_PORT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY && break; do 
+        until /usr/bin/mc alias set minio_docker http://minio:$MINIO_PORT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY && break; do 
           echo '...waiting...' && sleep 5; 
         done;
         /usr/bin/mc mb minio_docker/$AWS_STORAGE_BUCKET_NAME || echo 'Bucket $AWS_STORAGE_BUCKET_NAME already exists.';

From 60055ea24d8587b9b73796b426a7c94fb547e7d1 Mon Sep 17 00:00:00 2001
From: Moritz 
Date: Mon, 14 Jul 2025 19:43:15 +0200
Subject: [PATCH 07/37] clamp length of competition search results description

---
 src/static/stylus/base_template.styl | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/static/stylus/base_template.styl b/src/static/stylus/base_template.styl
index 1545615e6..a69489846 100644
--- a/src/static/stylus/base_template.styl
+++ b/src/static/stylus/base_template.styl
@@ -252,6 +252,12 @@ body.pushable>.pusher
     max-height 25em !important
     overflow-y scroll !important
 
+#site-wide-competition-search .description
+    display -webkit-box
+    -webkit-box-orient vertical
+    -webkit-line-clamp 5
+    overflow hidden
+
 /* --------------------------------------------------------------------------------------
  Modals
  */

From 850c1f9b49a4f4ddea8c2f2c7a28329b1c7cc5cb Mon Sep 17 00:00:00 2001
From: Ihsan Ullah 
Date: Tue, 15 Jul 2025 15:25:32 +0500
Subject: [PATCH 08/37] Removed `num entries` from leaderboard (#1912)

* Removed num_entries becasue the count was slowing down the platform

* num entries completely removed

* test updated

---------

Co-authored-by: didayolo 
---
 src/apps/api/views/competitions.py             | 18 ------------------
 .../riot/competitions/detail/leaderboards.tag  |  2 --
 src/tests/functional/test_submissions.py       |  2 +-
 3 files changed, 1 insertion(+), 21 deletions(-)

diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py
index 056cb34bf..926da1b04 100644
--- a/src/apps/api/views/competitions.py
+++ b/src/apps/api/views/competitions.py
@@ -775,24 +775,7 @@ def get_leaderboard(self, request, pk):
         submissions_keys = {}
         submission_detailed_results = {}
         for submission in query['submissions']:
-            # count number of entries/number of submissions for the owner of this submission for this phase
-            # count all submissions except:
-            # - child submissions (submissions who has a parent i.e. parent field is not null)
-            # - Failed submissions
-            # - Cancelled submissions
-            num_entries = 1  # TMP, remove counting
-            # num_entries = Submission.objects.filter(
-            #     Q(owner__username=submission['owner']) |
-            #     Q(parent__owner__username=submission['owner']),
-            #     phase=phase,
-            # ).exclude(
-            #     Q(status=Submission.FAILED) |
-            #     Q(status=Submission.CANCELLED) |
-            #     Q(parent__isnull=False)
-            # ).count()
-
             submission_key = f"{submission['owner']}{submission['parent'] or submission['id']}"
-
             # gather detailed result from submissions for each task
             # detailed_results are gathered based on submission key
             # `id` is used to fetch the right detailed result in detailed results page
@@ -813,7 +796,6 @@ def get_leaderboard(self, request, pk):
                     'fact_sheet_answers': submission['fact_sheet_answers'],
                     'slug_url': submission['slug_url'],
                     'organization': submission['organization'],
-                    'num_entries': num_entries,
                     'created_when': submission['created_when']
                 })
             for score in submission['scores']:
diff --git a/src/static/riot/competitions/detail/leaderboards.tag b/src/static/riot/competitions/detail/leaderboards.tag
index b004d5ce9..25cc1e99a 100644
--- a/src/static/riot/competitions/detail/leaderboards.tag
+++ b/src/static/riot/competitions/detail/leaderboards.tag
@@ -34,7 +34,6 @@
- @@ -83,7 +82,6 @@ -
# ParticipantEntries Date ID {column.title}{ submission.owner } { submission.organization.name }{submission.num_entries} { pretty_date(submission.created_when) } {submission.id} diff --git a/src/tests/functional/test_submissions.py b/src/tests/functional/test_submissions.py index 3ee088ae0..6248ccaf2 100644 --- a/src/tests/functional/test_submissions.py +++ b/src/tests/functional/test_submissions.py @@ -74,7 +74,7 @@ def _run_submission_and_add_to_leaderboard(self, competition_zip_path, submissio # The leaderboard table lists our submission prediction_score = Submission.objects.get(pk=submission_id).scores.first().score - assert Decimal(self.find('leaderboards table tbody tr:nth-of-type(1) td:nth-of-type(6)').text) == round(Decimal(prediction_score), precision) + assert Decimal(self.find('leaderboards table tbody tr:nth-of-type(1) td:nth-of-type(5)').text) == round(Decimal(prediction_score), precision) def test_v15_iris_result_submission_end_to_end(self): self._run_submission_and_add_to_leaderboard('competition_15_iris.zip', 'submission_15_iris_result.zip', '======= Set 1 (Iris_test)', has_solutions=False, precision=4) From 645c8529261c90a5112607e4cad0d61cdd833fea Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Tue, 15 Jul 2025 16:00:21 +0500 Subject: [PATCH 09/37] Improved organization delete error + Remove organization reference from submission on soft detetion (#1911) * improved organization delete error * remove orgnaization from submission on soft-delete --- src/apps/api/tests/test_submissions.py | 24 ++++++++++++++++++++++++ src/apps/api/views/profiles.py | 22 +++++++++++++++------- src/apps/competitions/models.py | 4 ++++ 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/src/apps/api/tests/test_submissions.py b/src/apps/api/tests/test_submissions.py index fb5b3966f..a12e3ef13 100644 --- a/src/apps/api/tests/test_submissions.py +++ b/src/apps/api/tests/test_submissions.py @@ -554,6 +554,7 @@ def setUp(self): self.leaderboard = LeaderboardFactory() self.comp = CompetitionFactory(created_by=self.creator) self.phase = PhaseFactory(competition=self.comp) + self.organization = OrganizationFactory() # Approved participant CompetitionParticipantFactory(user=self.participant, competition=self.comp, status=CompetitionParticipant.APPROVED) @@ -591,6 +592,15 @@ def setUp(self): leaderboard=None ) + self.organization_submission = SubmissionFactory( + phase=self.phase, + owner=self.participant, + status=Submission.FINISHED, + is_soft_deleted=False, + leaderboard=None, + organization=self.organization + ) + def test_cannot_delete_submission_if_not_owner(self): """Ensure that a non-owner cannot soft delete a submission.""" self.client.login(username="other_user", password="other") @@ -639,3 +649,17 @@ def test_can_soft_delete_submission_successfully(self): # Refresh from DB to verify self.submission.refresh_from_db() assert self.submission.is_soft_deleted is True + + def test_organization_is_removed_from_soft_deleted_submission(self): + """Ensure a organization reference is removed from soft-deleted submission""" + self.client.login(username="participant", password="participant") + url = reverse("submission-soft-delete", args=[self.organization_submission.pk]) + resp = self.client.delete(url) + + assert resp.status_code == 200 + assert resp.data["message"] == "Submission deleted successfully" + + # Refresh from DB to verify + self.organization_submission.refresh_from_db() + assert self.organization_submission.is_soft_deleted is True + assert self.organization_submission.organization is None diff --git a/src/apps/api/views/profiles.py b/src/apps/api/views/profiles.py index 494b76a46..7c7f6dcd8 100644 --- a/src/apps/api/views/profiles.py +++ b/src/apps/api/views/profiles.py @@ -13,6 +13,7 @@ from rest_framework.response import Response from rest_framework import status from django.urls import reverse +from django.db import IntegrityError from api.permissions import IsUserAdminOrIsSelf, IsOrganizationEditor from api.serializers.profiles import MyProfileSerializer, UserSerializer, \ @@ -257,17 +258,24 @@ def delete_organization(self, request, pk=None): try: org = Organization.objects.get(id=pk) member = org.membership_set.get(user=request.user) - if member.group == Membership.OWNER: - org.delete() - return Response({ - "success": True, - "message": "Organization deleted!" - }) - else: + if member.group != Membership.OWNER: return Response({ "success": False, "message": "You do not have delete rights!" }) + + org.delete() + return Response({ + "success": True, + "message": "Organization deleted!" + }) + + except IntegrityError: + return Response({ + "success": False, + "message": "This organization cannot be deleted because it is associated with existing submissions. Please remove those submissions first." + }) + except Exception as e: return Response({ "success": False, diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index d8ce83dad..97b29ef8b 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -576,6 +576,7 @@ def soft_delete(self): """ Soft delete the submission: remove files but keep record in DB. Also deletes associated SubmissionDetails and cleans up storage. + Also removes organization reference from the submission """ # Remove related files from storage @@ -600,6 +601,9 @@ def soft_delete(self): # Clear the data field for this submission self.data = None + # Clear the organization field for this submission + self.organization = None + # Mark submission as deleted self.is_soft_deleted = True self.soft_deleted_when = now() From 9a44d2b737c7a852da5c26c6549f581284e7115b Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Tue, 15 Jul 2025 16:08:15 +0500 Subject: [PATCH 10/37] File size for submissions clarified (#1925) * datasets query clarified, total file size added for submissions, individual file sizes added for submissions, front end modified to show file sizes in details * flake fixes * updated names for better readability --- src/apps/api/serializers/datasets.py | 70 ++++++++++++++++++- src/apps/api/views/datasets.py | 11 ++- .../riot/submissions/resource_submissions.tag | 25 +++++++ 3 files changed, 103 insertions(+), 3 deletions(-) diff --git a/src/apps/api/serializers/datasets.py b/src/apps/api/serializers/datasets.py index 127f97478..110aba43f 100644 --- a/src/apps/api/serializers/datasets.py +++ b/src/apps/api/serializers/datasets.py @@ -78,8 +78,15 @@ class DataDetailSerializer(serializers.ModelSerializer): created_by = serializers.CharField(source='created_by.username', read_only=True) owner_display_name = serializers.SerializerMethodField() competition = serializers.SerializerMethodField() + file_size = serializers.SerializerMethodField() value = serializers.CharField(source='key', required=False) + # These fields will be conditionally returned for type == SUBMISSION only + submission_file_size = serializers.SerializerMethodField() + prediction_result_file_size = serializers.SerializerMethodField() + scoring_result_file_size = serializers.SerializerMethodField() + detailed_result_file_size = serializers.SerializerMethodField() + class Meta: model = Data fields = ( @@ -96,11 +103,72 @@ class Meta: 'value', 'was_created_by_competition', 'in_use', - 'file_size', 'competition', 'file_name', + 'file_size', + 'submission_file_size', + 'prediction_result_file_size', + 'scoring_result_file_size', + 'detailed_result_file_size', ) + def to_representation(self, instance): + """ + Called automatically by DRF when serializing a model instance to JSON. + + This method customizes the serialized output of the DataDetailSerializer. + Specifically, it removes detailed file size fields when the data type is not 'SUBMISSION'. + + Example: For input_data or scoring_program types, submission-related fields + are not relevant and will be excluded from the output. + """ + # First, generate the default serialized representation using the parent method + rep = super().to_representation(instance) + + # If this data object is NOT of type 'submission', remove the following fields + if instance.type != Data.SUBMISSION: + # These fields are only meaningful for submission-type data + rep.pop('submission_file_size', None) + rep.pop('prediction_result_file_size', None) + rep.pop('scoring_result_file_size', None) + rep.pop('detailed_result_file_size', None) + + # Return the final customized representation + return rep + + def get_file_size(self, obj): + # Check if the data object is of type 'SUBMISSION' + if obj.type == Data.SUBMISSION: + # Start with the base file size of the data file itself (if present) + total_size = obj.file_size or 0 + + # Loop through all submissions that use this data + for submission in obj.submission.all(): + # Add the size of the prediction result file (if any) + total_size += submission.prediction_result_file_size or 0 + # Add the size of the scoring result file (if any) + total_size += submission.scoring_result_file_size or 0 + # Add the size of the detailed result file (if any) + total_size += submission.detailed_result_file_size or 0 + + # Return the combined size of data file and all associated result files + return total_size + + # For non-submission data types, just return the file size as-is + return obj.file_size + + def get_submission_file_size(self, obj): + return obj.file_size or 0 + + def get_prediction_result_file_size(self, obj): + return sum([s.prediction_result_file_size or 0 for s in obj.submission.all()]) + + def get_scoring_result_file_size(self, obj): + return sum([s.scoring_result_file_size or 0 for s in obj.submission.all()]) + + def get_detailed_result_file_size(self, obj): + return sum([s.detailed_result_file_size or 0 for s in obj.submission.all()]) + def get_competition(self, obj): if obj.competition: # Submission diff --git a/src/apps/api/views/datasets.py b/src/apps/api/views/datasets.py index dade6bff9..8618af38e 100644 --- a/src/apps/api/views/datasets.py +++ b/src/apps/api/views/datasets.py @@ -52,8 +52,15 @@ def get_queryset(self): # filter datasets and programs if is_dataset: - qs = qs.filter(~Q(type=Data.SUBMISSION)) - qs = qs.exclude(Q(type=Data.COMPETITION_BUNDLE)) + qs = qs.filter(type__in=[ + Data.INPUT_DATA, + Data.PUBLIC_DATA, + Data.REFERENCE_DATA, + Data.INGESTION_PROGRAM, + Data.SCORING_PROGRAM, + Data.STARTING_KIT, + Data.SOLUTION + ]) # filter bundles if is_bundle: diff --git a/src/static/riot/submissions/resource_submissions.tag b/src/static/riot/submissions/resource_submissions.tag index 7fb10942e..f88983abd 100644 --- a/src/static/riot/submissions/resource_submissions.tag +++ b/src/static/riot/submissions/resource_submissions.tag @@ -125,6 +125,31 @@ {selected_row.description} + + + + + + + + + + + + + + + + + + + + + + + + +
File Sizes
Submission:{pretty_bytes(selected_row.submission_file_size)}
Prediction result:{pretty_bytes(selected_row.prediction_result_file_size)}
Scoring result:{pretty_bytes(selected_row.scoring_result_file_size)}
Detailed result:{pretty_bytes(selected_row.detailed_result_file_size)}