diff --git a/src/apps/api/serializers/submissions.py b/src/apps/api/serializers/submissions.py index 5511c027e..d14df9e11 100644 --- a/src/apps/api/serializers/submissions.py +++ b/src/apps/api/serializers/submissions.py @@ -56,6 +56,7 @@ class Meta: 'task', 'auto_run', 'can_make_submissions_public', + 'is_soft_deleted', ) read_only_fields = ( 'pk', @@ -67,7 +68,10 @@ class Meta: ) def get_filename(self, instance): - return basename(instance.data.data_file.name) + if instance.data and instance.data.data_file: + return basename(instance.data.data_file.name) + # NOTE: if submission data is None, it means it is soft deleted + return "Deleted File" def get_auto_run(self, instance): # returns this submission's competition auto_run_submissions Flag diff --git a/src/apps/api/tests/test_submissions.py b/src/apps/api/tests/test_submissions.py index 2ad0553a2..fb5b3966f 100644 --- a/src/apps/api/tests/test_submissions.py +++ b/src/apps/api/tests/test_submissions.py @@ -7,7 +7,7 @@ from competitions.models import Submission, CompetitionParticipant from factories import UserFactory, CompetitionFactory, PhaseFactory, CompetitionParticipantFactory, SubmissionFactory, \ - TaskFactory, OrganizationFactory, DataFactory + TaskFactory, OrganizationFactory, DataFactory, LeaderboardFactory from datasets.models import Data from profiles.models import Membership @@ -544,3 +544,98 @@ def test_cannot_re_run_submissions_with_specific_task_without_bot_user(self): resp = self.client.post(url) assert resp.status_code == 403 assert resp.data["detail"] == "You do not have permission to re-run submissions" + + +class SubmissionSoftDeletionTest(APITestCase): + def setUp(self): + self.creator = UserFactory(username='creator', password='creator') + self.participant = UserFactory(username='participant', password='participant') + + self.leaderboard = LeaderboardFactory() + self.comp = CompetitionFactory(created_by=self.creator) + self.phase = PhaseFactory(competition=self.comp) + + # Approved participant + CompetitionParticipantFactory(user=self.participant, competition=self.comp, status=CompetitionParticipant.APPROVED) + + # Submissions + self.submission = SubmissionFactory( + phase=self.phase, + owner=self.participant, + status=Submission.FINISHED, + is_soft_deleted=False, + leaderboard=None + ) + + self.leaderboard_submission = SubmissionFactory( + phase=self.phase, + owner=self.participant, + status=Submission.FINISHED, + is_soft_deleted=False, + leaderboard=self.leaderboard + ) + + self.running_submission = SubmissionFactory( + phase=self.phase, + owner=self.participant, + status=Submission.SUBMITTED, + is_soft_deleted=False, + leaderboard=None + ) + + self.soft_deleted_submission = SubmissionFactory( + phase=self.phase, + owner=self.participant, + status=Submission.FINISHED, + is_soft_deleted=True, + leaderboard=None + ) + + def test_cannot_delete_submission_if_not_owner(self): + """Ensure that a non-owner cannot soft delete a submission.""" + self.client.login(username="other_user", password="other") + url = reverse("submission-soft-delete", args=[self.submission.pk]) + resp = self.client.delete(url) + + assert resp.status_code == 403 + assert resp.data["error"] == "You are not allowed to delete this submission" + + def test_cannot_delete_leaderboard_submission(self): + """Ensure that a leaderboard submission cannot be deleted.""" + self.client.login(username="participant", password="participant") + url = reverse("submission-soft-delete", args=[self.leaderboard_submission.pk]) + resp = self.client.delete(url) + + assert resp.status_code == 403 + assert resp.data["error"] == "You are not allowed to delete a leaderboard submission" + + def test_cannot_delete_running_submission(self): + """Ensure that a running submission cannot be deleted.""" + self.client.login(username="participant", password="participant") + url = reverse("submission-soft-delete", args=[self.running_submission.pk]) + resp = self.client.delete(url) + + assert resp.status_code == 403 + assert resp.data["error"] == "You are not allowed to delete a running submission" + + def test_cannot_delete_already_soft_deleted_submission(self): + """Ensure that an already soft-deleted submission cannot be deleted again.""" + self.client.login(username="participant", password="participant") + url = reverse("submission-soft-delete", args=[self.soft_deleted_submission.pk]) + resp = self.client.delete(url) + + assert resp.status_code == 400 + assert resp.data["error"] == "Submission already deleted" + + def test_can_soft_delete_submission_successfully(self): + """Ensure a valid submission can be soft deleted successfully by its owner.""" + self.client.login(username="participant", password="participant") + url = reverse("submission-soft-delete", args=[self.submission.pk]) + resp = self.client.delete(url) + + assert resp.status_code == 200 + assert resp.data["message"] == "Submission deleted successfully" + + # Refresh from DB to verify + self.submission.refresh_from_db() + assert self.submission.is_soft_deleted is True diff --git a/src/apps/api/views/datasets.py b/src/apps/api/views/datasets.py index 1c2d10a10..c3c48c6ef 100644 --- a/src/apps/api/views/datasets.py +++ b/src/apps/api/views/datasets.py @@ -158,7 +158,7 @@ def check_delete_permissions(self, request, dataset): if dataset.submission.first(): sub = dataset.submission.first() if sub.phase: - return 'Cannot delete submission: submission belongs to an existing competition' + return 'Cannot delete submission: submission belongs to an existing competition. Please visit the competition and delete your submission from there.' class DataGroupViewSet(ModelViewSet): diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 8321de927..5bcbffc1a 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -32,7 +32,7 @@ class SubmissionViewSet(ModelViewSet): queryset = Submission.objects.all().order_by('-pk') permission_classes = [] filter_backends = (DjangoFilterBackend, SearchFilter) - filter_fields = ('phase__competition', 'phase', 'status') + filter_fields = ('phase__competition', 'phase', 'status', 'is_soft_deleted') search_fields = ('data__data_file', 'description', 'name', 'owner__username') renderer_classes = api_settings.DEFAULT_RENDERER_CLASSES + [renderers.CSVRenderer] @@ -119,6 +119,9 @@ def get_queryset(self): if not self.request.user.is_authenticated: return Submission.objects.none() + # Check if admin is requesting to see soft-deleted submissions + show_is_soft_deleted = self.request.query_params.get('show_is_soft_deleted', 'false').lower() == 'true' + if not self.request.user.is_superuser and not self.request.user.is_staff and not self.request.user.is_bot: # if you're the creator of the submission or a collaborator on the competition qs = qs.filter( @@ -126,6 +129,11 @@ def get_queryset(self): Q(phase__competition__created_by=self.request.user) | Q(phase__competition__collaborators__in=[self.request.user.pk]) ).distinct() + + # By default, exclude soft-deleted submissions unless explicitly requested by an admin + if not show_is_soft_deleted: + qs = qs.filter(is_soft_deleted=False) + qs = qs.select_related( 'phase', 'phase__competition', @@ -179,6 +187,34 @@ def destroy(self, request, *args, **kwargs): self.perform_destroy(submission) return Response(status=status.HTTP_204_NO_CONTENT) + @action(detail=True, methods=('DELETE',)) + def soft_delete(self, request, pk): + submission = self.get_object() + + # Check if submission exists + if not submission: + return Response({'error': 'Submission not found'}, status=status.HTTP_404_NOT_FOUND) + + # Check if owner is requesting soft delete + if submission.owner != request.user: + return Response({'error': 'You are not allowed to delete this submission'}, status=status.HTTP_403_FORBIDDEN) + + # Check if submission is finished and on the leaderboard + if submission.status == Submission.FINISHED and submission.on_leaderboard: + return Response({'error': 'You are not allowed to delete a leaderboard submission'}, status=status.HTTP_403_FORBIDDEN) + + # Check if submission is in running state + if submission.status not in [Submission.FAILED, Submission.FINISHED]: + return Response({'error': 'You are not allowed to delete a running submission'}, status=status.HTTP_403_FORBIDDEN) + + # Check if submission is not already soft deleted + if submission.is_soft_deleted: + return Response({'error': 'Submission already deleted'}, status=status.HTTP_400_BAD_REQUEST) + + # soft delete submission and return success response + submission.soft_delete() + return Response({'message': 'Submission deleted successfully'}, status=status.HTTP_200_OK) + @action(detail=False, methods=('DELETE',)) def delete_many(self, request, *args, **kwargs): qs = self.get_queryset() diff --git a/src/apps/competitions/migrations/0052_auto_20250129_1058.py b/src/apps/competitions/migrations/0052_auto_20250129_1058.py new file mode 100644 index 000000000..dcdd25c8a --- /dev/null +++ b/src/apps/competitions/migrations/0052_auto_20250129_1058.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2.28 on 2025-01-29 10:58 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('competitions', '0051_merge_20241203_1313'), + ] + + operations = [ + migrations.AddField( + model_name='submission', + name='is_soft_deleted', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='submission', + name='soft_deleted_when', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='submission', + name='data', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='submission', to='datasets.Data'), + ), + ] diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index 24efd11c2..cd381b56b 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -515,7 +515,7 @@ class Submission(ChaHubSaveMixin, models.Model): status_details = models.TextField(null=True, blank=True) phase = models.ForeignKey(Phase, related_name='submissions', on_delete=models.CASCADE) appear_on_leaderboards = models.BooleanField(default=False) - data = models.ForeignKey("datasets.Data", on_delete=models.CASCADE, related_name='submission') + data = models.ForeignKey("datasets.Data", on_delete=models.SET_NULL, related_name='submission', null=True, blank=True) md5 = models.CharField(max_length=32, null=True, blank=True) prediction_result = models.FileField(upload_to=PathWrapper('prediction_result'), null=True, blank=True, @@ -561,9 +561,47 @@ class Submission(ChaHubSaveMixin, models.Model): fact_sheet_answers = JSONField(null=True, blank=True, max_length=4096) + # True when submission owner deletes a submission + is_soft_deleted = models.BooleanField(default=False) + # DataTime of when a submission is soft_deleted + soft_deleted_when = models.DateTimeField(null=True, blank=True) + def __str__(self): return f"{self.phase.competition.title} submission PK={self.pk} by {self.owner.username}" + def soft_delete(self): + """ + Soft delete the submission: remove files but keep record in DB. + Also deletes associated SubmissionDetails and cleans up storage. + """ + + # Remove related files from storage + # 'save=False' prevents a database save, which is handled later after marking the submission as soft-deleted. + self.prediction_result.delete(save=False) + self.prediction_result_file_size = 0 + self.scoring_result.delete(save=False) + self.scoring_result_file_size = 0 + self.detailed_result.delete(save=False) + self.detailed_result_file_size = 0 + + # Delete related SubmissionDetails files and records + for detail in self.details.all(): + detail.data_file.delete(save=False) # Delete file from storage + detail.delete() # Remove record from DB + + # Clear the data field if no other submissions are using it + other_submissions_using_data = Submission.objects.filter(data=self.data).exclude(pk=self.pk).exists() + if not other_submissions_using_data: + self.data.delete() + + # Clear the data field for this submission + self.data = None + + # Mark submission as deleted + self.is_soft_deleted = True + self.soft_deleted_when = now() + self.save() + def delete(self, **kwargs): # Check if any other submissions are using the same data diff --git a/src/apps/competitions/statistics.py b/src/apps/competitions/statistics.py index d77cc624a..4a06e0022 100644 --- a/src/apps/competitions/statistics.py +++ b/src/apps/competitions/statistics.py @@ -1,23 +1,163 @@ +""" +This script is created to compute two types of statistics: + 1. Overall platform statistics for a specified year + 2. Overall published competitions statistics + +Usage: + Bash into django console + ``` + docker compose exec django ./manage.py shell_plus + ``` + + For overall platform statistics + ``` + from competitions.statistics import create_codabench_statistics + create_codabench_statistics(year=2024) + + # if year is not specified, current year is used by default + # a csv file named codabench_statistics_2024.csv is generated in statistics folder (for year=2024) + ``` + + For overall published competitions statistics + ``` + from competitions.statistics import create_codabench_statistics_published_comps + create_codabench_statistics_published_comps() + + # a csv file named codabench_statistics_published_comps.csv is generated in statistics folder + ``` +""" + # -------------------------------------------------- # Imports # -------------------------------------------------- import os -from competitions.models import Competition +from datetime import datetime +from competitions.models import Competition, Submission, CompetitionParticipant +from profiles.models import User # -------------------------------------------------- # Setting constants # -------------------------------------------------- BASE_URL = "https://www.codabench.org/competitions/" STATISTICS_DIR = "/app/statistics/" -CSV_FILE_NAME = "codabench_competition_statistics.csv" -CSV_PATH = STATISTICS_DIR + CSV_FILE_NAME -def create_codabench_statistics(): +def create_codabench_statistics(year=None): + """ + This function prepares a CSV file with different statistics per month + """ + + # Set year to current year if None + if year is None: + year = datetime.now().year + + # Create statistics directory if not already createad + if not os.path.exists(STATISTICS_DIR): + os.makedirs(STATISTICS_DIR) + + rows_dict = {} + + # Initialize sets for tracking total of users, participants and submissions + total_users = set() + total_participants = set() + total_submissions = set() + + # Loop over months + for month in range(1, 13): + + # count total competitions + tota_competitions_count = Competition.objects.filter(created_when__year=year, created_when__month=month).count() + rows_dict.setdefault("total_competitions", []).append(tota_competitions_count) + + # count public competitions + public_competitions_count = Competition.objects.filter(created_when__year=year, created_when__month=month, published=True).count() + rows_dict.setdefault("public_competitions", []).append(public_competitions_count) + + # count private competitions + private_competitions_count = Competition.objects.filter(created_when__year=year, created_when__month=month, published=False).count() + rows_dict.setdefault("private_competitions", []).append(private_competitions_count) + + # Count new users + new_users_count = User.objects.filter(date_joined__year=year, date_joined__month=month).count() + rows_dict.setdefault("new_users", []).append(new_users_count) + + # Count total users (including the current month) + new_user_ids = set(User.objects.filter(date_joined__year=year, date_joined__month=month).values_list('id', flat=True)) + total_users.update(new_user_ids) + rows_dict.setdefault("total_users", []).append(len(total_users)) + + # Count new participants + new_participants_count = CompetitionParticipant.objects.filter(competition__created_when__year=year, competition__created_when__month=month).count() + rows_dict.setdefault("new_participants", []).append(new_participants_count) + + # Count total participants (including the current month) + new_participants_ids = set(CompetitionParticipant.objects.filter(competition__created_when__year=year, competition__created_when__month=month).values_list('id', flat=True)) + total_participants.update(new_participants_ids) + rows_dict.setdefault("total_participants", []).append(len(total_participants)) + + # Count new submissions + new_submissions_count = Submission.objects.filter(created_when__year=year, created_when__month=month).count() + rows_dict.setdefault("new_submissions", []).append(new_submissions_count) + + # Submissions per day = total submissions/30 + submissions_per_day = 0 + if new_submissions_count > 0: + submissions_per_day = int(new_submissions_count / 30) + rows_dict.setdefault("submissions_per_day", []).append(submissions_per_day) + + # Count successful submissions (i.e., those that are finished) + successful_submissions = Submission.objects.filter(created_when__year=year, created_when__month=month, status=Submission.FINISHED).count() + rows_dict.setdefault("finished_submissions", []).append(successful_submissions) + + # Count failed submissions (i.e., those that are failed) + failed_submissions = Submission.objects.filter(created_when__year=year, created_when__month=month, status=Submission.FAILED).count() + rows_dict.setdefault("failed_submissions", []).append(failed_submissions) + + # Count total submissions (including the current month) + new_submissions_ids = set(Submission.objects.filter(created_when__year=year, created_when__month=month).values_list('id', flat=True)) + total_submissions.update(new_submissions_ids) + rows_dict.setdefault("total_submissions", []).append(len(total_submissions)) + + # Set CSV file and path + CSV_FILE_NAME = f"codabench_statistics_{year}.csv" + CSV_PATH = STATISTICS_DIR + CSV_FILE_NAME + + # Define month abbreviations + month_abbr = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] + + # Open the CSV file in write mode + with open(CSV_PATH, 'w') as output_file: + # Write the header row only once if the file is empty + if output_file.tell() == 0: + header = f"{year}; " + "; ".join(month_abbr) + "; Total \n" + output_file.write(header) + + # Loop over each metric in the rows_dict and write the corresponding row + for metric, values in rows_dict.items(): + + # for total_users, total_participants, and total_submissions + # total is the last value + # for others total is the sum + if metric in ["total_users", "total_participants", "total_submissions"]: + total = values[-1] + else: + # Calculate the total for the metric (sum of all monthly counts) + total = sum(values) + + # Create a row with the metric name followed by the values for each month + row = f"{metric}; " + "; ".join(map(str, values)) + f"; {total} \n" + output_file.write(row) + + +def create_codabench_statistics_published_comps(): """ This function prepares a CSV file with all published competitions """ + # Set CSV file and path + CSV_FILE_NAME = "codabench_statistics_published_comps.csv" + CSV_PATH = STATISTICS_DIR + CSV_FILE_NAME + # Create statistics directory if not already createad if not os.path.exists(STATISTICS_DIR): os.makedirs(STATISTICS_DIR) diff --git a/src/apps/forums/views.py b/src/apps/forums/views.py index 39f80ce3b..d74d59509 100644 --- a/src/apps/forums/views.py +++ b/src/apps/forums/views.py @@ -1,16 +1,18 @@ import datetime +from django.contrib import messages from django.contrib.auth import get_user_model from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin from django.core.exceptions import PermissionDenied from django.http import HttpResponseRedirect, Http404 -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, redirect from django.utils.timezone import now from django.views.generic import DetailView, CreateView, DeleteView from .forms import PostForm, ThreadForm from .models import Forum, Thread, Post +from competitions.models import CompetitionParticipant User = get_user_model() @@ -26,16 +28,31 @@ def dispatch(self, *args, **kwargs): self.forum = get_object_or_404(Forum, pk=self.kwargs['forum_pk']) if 'thread_pk' in self.kwargs: self.thread = get_object_or_404(Thread, pk=self.kwargs['thread_pk']) + + # Determine if the user is a participant and store it as an instance variable + self.is_participant = self.is_user_participant(self.request.user, self.forum) + return super().dispatch(*args, **kwargs) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context['forum'] = self.forum context['thread'] = self.thread if hasattr(self, 'thread') else None + context['is_participant'] = self.is_participant return context + def is_user_participant(self, user, forum): + is_participant = False + if user.is_authenticated: + is_participant = CompetitionParticipant.objects.filter( + competition=forum.competition, + user=user, + status=CompetitionParticipant.APPROVED + ).exists() + return is_participant + -class ForumDetailView(DetailView): +class ForumDetailView(ForumBaseMixin, DetailView): """ Shows the details of a particular Forum. """ @@ -45,9 +62,15 @@ class ForumDetailView(DetailView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context['thread_list_sorted'] = self.object.threads.order_by('pinned_date', '-date_created')\ - .select_related('forum', 'forum__competition', 'forum__competition__created_by', 'started_by')\ - .prefetch_related('forum__competition__collaborators', 'posts') + + context['thread_list_sorted'] = self.object.threads.order_by( + 'pinned_date', '-date_created' + ).select_related( + 'forum', 'forum__competition', 'forum__competition__created_by', 'started_by' + ).prefetch_related( + 'forum__competition__collaborators', 'posts' + ) + return context @@ -66,6 +89,12 @@ class CreatePostView(ForumBaseMixin, RedirectToThreadMixin, LoginRequiredMixin, form_class = PostForm def form_valid(self, form): + + if not self.is_participant: + messages.error(self.request, "You must be a participant of this competition to create a post.") + return redirect("forums:forum_thread_detail", forum_pk=self.forum.pk, thread_pk=self.thread.pk) + + # Create the post since the user is a participant self.post = form.save(commit=False) self.post.thread = self.thread self.post.posted_by = self.request.user @@ -106,6 +135,13 @@ class CreateThreadView(ForumBaseMixin, RedirectToThreadMixin, LoginRequiredMixin form_class = ThreadForm def form_valid(self, form): + + if not self.is_participant: + messages.error(self.request, "You must be a participant of this competition to create a thread.") + return redirect("forums:forum_detail", forum_pk=self.forum.pk) + + # Create the thread since the user is a participant + self.thread = form.save(commit=False) self.thread = form.save(commit=False) self.thread.forum = self.forum self.thread.started_by = self.request.user diff --git a/src/apps/pages/views.py b/src/apps/pages/views.py index a69ee2d10..881ec885c 100644 --- a/src/apps/pages/views.py +++ b/src/apps/pages/views.py @@ -49,13 +49,19 @@ def get_context_data(self, *args, **kwargs): # filter this user's own submissions # and # submissions running on queue which belongs to this user + # NOTE: exclude all soft-deleted submissions if not self.request.user.is_superuser: qs = Submission.objects.filter( - Q(owner=self.request.user) | - Q(phase__competition__queue__isnull=False, phase__competition__queue__owner=self.request.user) + Q(is_soft_deleted=False) & + ( + Q(owner=self.request.user) | + Q(phase__competition__queue__isnull=False, phase__competition__queue__owner=self.request.user) + ) ) else: - qs = Submission.objects.all() + qs = Submission.objects.filter( + Q(is_soft_deleted=False) + ) # Filter out child submissions i.e. submission has no parent if not show_child_submissions: @@ -82,7 +88,10 @@ def get_context_data(self, *args, **kwargs): for submission in context['submissions']: # Get filesize from each submissions's data - submission.file_size = self.format_file_size(submission.data.file_size) + if submission.data: + submission.file_size = self.format_file_size(submission.data.file_size) + else: + submission.file_size = self.format_file_size(0) # Get queue from each submission queue_name = "" diff --git a/src/apps/profiles/admin.py b/src/apps/profiles/admin.py index 99d23029a..e22e8f9ea 100644 --- a/src/apps/profiles/admin.py +++ b/src/apps/profiles/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from .models import User, Organization, Membership +from .models import User, DeletedUser, Organization, Membership class UserAdmin(admin.ModelAdmin): @@ -12,7 +12,14 @@ class UserAdmin(admin.ModelAdmin): list_display = ['username', 'email', 'is_staff', 'is_superuser'] +class DeletedUserAdmin(admin.ModelAdmin): + list_display = ('user_id', 'username', 'email', 'deleted_at') + search_fields = ('username', 'email') + list_filter = ('deleted_at',) + + admin.site.register(User, UserAdmin) +admin.site.register(DeletedUser, DeletedUserAdmin) admin.site.register(Organization) admin.site.register(Membership) diff --git a/src/apps/profiles/migrations/0015_deleteduser.py b/src/apps/profiles/migrations/0015_deleteduser.py new file mode 100644 index 000000000..39443aaae --- /dev/null +++ b/src/apps/profiles/migrations/0015_deleteduser.py @@ -0,0 +1,22 @@ +# Generated by Django 2.2.28 on 2025-02-04 07:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('profiles', '0014_auto_20241120_1607'), + ] + + operations = [ + migrations.CreateModel( + name='DeletedUser', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('username', models.CharField(max_length=255)), + ('email', models.EmailField(max_length=254)), + ('deleted_at', models.DateTimeField(auto_now_add=True)), + ], + ), + ] diff --git a/src/apps/profiles/migrations/0016_deleteduser_user_id.py b/src/apps/profiles/migrations/0016_deleteduser_user_id.py new file mode 100644 index 000000000..e1637c9e6 --- /dev/null +++ b/src/apps/profiles/migrations/0016_deleteduser_user_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.28 on 2025-02-04 16:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('profiles', '0015_deleteduser'), + ] + + operations = [ + migrations.AddField( + model_name='deleteduser', + name='user_id', + field=models.IntegerField(blank=True, null=True), + ), + ] diff --git a/src/apps/profiles/models.py b/src/apps/profiles/models.py index 991b4451f..d66be9804 100644 --- a/src/apps/profiles/models.py +++ b/src/apps/profiles/models.py @@ -33,6 +33,16 @@ def all_objects(self): return super().get_queryset() +class DeletedUser(models.Model): + user_id = models.IntegerField(null=True, blank=True) # Store the same ID as in the User table + username = models.CharField(max_length=255) + email = models.EmailField() + deleted_at = models.DateTimeField(auto_now_add=True) # Automatically sets to current time when the record is created + + def __str__(self): + return f"{self.username} ({self.email})" + + class User(ChaHubSaveMixin, AbstractBaseUser, PermissionsMixin): # Social needs the below setting. Username is not really set to UID. USERNAME_FIELD = 'username' @@ -143,7 +153,9 @@ def get_chahub_is_valid(self): # By default, always push return True - def get_used_storage_space(self): + def get_used_storage_space(self, binary=False): + + factor = 1024 if binary else 1000 from datasets.models import Data from competitions.models import Submission, SubmissionDetails @@ -154,7 +166,7 @@ def get_used_storage_space(self): created_by_id=self.id, file_size__gt=0, file_size__isnull=False ).aggregate(Sum("file_size"))["file_size__sum"] - storage_used += users_datasets * 1024 if users_datasets else 0 + storage_used += users_datasets * factor if users_datasets else 0 # Submissions users_submissions = Submission.objects.filter(owner_id=self.id).aggregate( @@ -186,20 +198,21 @@ def get_used_storage_space(self): ) ) - storage_used += users_submissions["size"] * 1024 if users_submissions["size"] else 0 + storage_used += users_submissions["size"] * factor if users_submissions["size"] else 0 # Submissions details users_submissions_details = SubmissionDetails.objects.filter( submission__owner_id=self.id, file_size__gt=0, file_size__isnull=False ).aggregate(Sum("file_size"))["file_size__sum"] - storage_used += users_submissions_details * 1024 if users_submissions_details else 0 + storage_used += users_submissions_details * factor if users_submissions_details else 0 return storage_used def delete(self, *args, **kwargs): """Soft delete the user and anonymize personal data.""" from .views import send_user_deletion_notice_to_admin, send_user_deletion_confirmed + from .models import DeletedUser # Send a notice to admins send_user_deletion_notice_to_admin(self) @@ -212,6 +225,13 @@ def delete(self, *args, **kwargs): # Anonymize or removed personal data user_email = self.email # keep track of the email for the end of the procedure + # Store the deleted user's data in the DeletedUser table + DeletedUser.objects.create( + user_id=self.id, + username=self.username, + email=self.email + ) + # Github related self.github_uid = None self.avatar_url = None diff --git a/src/apps/profiles/tasks.py b/src/apps/profiles/tasks.py new file mode 100644 index 000000000..4dabbf2f6 --- /dev/null +++ b/src/apps/profiles/tasks.py @@ -0,0 +1,28 @@ +import time +import logging +from datetime import timedelta +from django.utils.timezone import now +from celery_config import app + +from profiles.models import DeletedUser + +logger = logging.getLogger() + + +@app.task(queue="site-worker") +def clean_deleted_users(): + starting_time = time.process_time() + logger.info("Task clean_deleted_users Started") + + # Calculate the threshold date (one month ago) + one_month_ago = now() - timedelta(days=30) + + # Delete users who were deleted more than a month ago + deleted_count, _ = DeletedUser.objects.filter(deleted_at__lt=one_month_ago).delete() + + logger.info(f"Deleted {deleted_count} users from DeletedUser table.") + + elapsed_time = time.process_time() - starting_time + logger.info( + "Task clean_deleted_users Completed. Duration = {:.3f} seconds".format(elapsed_time) + ) diff --git a/src/apps/profiles/views.py b/src/apps/profiles/views.py index 10262c699..6c3a1533b 100644 --- a/src/apps/profiles/views.py +++ b/src/apps/profiles/views.py @@ -20,7 +20,7 @@ from api.serializers.profiles import UserSerializer, OrganizationDetailSerializer, OrganizationEditSerializer, \ UserNotificationSerializer from .forms import SignUpForm, LoginForm, ActivationForm -from .models import User, Organization, Membership +from .models import User, DeletedUser, Organization, Membership from oidc_configurations.models import Auth_Organization from .tokens import account_activation_token, account_deletion_token from competitions.models import Competition @@ -208,14 +208,20 @@ def sign_up(request): if request.method == 'POST': form = SignUpForm(request.POST) if form.is_valid(): - form.save() - username = form.cleaned_data.get('username') - raw_password = form.cleaned_data.get('password1') - user = authenticate(username=username, password=raw_password) - user.is_active = False - user.save() - activateEmail(request, user, form.cleaned_data.get('email')) - return redirect('pages:home') + # Check if the email is in the DeletedUser table + email = form.cleaned_data.get('email') + if DeletedUser.objects.filter(email=email).exists(): + messages.error(request, "This email has been previously deleted and cannot be used.") + context['form'] = form + else: + form.save() + username = form.cleaned_data.get('username') + raw_password = form.cleaned_data.get('password1') + user = authenticate(username=username, password=raw_password) + user.is_active = False + user.save() + activateEmail(request, user, form.cleaned_data.get('email')) + return redirect('pages:home') else: context['form'] = form diff --git a/src/settings/base.py b/src/settings/base.py index 70a8fc01e..a8e297519 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -236,6 +236,10 @@ 'task': 'analytics.tasks.update_home_page_counters', 'schedule': timedelta(days=1), # Run every 24 hours }, + 'clean_deleted_users': { + 'task': 'profiles.tasks.clean_deleted_users', + 'schedule': timedelta(days=1), # Run every 24 hours + }, 'reset_computed_storage_analytics': { 'task': 'analytics.tasks.reset_computed_storage_analytics', 'schedule': crontab(hour='2', minute='0', day_of_month='1', month_of_year="*/3") # Every 3 month at 02:00 UTC on the 1st @@ -416,7 +420,8 @@ GS_BUCKET_NAME = GS_PUBLIC_BUCKET_NAME # Default bucket set to public bucket # Quota -DEFAULT_USER_QUOTA = 15 * 1024 * 1024 * 1024 # 15GB +DEFAULT_USER_QUOTA = 15 * 1000 * 1000 * 1000 # 15GB + # ============================================================================= # Debug diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index cf4baf563..410ec34cf 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -93,6 +93,9 @@ CODALAB.api = { delete_submission: function (pk) { return CODALAB.api.request('DELETE', `${URLS.API}submissions/${pk}/`) }, + soft_delete_submission: function (pk) { + return CODALAB.api.request('DELETE', `${URLS.API}submissions/${pk}/soft_delete/`) + }, delete_many_submissions: function (pks) { return CODALAB.api.request('DELETE', `${URLS.API}submissions/delete_many/`, pks) }, diff --git a/src/static/js/ours/utils.js b/src/static/js/ours/utils.js index 5a0edc358..7b2f7cdeb 100644 --- a/src/static/js/ours/utils.js +++ b/src/static/js/ours/utils.js @@ -89,15 +89,17 @@ function pretty_date(date_string) { } } -function pretty_bytes(bytes, decimal_places=1, suffix="B") { - const units = ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']; +function pretty_bytes(bytes, decimalPlaces = 1, suffix = "B", binary = false) { + const factor = binary ? 1024.0 : 1000.0; + const units = binary ? ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi'] : ['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z']; + for (const unit of units) { - if (Math.abs(bytes) < 1024.0 || unit == 'PiB') { - return bytes.toFixed(decimal_places) + unit + suffix; + if (Math.abs(bytes) < factor || unit === units[units.length - 1]) { + return bytes.toFixed(decimalPlaces) + unit + suffix; } - bytes /= 1024.0; + bytes /= factor; } - return bytes.toFixed(decimal_places) + "Pi" + suffix; + return bytes.toFixed(decimalPlaces) + units[units.length - 1] + suffix; } /* ---------------------------------------------------------------------------- diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index e714718d8..e5a5f1f7c 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -46,6 +46,13 @@ +
+ + +
@@ -62,10 +69,10 @@ - - + + @@ -78,9 +85,9 @@ - - + @@ -201,14 +219,13 @@
Date Status Score - Detailed ResultsActions + Detailed Results + Actions
-
+ onclick="{ submission_clicked.bind(this, submission) }" class="submission_row {submission.is_soft_deleted ? 'soft-deleted' : ''}"> +
+
@@ -108,8 +115,11 @@ 		- + + + + @@ -157,6 +167,14 @@ onclick="{toggle_submission_is_public.bind(this, submission)}"> + + + + + +