From f0bb5c1138307f181a5eb09f5720cdbd5bbc87d7 Mon Sep 17 00:00:00 2001 From: sainath Date: Fri, 13 Dec 2024 15:41:46 +0530 Subject: [PATCH 1/6] chore: added fields in issue_version and profile tables and created a new sticky table --- ...emove_issueversion_description_and_more.py | 117 ++++++++++++++++++ apiserver/plane/db/models/__init__.py | 11 +- apiserver/plane/db/models/issue.py | 115 +++++++++++++---- apiserver/plane/db/models/sticky.py | 32 +++++ apiserver/plane/db/models/user.py | 12 ++ 5 files changed, 252 insertions(+), 35 deletions(-) create mode 100644 apiserver/plane/db/migrations/0087_remove_issueversion_description_and_more.py create mode 100644 apiserver/plane/db/models/sticky.py diff --git a/apiserver/plane/db/migrations/0087_remove_issueversion_description_and_more.py b/apiserver/plane/db/migrations/0087_remove_issueversion_description_and_more.py new file mode 100644 index 00000000000..9ca9fd747f6 --- /dev/null +++ b/apiserver/plane/db/migrations/0087_remove_issueversion_description_and_more.py @@ -0,0 +1,117 @@ +# Generated by Django 4.2.17 on 2024-12-13 10:09 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import plane.db.models.user +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('db', '0086_issueversion_alter_teampage_unique_together_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='issueversion', + name='description', + ), + migrations.RemoveField( + model_name='issueversion', + name='description_binary', + ), + migrations.RemoveField( + model_name='issueversion', + name='description_html', + ), + migrations.RemoveField( + model_name='issueversion', + name='description_stripped', + ), + migrations.AddField( + model_name='issueversion', + name='activity', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='versions', to='db.issueactivity'), + ), + migrations.AddField( + model_name='issueversion', + name='point', + field=models.IntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(12)]), + ), + migrations.AddField( + model_name='profile', + name='is_mobile_onboarded', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='profile', + name='mobile_onboarding_step', + field=models.JSONField(default=plane.db.models.user.get_mobile_default_onboarding), + ), + migrations.AddField( + model_name='profile', + name='mobile_timezone_auto_set', + field=models.BooleanField(default=False), + ), + migrations.AlterField( + model_name='issueversion', + name='owned_by', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='issue_versions', to=settings.AUTH_USER_MODEL), + ), + migrations.CreateModel( + name='Sticky', + fields=[ + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='Created At')), + ('updated_at', models.DateTimeField(auto_now=True, verbose_name='Last Modified At')), + ('deleted_at', models.DateTimeField(blank=True, null=True, verbose_name='Deleted At')), + ('id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('name', models.TextField()), + ('description', models.JSONField(blank=True, default=dict)), + ('description_html', models.TextField(blank=True, default='

')), + ('description_stripped', models.TextField(blank=True, null=True)), + ('description_binary', models.BinaryField(null=True)), + ('logo_props', models.JSONField(default=dict)), + ('color', models.CharField(blank=True, max_length=255, null=True)), + ('background_color', models.CharField(blank=True, max_length=255, null=True)), + ('created_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='%(class)s_created_by', to=settings.AUTH_USER_MODEL, verbose_name='Created By')), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='stickies', to=settings.AUTH_USER_MODEL)), + ('updated_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='%(class)s_updated_by', to=settings.AUTH_USER_MODEL, verbose_name='Last Modified By')), + ('workspace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='stickies', to='db.workspace')), + ], + options={ + 'verbose_name': 'Sticky', + 'verbose_name_plural': 'Stickies', + 'db_table': 'stickies', + 'ordering': ('-created_at',), + }, + ), + migrations.CreateModel( + name='IssueDescriptionVersion', + fields=[ + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='Created At')), + ('updated_at', models.DateTimeField(auto_now=True, verbose_name='Last Modified At')), + ('deleted_at', models.DateTimeField(blank=True, null=True, verbose_name='Deleted At')), + ('id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('description_binary', models.BinaryField(null=True)), + ('description_html', models.TextField(blank=True, default='

')), + ('description_stripped', models.TextField(blank=True, null=True)), + ('description_json', models.JSONField(blank=True, default=dict)), + ('last_saved_at', models.DateTimeField(default=django.utils.timezone.now)), + ('created_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='%(class)s_created_by', to=settings.AUTH_USER_MODEL, verbose_name='Created By')), + ('issue', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='description_versions', to='db.issue')), + ('owned_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='issue_description_versions', to=settings.AUTH_USER_MODEL)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='project_%(class)s', to='db.project')), + ('updated_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='%(class)s_updated_by', to=settings.AUTH_USER_MODEL, verbose_name='Last Modified By')), + ('workspace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='workspace_%(class)s', to='db.workspace')), + ], + options={ + 'verbose_name': 'Issue Description Version', + 'verbose_name_plural': 'Issue Description Versions', + 'db_table': 'issue_description_versions', + }, + ), + ] diff --git a/apiserver/plane/db/models/__init__.py b/apiserver/plane/db/models/__init__.py index e3a9df2542a..4c2d57d80f2 100644 --- a/apiserver/plane/db/models/__init__.py +++ b/apiserver/plane/db/models/__init__.py @@ -68,15 +68,6 @@ WorkspaceUserProperties, ) - - - - - - - - - from .favorite import UserFavorite from .issue_type import IssueType @@ -86,3 +77,5 @@ from .label import Label from .device import Device, DeviceSession + +from .sticky import Sticky diff --git a/apiserver/plane/db/models/issue.py b/apiserver/plane/db/models/issue.py index 9ea1d3b2646..e3933cefa2a 100644 --- a/apiserver/plane/db/models/issue.py +++ b/apiserver/plane/db/models/issue.py @@ -15,6 +15,7 @@ from plane.utils.html_processor import strip_tags from plane.db.mixins import SoftDeletionManager from plane.utils.exception_logger import log_exception +from .base import BaseModel from .project import ProjectBaseModel @@ -660,9 +661,6 @@ def __str__(self): class IssueVersion(ProjectBaseModel): - issue = models.ForeignKey( - "db.Issue", on_delete=models.CASCADE, related_name="versions" - ) PRIORITY_CHOICES = ( ("urgent", "Urgent"), ("high", "High"), @@ -670,14 +668,14 @@ class IssueVersion(ProjectBaseModel): ("low", "Low"), ("none", "None"), ) + parent = models.UUIDField(blank=True, null=True) state = models.UUIDField(blank=True, null=True) + point = models.IntegerField( + validators=[MinValueValidator(0), MaxValueValidator(12)], null=True, blank=True + ) estimate_point = models.UUIDField(blank=True, null=True) name = models.CharField(max_length=255, verbose_name="Issue Name") - description = models.JSONField(blank=True, default=dict) - description_html = models.TextField(blank=True, default="

") - description_stripped = models.TextField(blank=True, null=True) - description_binary = models.BinaryField(null=True) priority = models.CharField( max_length=30, choices=PRIORITY_CHOICES, @@ -686,7 +684,9 @@ class IssueVersion(ProjectBaseModel): ) start_date = models.DateField(null=True, blank=True) target_date = models.DateField(null=True, blank=True) + assignees = ArrayField(models.UUIDField(), blank=True, default=list) sequence_id = models.IntegerField(default=1, verbose_name="Issue Sequence ID") + labels = ArrayField(models.UUIDField(), blank=True, default=list) sort_order = models.FloatField(default=65535) completed_at = models.DateTimeField(null=True) archived_at = models.DateField(null=True) @@ -694,14 +694,26 @@ class IssueVersion(ProjectBaseModel): external_source = models.CharField(max_length=255, null=True, blank=True) external_id = models.CharField(max_length=255, blank=True, null=True) type = models.UUIDField(blank=True, null=True) - last_saved_at = models.DateTimeField(default=timezone.now) - owned_by = models.UUIDField() - assignees = ArrayField(models.UUIDField(), blank=True, default=list) - labels = ArrayField(models.UUIDField(), blank=True, default=list) cycle = models.UUIDField(null=True, blank=True) modules = ArrayField(models.UUIDField(), blank=True, default=list) - properties = models.JSONField(default=dict) - meta = models.JSONField(default=dict) + properties = models.JSONField(default=dict) # issue properties + meta = models.JSONField(default=dict) # issue meta + last_saved_at = models.DateTimeField(default=timezone.now) + + issue = models.ForeignKey( + "db.Issue", on_delete=models.CASCADE, related_name="versions" + ) + activity = models.ForeignKey( + "db.IssueActivity", + on_delete=models.SET_NULL, + null=True, + related_name="versions", + ) + owned_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + related_name="issue_versions", + ) class Meta: verbose_name = "Issue Version" @@ -721,36 +733,87 @@ def log_issue_version(cls, issue, user): Module = apps.get_model("db.Module") CycleIssue = apps.get_model("db.CycleIssue") + IssueAssignee = apps.get_model("db.IssueAssignee") + IssueLabel = apps.get_model("db.IssueLabel") cycle_issue = CycleIssue.objects.filter(issue=issue).first() cls.objects.create( issue=issue, - parent=issue.parent, - state=issue.state, + parent=issue.parent_id, + state=issue.state_id, point=issue.point, - estimate_point=issue.estimate_point, + estimate_point=issue.estimate_point_id, name=issue.name, - description=issue.description, - description_html=issue.description_html, - description_stripped=issue.description_stripped, - description_binary=issue.description_binary, priority=issue.priority, start_date=issue.start_date, target_date=issue.target_date, + assignees=list( + IssueAssignee.objects.filter(issue=issue).values_list( + "assignee_id", flat=True + ) + ), sequence_id=issue.sequence_id, + labels=list( + IssueLabel.objects.filter(issue=issue).values_list( + "label_id", flat=True + ) + ), sort_order=issue.sort_order, completed_at=issue.completed_at, archived_at=issue.archived_at, is_draft=issue.is_draft, external_source=issue.external_source, external_id=issue.external_id, - type=issue.type, - last_saved_at=issue.last_saved_at, - assignees=issue.assignees, - labels=issue.labels, - cycle=cycle_issue.cycle if cycle_issue else None, - modules=Module.objects.filter(issue=issue).values_list("id", flat=True), + type=issue.type_id, + cycle=cycle_issue.cycle_id if cycle_issue else None, + modules=list( + Module.objects.filter(issue=issue).values_list("id", flat=True) + ), + properties={}, + meta={}, + last_saved_at=timezone.now(), + owned_by=user, + ) + return True + except Exception as e: + log_exception(e) + return False + + +class IssueDescriptionVersion(ProjectBaseModel): + issue = models.ForeignKey( + "db.Issue", on_delete=models.CASCADE, related_name="description_versions" + ) + description_binary = models.BinaryField(null=True) + description_html = models.TextField(blank=True, default="

") + description_stripped = models.TextField(blank=True, null=True) + description_json = models.JSONField(default=dict, blank=True) + last_saved_at = models.DateTimeField(default=timezone.now) + owned_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + related_name="issue_description_versions", + ) + + class Meta: + verbose_name = "Issue Description Version" + verbose_name_plural = "Issue Description Versions" + db_table = "issue_description_versions" + + @classmethod + def log_issue_description_version(cls, issue, user): + try: + """ + Log the issue description version + """ + cls.objects.create( + issue=issue, + description_binary=issue.description_binary, + description_html=issue.description_html, + description_stripped=issue.description_stripped, + description_json=issue.description, + last_saved_at=timezone.now(), owned_by=user, ) return True diff --git a/apiserver/plane/db/models/sticky.py b/apiserver/plane/db/models/sticky.py new file mode 100644 index 00000000000..5f1c62660ba --- /dev/null +++ b/apiserver/plane/db/models/sticky.py @@ -0,0 +1,32 @@ +# Django imports +from django.conf import settings +from django.db import models + +# Module imports +from .base import BaseModel + + +class Sticky(BaseModel): + name = models.TextField() + + description = models.JSONField(blank=True, default=dict) + description_html = models.TextField(blank=True, default="

") + description_stripped = models.TextField(blank=True, null=True) + description_binary = models.BinaryField(null=True) + + logo_props = models.JSONField(default=dict) + color = models.CharField(max_length=255, blank=True, null=True) + background_color = models.CharField(max_length=255, blank=True, null=True) + + workspace = models.ForeignKey( + "db.Workspace", on_delete=models.CASCADE, related_name="stickies" + ) + owner = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="stickies" + ) + + class Meta: + verbose_name = "Sticky" + verbose_name_plural = "Stickies" + db_table = "stickies" + ordering = ("-created_at",) diff --git a/apiserver/plane/db/models/user.py b/apiserver/plane/db/models/user.py index 34a86a2519e..001889875f5 100644 --- a/apiserver/plane/db/models/user.py +++ b/apiserver/plane/db/models/user.py @@ -26,6 +26,14 @@ def get_default_onboarding(): } +def get_mobile_default_onboarding(): + return { + "profile_complete": False, + "workspace_create": False, + "workspace_join": False, + } + + class User(AbstractBaseUser, PermissionsMixin): id = models.UUIDField( default=uuid.uuid4, unique=True, editable=False, db_index=True, primary_key=True @@ -178,6 +186,10 @@ class Profile(TimeAuditModel): billing_address = models.JSONField(null=True) has_billing_address = models.BooleanField(default=False) company_name = models.CharField(max_length=255, blank=True) + # mobile + is_mobile_onboarded = models.BooleanField(default=False) + mobile_onboarding_step = models.JSONField(default=get_mobile_default_onboarding) + mobile_timezone_auto_set = models.BooleanField(default=False) class Meta: verbose_name = "Profile" From b30869803d45499e7a64b4de05902dfca6a2a583 Mon Sep 17 00:00:00 2001 From: sainath Date: Fri, 13 Dec 2024 15:47:49 +0530 Subject: [PATCH 2/6] chore: removed point in issue version --- .../0087_remove_issueversion_description_and_more.py | 5 ----- apiserver/plane/db/models/issue.py | 4 ---- 2 files changed, 9 deletions(-) diff --git a/apiserver/plane/db/migrations/0087_remove_issueversion_description_and_more.py b/apiserver/plane/db/migrations/0087_remove_issueversion_description_and_more.py index 9ca9fd747f6..564a87f22d2 100644 --- a/apiserver/plane/db/migrations/0087_remove_issueversion_description_and_more.py +++ b/apiserver/plane/db/migrations/0087_remove_issueversion_description_and_more.py @@ -37,11 +37,6 @@ class Migration(migrations.Migration): name='activity', field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='versions', to='db.issueactivity'), ), - migrations.AddField( - model_name='issueversion', - name='point', - field=models.IntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(12)]), - ), migrations.AddField( model_name='profile', name='is_mobile_onboarded', diff --git a/apiserver/plane/db/models/issue.py b/apiserver/plane/db/models/issue.py index e3933cefa2a..8d1a3f320d2 100644 --- a/apiserver/plane/db/models/issue.py +++ b/apiserver/plane/db/models/issue.py @@ -671,9 +671,6 @@ class IssueVersion(ProjectBaseModel): parent = models.UUIDField(blank=True, null=True) state = models.UUIDField(blank=True, null=True) - point = models.IntegerField( - validators=[MinValueValidator(0), MaxValueValidator(12)], null=True, blank=True - ) estimate_point = models.UUIDField(blank=True, null=True) name = models.CharField(max_length=255, verbose_name="Issue Name") priority = models.CharField( @@ -742,7 +739,6 @@ def log_issue_version(cls, issue, user): issue=issue, parent=issue.parent_id, state=issue.state_id, - point=issue.point, estimate_point=issue.estimate_point_id, name=issue.name, priority=issue.priority, From c6f5c9fc34cd58385b31dc9e010ff17f6907d7ce Mon Sep 17 00:00:00 2001 From: sainath Date: Fri, 13 Dec 2024 16:02:24 +0530 Subject: [PATCH 3/6] chore: add imports in init --- apiserver/plane/db/models/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apiserver/plane/db/models/__init__.py b/apiserver/plane/db/models/__init__.py index 4c2d57d80f2..1cbd6276161 100644 --- a/apiserver/plane/db/models/__init__.py +++ b/apiserver/plane/db/models/__init__.py @@ -41,6 +41,8 @@ IssueSequence, IssueSubscriber, IssueVote, + IssueVersion, + IssueDescriptionVersion, ) from .module import Module, ModuleIssue, ModuleLink, ModuleMember, ModuleUserProperties from .notification import EmailNotificationLog, Notification, UserNotificationPreference From 1e77d1daa9ada6d397906ea422a90b590526f968 Mon Sep 17 00:00:00 2001 From: gurusinath Date: Fri, 13 Dec 2024 16:08:09 +0530 Subject: [PATCH 4/6] chore: added sync jobs for issue_version and issue_description_version --- .../bgtasks/issue_description_version_sync.py | 122 +++++++++ apiserver/plane/bgtasks/issue_version_sync.py | 254 ++++++++++++++++++ .../sync_issue_description_version.py | 23 ++ .../management/commands/sync_issue_version.py | 19 ++ 4 files changed, 418 insertions(+) create mode 100644 apiserver/plane/bgtasks/issue_description_version_sync.py create mode 100644 apiserver/plane/bgtasks/issue_version_sync.py create mode 100644 apiserver/plane/db/management/commands/sync_issue_description_version.py create mode 100644 apiserver/plane/db/management/commands/sync_issue_version.py diff --git a/apiserver/plane/bgtasks/issue_description_version_sync.py b/apiserver/plane/bgtasks/issue_description_version_sync.py new file mode 100644 index 00000000000..41712cabee1 --- /dev/null +++ b/apiserver/plane/bgtasks/issue_description_version_sync.py @@ -0,0 +1,122 @@ +# Python imports +from typing import Optional + +# Django imports +from django.utils import timezone +from django.db import transaction + +# Third party imports +from celery import shared_task + +# Module imports +from plane.db.models import Issue, IssueDescriptionVersion, ProjectMember +from plane.utils.exception_logger import log_exception + + +def get_owner_id(issue: Issue) -> Optional[int]: + """Get the owner ID of the issue""" + + if issue.updated_by_id: + return issue.updated_by_id + + if issue.created_by_id: + return issue.created_by_id + + # Find project admin as fallback + project_member = ProjectMember.objects.filter( + project_id=issue.project_id, + role=20, # Admin role + ).first() + + return project_member.member_id if project_member else None + + +@shared_task +def sync_issue_description_version(batch_size=5000, offset=0, countdown=300): + """Task to create IssueDescriptionVersion records for existing Issues in batches""" + try: + with transaction.atomic(): + base_query = Issue.objects + total_issues_count = base_query.count() + + if total_issues_count == 0: + return + + # Calculate batch range + end_offset = min(offset + batch_size, total_issues_count) + + # Fetch issues with related data + issues_batch = ( + base_query.order_by("created_at") + .select_related("workspace", "project") + .only( + "id", + "workspace_id", + "project_id", + "created_by_id", + "updated_by_id", + "description_binary", + "description_html", + "description_stripped", + "description", + )[offset:end_offset] + ) + + if not issues_batch: + return + + version_objects = [] + for issue in issues_batch: + # Validate required fields + if not issue.workspace_id or not issue.project_id: + print(f"Skipping {issue.id} - missing workspace_id or project_id") + continue + + # Determine owned_by_id + owned_by_id = get_owner_id(issue) + if owned_by_id is None: + print(f"Skipping issue {issue.id} - missing owned_by") + continue + + # Create version object + version_objects.append( + IssueDescriptionVersion( + workspace_id=issue.workspace_id, + project_id=issue.project_id, + created_by_id=issue.created_by_id, + updated_by_id=issue.updated_by_id, + owned_by_id=owned_by_id, + last_saved_at=timezone.now(), + issue_id=issue.id, + description_binary=issue.description_binary, + description_html=issue.description_html, + description_stripped=issue.description_stripped, + description_json=issue.description, + ) + ) + + # Bulk create version objects + if version_objects: + IssueDescriptionVersion.objects.bulk_create(version_objects) + + # Schedule next batch if needed + if end_offset < total_issues_count: + sync_issue_description_version.apply_async( + kwargs={ + "batch_size": batch_size, + "offset": end_offset, + "countdown": countdown, + }, + countdown=countdown, + ) + return + except Exception as e: + log_exception(e) + return + + +@shared_task +def schedule_issue_description_version(batch_size=5000, countdown=300): + sync_issue_description_version.delay( + batch_size=int(batch_size), countdown=countdown + ) diff --git a/apiserver/plane/bgtasks/issue_version_sync.py b/apiserver/plane/bgtasks/issue_version_sync.py new file mode 100644 index 00000000000..13233871a57 --- /dev/null +++ b/apiserver/plane/bgtasks/issue_version_sync.py @@ -0,0 +1,254 @@ +# Python imports +import json +from typing import Optional, List, Dict +from uuid import UUID +from itertools import groupby + +# Django imports +from django.utils import timezone +from django.db import transaction + +# Third party imports +from celery import shared_task + +# Module imports +from plane.db.models import ( + Issue, + IssueVersion, + ProjectMember, + CycleIssue, + ModuleIssue, + IssueActivity, + IssueAssignee, + IssueLabel, +) +from plane.utils.exception_logger import log_exception + + +@shared_task +def issue_task(updated_issue, issue_id, user_id): + try: + current_issue = json.loads(updated_issue) if updated_issue else {} + issue = Issue.objects.get(id=issue_id) + + updated_current_issue = {} + for key, value in current_issue.items(): + if getattr(issue, key) != value: + updated_current_issue[key] = value + + if updated_current_issue: + issue_version = ( + IssueVersion.objects.filter(issue_id=issue_id) + .order_by("-last_saved_at") + .first() + ) + + if ( + issue_version + and str(issue_version.owned_by) == str(user_id) + and (timezone.now() - issue_version.last_saved_at).total_seconds() + <= 600 + ): + for key, value in updated_current_issue.items(): + setattr(issue_version, key, value) + issue_version.last_saved_at = timezone.now() + issue_version.save( + update_fields=list(updated_current_issue.keys()) + ["last_saved_at"] + ) + else: + IssueVersion.log_issue_version(issue, user_id) + + return + except Issue.DoesNotExist: + return + except Exception as e: + log_exception(e) + return + + +def get_owner_id(issue: Issue) -> Optional[int]: + """Get the owner ID of the issue""" + + if issue.updated_by_id: + return issue.updated_by_id + + if issue.created_by_id: + return issue.created_by_id + + # Find project admin as fallback + project_member = ProjectMember.objects.filter( + project_id=issue.project_id, + role=20, # Admin role + ).first() + + return project_member.member_id if project_member else None + + +def get_related_data(issue_ids: List[UUID]) -> Dict: + """Get related data for the given issue IDs""" + + cycle_issues = { + ci.issue_id: ci.cycle_id + for ci in CycleIssue.objects.filter(issue_id__in=issue_ids) + } + + # Get assignees with proper grouping + assignee_records = list( + IssueAssignee.objects.filter(issue_id__in=issue_ids) + .values_list("issue_id", "assignee_id") + .order_by("issue_id") + ) + assignees = {} + for issue_id, group in groupby(assignee_records, key=lambda x: x[0]): + assignees[issue_id] = [str(g[1]) for g in group] + + # Get labels with proper grouping + label_records = list( + IssueLabel.objects.filter(issue_id__in=issue_ids) + .values_list("issue_id", "label_id") + .order_by("issue_id") + ) + labels = {} + for issue_id, group in groupby(label_records, key=lambda x: x[0]): + labels[issue_id] = [str(g[1]) for g in group] + + # Get modules with proper grouping + module_records = list( + ModuleIssue.objects.filter(issue_id__in=issue_ids) + .values_list("issue_id", "module_id") + .order_by("issue_id") + ) + modules = {} + for issue_id, group in groupby(module_records, key=lambda x: x[0]): + modules[issue_id] = [str(g[1]) for g in group] + + # Get latest activities + latest_activities = {} + activities = IssueActivity.objects.filter(issue_id__in=issue_ids).order_by( + "issue_id", "-created_at" + ) + for issue_id, activities_group in groupby(activities, key=lambda x: x.issue_id): + first_activity = next(activities_group, None) + if first_activity: + latest_activities[issue_id] = first_activity.id + + return { + "cycle_issues": cycle_issues, + "assignees": assignees, + "labels": labels, + "modules": modules, + "activities": latest_activities, + } + + +def create_issue_version(issue: Issue, related_data: Dict) -> Optional[IssueVersion]: + """Create IssueVersion object from the given issue and related data""" + + try: + if not issue.workspace_id or not issue.project_id: + print(f"Skipping issue {issue.id} - missing workspace_id or project_id") + return None + + owned_by_id = get_owner_id(issue) + if owned_by_id is None: + print(f"Skipping issue {issue.id} - missing owned_by") + return None + + return IssueVersion( + workspace_id=issue.workspace_id, + project_id=issue.project_id, + created_by_id=issue.created_by_id, + updated_by_id=issue.updated_by_id, + owned_by_id=owned_by_id, + last_saved_at=timezone.now(), + activity_id=related_data["activities"].get(issue.id), + properties=getattr(issue, "properties", {}), + meta=getattr(issue, "meta", {}), + issue_id=issue.id, + parent=issue.parent_id, + state=issue.state_id, + estimate_point=issue.estimate_point_id, + name=issue.name, + priority=issue.priority, + start_date=issue.start_date, + target_date=issue.target_date, + assignees=related_data["assignees"].get(issue.id, []), + sequence_id=issue.sequence_id, + labels=related_data["labels"].get(issue.id, []), + sort_order=issue.sort_order, + completed_at=issue.completed_at, + archived_at=issue.archived_at, + is_draft=issue.is_draft, + external_source=issue.external_source, + external_id=issue.external_id, + type=issue.type_id, + cycle=related_data["cycle_issues"].get(issue.id), + modules=related_data["modules"].get(issue.id, []), + ) + except Exception as e: + log_exception(e) + return None + + +@shared_task +def sync_issue_version(batch_size=5000, offset=0, countdown=300): + """Task to create IssueVersion records for existing Issues in batches""" + + try: + with transaction.atomic(): + base_query = Issue.objects + total_issues_count = base_query.count() + + if total_issues_count == 0: + return + + print(f"Offset: {offset}") + print(f"Total Issues: {total_issues_count}") + + end_offset = min(offset + batch_size, total_issues_count) + + # Get issues batch with optimized queries + issues_batch = list( + base_query.order_by("created_at") + .select_related("workspace", "project") + .all()[offset:end_offset] + ) + + if not issues_batch: + return + + # Get all related data in bulk + issue_ids = [issue.id for issue in issues_batch] + related_data = get_related_data(issue_ids) + + issue_versions = [] + for issue in issues_batch: + version = create_issue_version(issue, related_data) + if version: + issue_versions.append(version) + + # Bulk create versions + if issue_versions: + IssueVersion.objects.bulk_create(issue_versions, batch_size=1000) + + # Schedule the next batch if there are more workspaces to process + if end_offset < total_issues_count: + sync_issue_version.apply_async( + kwargs={ + "batch_size": batch_size, + "offset": end_offset, + "countdown": countdown, + }, + countdown=countdown, + ) + + print(f"Processed Issues: {end_offset}") + return + except Exception as e: + log_exception(e) + return + + +@shared_task +def schedule_issue_version(batch_size=5000, countdown=300): + sync_issue_version.delay(batch_size=int(batch_size), countdown=countdown) diff --git a/apiserver/plane/db/management/commands/sync_issue_description_version.py b/apiserver/plane/db/management/commands/sync_issue_description_version.py new file mode 100644 index 00000000000..7ff2fc39147 --- /dev/null +++ b/apiserver/plane/db/management/commands/sync_issue_description_version.py @@ -0,0 +1,23 @@ +# Django imports +from django.core.management.base import BaseCommand + +# Module imports +from plane.bgtasks.issue_description_version_sync import ( + schedule_issue_description_version, +) + + +class Command(BaseCommand): + help = "Creates IssueDescriptionVersion records for existing Issues in batches" + + def handle(self, *args, **options): + batch_size = input("Enter the batch size: ") + batch_countdown = input("Enter the batch countdown: ") + + schedule_issue_description_version.delay( + batch_size=batch_size, countdown=int(batch_countdown) + ) + + self.stdout.write( + self.style.SUCCESS("Successfully created issue description version task") + ) diff --git a/apiserver/plane/db/management/commands/sync_issue_version.py b/apiserver/plane/db/management/commands/sync_issue_version.py new file mode 100644 index 00000000000..2b6632f2618 --- /dev/null +++ b/apiserver/plane/db/management/commands/sync_issue_version.py @@ -0,0 +1,19 @@ +# Django imports +from django.core.management.base import BaseCommand + +# Module imports +from plane.bgtasks.issue_version_sync import schedule_issue_version + + +class Command(BaseCommand): + help = "Creates IssueVersion records for existing Issues in batches" + + def handle(self, *args, **options): + batch_size = input("Enter the batch size: ") + batch_countdown = input("Enter the batch countdown: ") + + schedule_issue_version.delay( + batch_size=batch_size, countdown=int(batch_countdown) + ) + + self.stdout.write(self.style.SUCCESS("Successfully created issue version task")) From 3d8da7b8ff5f4703ee891464b1eb281564cab62a Mon Sep 17 00:00:00 2001 From: gurusinath Date: Fri, 13 Dec 2024 16:13:58 +0530 Subject: [PATCH 5/6] chore: removed logs --- apiserver/plane/bgtasks/issue_version_sync.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/apiserver/plane/bgtasks/issue_version_sync.py b/apiserver/plane/bgtasks/issue_version_sync.py index 13233871a57..919fc0c8ab9 100644 --- a/apiserver/plane/bgtasks/issue_version_sync.py +++ b/apiserver/plane/bgtasks/issue_version_sync.py @@ -202,9 +202,6 @@ def sync_issue_version(batch_size=5000, offset=0, countdown=300): if total_issues_count == 0: return - print(f"Offset: {offset}") - print(f"Total Issues: {total_issues_count}") - end_offset = min(offset + batch_size, total_issues_count) # Get issues batch with optimized queries From 0c6d2e2686fdbbdd2f7396442f84552f9e3c5b28 Mon Sep 17 00:00:00 2001 From: gurusinath Date: Fri, 13 Dec 2024 17:42:17 +0530 Subject: [PATCH 6/6] chore: updated logginh --- .../plane/bgtasks/issue_description_version_sync.py | 7 +++++-- apiserver/plane/bgtasks/issue_version_sync.py | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/apiserver/plane/bgtasks/issue_description_version_sync.py b/apiserver/plane/bgtasks/issue_description_version_sync.py index 41712cabee1..14956cb50cf 100644 --- a/apiserver/plane/bgtasks/issue_description_version_sync.py +++ b/apiserver/plane/bgtasks/issue_description_version_sync.py @@ -1,5 +1,6 @@ # Python imports from typing import Optional +import logging # Django imports from django.utils import timezone @@ -69,13 +70,15 @@ def sync_issue_description_version(batch_size=5000, offset=0, countdown=300): for issue in issues_batch: # Validate required fields if not issue.workspace_id or not issue.project_id: - print(f"Skipping {issue.id} - missing workspace_id or project_id") + logging.warning( + f"Skipping {issue.id} - missing workspace_id or project_id" + ) continue # Determine owned_by_id owned_by_id = get_owner_id(issue) if owned_by_id is None: - print(f"Skipping issue {issue.id} - missing owned_by") + logging.warning(f"Skipping issue {issue.id} - missing owned_by") continue # Create version object diff --git a/apiserver/plane/bgtasks/issue_version_sync.py b/apiserver/plane/bgtasks/issue_version_sync.py index 919fc0c8ab9..698cedf1553 100644 --- a/apiserver/plane/bgtasks/issue_version_sync.py +++ b/apiserver/plane/bgtasks/issue_version_sync.py @@ -3,6 +3,7 @@ from typing import Optional, List, Dict from uuid import UUID from itertools import groupby +import logging # Django imports from django.utils import timezone @@ -146,12 +147,14 @@ def create_issue_version(issue: Issue, related_data: Dict) -> Optional[IssueVers try: if not issue.workspace_id or not issue.project_id: - print(f"Skipping issue {issue.id} - missing workspace_id or project_id") + logging.warning( + f"Skipping issue {issue.id} - missing workspace_id or project_id" + ) return None owned_by_id = get_owner_id(issue) if owned_by_id is None: - print(f"Skipping issue {issue.id} - missing owned_by") + logging.warning(f"Skipping issue {issue.id} - missing owned_by") return None return IssueVersion( @@ -239,7 +242,7 @@ def sync_issue_version(batch_size=5000, offset=0, countdown=300): countdown=countdown, ) - print(f"Processed Issues: {end_offset}") + logging.info(f"Processed Issues: {end_offset}") return except Exception as e: log_exception(e)