From 98c7eac825e139144a62d695def549d36e0ee076 Mon Sep 17 00:00:00 2001 From: sangeethailango Date: Thu, 1 May 2025 17:57:44 +0530 Subject: [PATCH 01/12] chore: comment details of work item --- apiserver/plane/bgtasks/export_task.py | 81 +++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index 33e382f4415..c64c0070d31 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -14,6 +14,10 @@ from django.conf import settings from django.utils import timezone from openpyxl import Workbook +from django.db.models import Value +from django.db.models.functions import Coalesce +from django.db.models import Q +from django.contrib.postgres.aggregates import ArrayAgg # Module imports from plane.db.models import ExporterHistory, Issue @@ -181,6 +185,23 @@ def generate_table_row(issue): dateTimeConverter(issue["updated_at"]), dateTimeConverter(issue["completed_at"]), dateTimeConverter(issue["archived_at"]), + [ + { + "comment": comment, + "created_at": dateConverter(created_at), + "created_by": ( + f"{created_by_first_name} {created_by_last_name}" + if created_by_first_name and created_by_last_name + else "" + ), + } + for comment, created_at, created_by_first_name, created_by_last_name in zip( + issue["comment_stripped"], + issue["comment_created_at"], + issue["comment_created_by_first_name"], + issue["comment_created_by_last_name"], + ) + ], ] @@ -215,6 +236,23 @@ def generate_json_row(issue): "Updated At": dateTimeConverter(issue["updated_at"]), "Completed At": dateTimeConverter(issue["completed_at"]), "Archived At": dateTimeConverter(issue["archived_at"]), + "Comments": [ + { + "comment": comment, + "created_at": dateConverter(created_at), + "created_by": ( + f"{created_by_first_name} {created_by_last_name}" + if created_by_first_name and created_by_last_name + else "" + ), + } + for comment, created_at, created_by_first_name, created_by_last_name in zip( + issue["comment_stripped"], + issue["comment_created_at"], + issue["comment_created_by_first_name"], + issue["comment_created_by_last_name"], + ) + ], } @@ -317,7 +355,42 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s ) .select_related("project", "workspace", "state", "parent", "created_by") .prefetch_related( - "assignees", "labels", "issue_cycle__cycle", "issue_module__module" + "assignees", + "labels", + "issue_cycle__cycle", + "issue_module__module", + "issue_comments", + ) + .annotate( + comment_stripped=Coalesce( + ArrayAgg( + "issue_comments__comment_stripped", + filter=Q(issue_comments__comment_stripped__isnull=False), + order_by=["-issue_comments__created_at"], + ), + Value([]), + ), + comment_created_at=Coalesce( + ArrayAgg( + "issue_comments__created_at", + order_by=["-issue_comments__created_at"], + ), + Value([]), + ), + comment_created_by_first_name=Coalesce( + ArrayAgg( + "issue_comments__created_by__first_name", + order_by=["-issue_comments__created_at"], + ), + Value([]), + ), + comment_created_by_last_name=Coalesce( + ArrayAgg( + "issue_comments__created_by__last_name", + order_by=["-issue_comments__created_at"], + ), + Value([]), + ), ) .values( "id", @@ -346,11 +419,16 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "assignees__first_name", "assignees__last_name", "labels__name", + "comment_stripped", + "comment_created_at", + "comment_created_by_first_name", + "comment_created_by_last_name", ) ) .order_by("project__identifier", "sequence_id") .distinct() ) + # CSV header header = [ "ID", @@ -374,6 +452,7 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "Updated At", "Completed At", "Archived At", + "Comments", ] EXPORTER_MAPPER = { From 45238959123f0042543d28b16b5745234606fd2e Mon Sep 17 00:00:00 2001 From: sangeethailango Date: Fri, 2 May 2025 22:36:17 +0530 Subject: [PATCH 02/12] chore: attachment count and attachment name --- apiserver/plane/bgtasks/export_task.py | 33 +++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index c64c0070d31..99092ead9e3 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -10,6 +10,7 @@ # Third party imports from celery import shared_task + # Django imports from django.conf import settings from django.utils import timezone @@ -18,9 +19,12 @@ from django.db.models.functions import Coalesce from django.db.models import Q from django.contrib.postgres.aggregates import ArrayAgg +from django.db.models import OuterRef, Func, F +from django.db import models +from django.contrib.postgres.fields import ArrayField # Module imports -from plane.db.models import ExporterHistory, Issue +from plane.db.models import ExporterHistory, Issue, FileAsset from plane.utils.exception_logger import log_exception @@ -202,6 +206,8 @@ def generate_table_row(issue): issue["comment_created_by_last_name"], ) ], + issue["attachment_count"], + issue["attachment_links"], ] @@ -392,6 +398,27 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s Value([]), ), ) + .annotate( + attachment_count=FileAsset.objects.filter( + issue_id=OuterRef("id"), + entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT, + ) + .order_by() + .annotate(count=Func(F("id"), function="Count")) + .values("count") + ) + .annotate( + attachment_links=Coalesce( + ArrayAgg( + "assets__asset", + filter=Q( + assets__entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT + ), + order_by=["-assets__created_at"], + ), + Value([]), + ) + ) .values( "id", "project__identifier", @@ -423,6 +450,8 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "comment_created_at", "comment_created_by_first_name", "comment_created_by_last_name", + "attachment_count", + "attachment_links", ) ) .order_by("project__identifier", "sequence_id") @@ -453,6 +482,8 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "Completed At", "Archived At", "Comments", + "Attachment Count", + "Attachment Links", ] EXPORTER_MAPPER = { From f066059d8986dc100699962639e64e6955865b9a Mon Sep 17 00:00:00 2001 From: sangeethailango Date: Fri, 2 May 2025 23:06:05 +0530 Subject: [PATCH 03/12] chore: issue link and subscriber count --- apiserver/plane/bgtasks/export_task.py | 38 +++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index 99092ead9e3..d5294f0ecb3 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -19,9 +19,7 @@ from django.db.models.functions import Coalesce from django.db.models import Q from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models import OuterRef, Func, F -from django.db import models -from django.contrib.postgres.fields import ArrayField +from django.db.models import OuterRef, Func, F, Count # Module imports from plane.db.models import ExporterHistory, Issue, FileAsset @@ -208,6 +206,8 @@ def generate_table_row(issue): ], issue["attachment_count"], issue["attachment_links"], + issue["issue_link_url"], + issue["issue_subscriber_count"], ] @@ -359,13 +359,21 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s project__project_projectmember__is_active=True, project__archived_at__isnull=True, ) - .select_related("project", "workspace", "state", "parent", "created_by") + .select_related( + "project", + "workspace", + "state", + "parent", + "created_by", + "issue_link", + ) .prefetch_related( "assignees", "labels", "issue_cycle__cycle", "issue_module__module", "issue_comments", + "issue_subscribers", ) .annotate( comment_stripped=Coalesce( @@ -419,6 +427,23 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s Value([]), ) ) + .annotate( + issue_link_url=Coalesce( + ArrayAgg( + "issue_link__url", + filter=Q(issue_link__url__isnull=False), + distinct=True, + ), + Value([]), + ) + ) + .annotate( + issue_subscriber_count=Count( + "issue_subscribers", + filter=Q(issue_subscribers__subscriber__isnull=False), + distinct=True, + ) + ) .values( "id", "project__identifier", @@ -452,6 +477,9 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "comment_created_by_last_name", "attachment_count", "attachment_links", + "issue_link_url", + "issue_subscriber_count", + "estimate_point__estimate__name", ) ) .order_by("project__identifier", "sequence_id") @@ -484,6 +512,8 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "Comments", "Attachment Count", "Attachment Links", + "Link", + "Subscriber Count", ] EXPORTER_MAPPER = { From 51aeecff5cf274d24b67dcd6df19a27bba8ac486 Mon Sep 17 00:00:00 2001 From: sangeethailango Date: Mon, 5 May 2025 15:06:37 +0530 Subject: [PATCH 04/12] chore: list of assignees --- apiserver/plane/bgtasks/export_task.py | 68 +++++++++++--------------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index d5294f0ecb3..85c911a197b 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -19,11 +19,13 @@ from django.db.models.functions import Coalesce from django.db.models import Q from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models import OuterRef, Func, F, Count +from django.db.models import OuterRef, Func, F +from django.db.models.functions import Concat # Module imports from plane.db.models import ExporterHistory, Issue, FileAsset from plane.utils.exception_logger import log_exception +# from plane.app.serializers import IssueAttachmentLiteSerializer def dateTimeConverter(time): @@ -171,11 +173,6 @@ def generate_table_row(issue): if issue["created_by__first_name"] and issue["created_by__last_name"] else "" ), - ( - f"{issue['assignees__first_name']} {issue['assignees__last_name']}" - if issue["assignees__first_name"] and issue["assignees__last_name"] - else "" - ), issue["labels__name"] if issue["labels__name"] else "", issue["issue_cycle__cycle__name"], dateConverter(issue["issue_cycle__cycle__start_date"]), @@ -204,10 +201,10 @@ def generate_table_row(issue): issue["comment_created_by_last_name"], ) ], - issue["attachment_count"], - issue["attachment_links"], - issue["issue_link_url"], - issue["issue_subscriber_count"], + issue["attachment_count"] if issue["attachment_count"] else "", + issue["issue_link_url"] if issue["issue_link_url"] else "", + issue["estimate_point__estimate__name"], + issue["assignee"] if issue["assignee"] else "", ] @@ -368,12 +365,7 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "issue_link", ) .prefetch_related( - "assignees", - "labels", - "issue_cycle__cycle", - "issue_module__module", - "issue_comments", - "issue_subscribers", + "labels", "issue_cycle__cycle", "issue_module__module" ) .annotate( comment_stripped=Coalesce( @@ -381,6 +373,7 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "issue_comments__comment_stripped", filter=Q(issue_comments__comment_stripped__isnull=False), order_by=["-issue_comments__created_at"], + distinct=True, ), Value([]), ), @@ -388,6 +381,7 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s ArrayAgg( "issue_comments__created_at", order_by=["-issue_comments__created_at"], + distinct=True, ), Value([]), ), @@ -395,6 +389,7 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s ArrayAgg( "issue_comments__created_by__first_name", order_by=["-issue_comments__created_at"], + distinct=True, ), Value([]), ), @@ -402,6 +397,7 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s ArrayAgg( "issue_comments__created_by__last_name", order_by=["-issue_comments__created_at"], + distinct=True, ), Value([]), ), @@ -415,18 +411,6 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s .annotate(count=Func(F("id"), function="Count")) .values("count") ) - .annotate( - attachment_links=Coalesce( - ArrayAgg( - "assets__asset", - filter=Q( - assets__entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT - ), - order_by=["-assets__created_at"], - ), - Value([]), - ) - ) .annotate( issue_link_url=Coalesce( ArrayAgg( @@ -438,10 +422,20 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s ) ) .annotate( - issue_subscriber_count=Count( - "issue_subscribers", - filter=Q(issue_subscribers__subscriber__isnull=False), - distinct=True, + assignee=Coalesce( + ArrayAgg( + Concat( + F("assignees__first_name"), + Value(" "), + F("assignees__last_name"), + ), + filter=Q( + assignees__first_name__isnull=False, + assignees__last_name__isnull=False, + ), + distinct=True, + ), + Value([]), ) ) .values( @@ -468,18 +462,15 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "issue_module__module__target_date", "created_by__first_name", "created_by__last_name", - "assignees__first_name", - "assignees__last_name", "labels__name", "comment_stripped", "comment_created_at", "comment_created_by_first_name", "comment_created_by_last_name", "attachment_count", - "attachment_links", "issue_link_url", - "issue_subscriber_count", "estimate_point__estimate__name", + "assignee", ) ) .order_by("project__identifier", "sequence_id") @@ -497,7 +488,6 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "Target Date", "Priority", "Created By", - "Assignee", "Labels", "Cycle Name", "Cycle Start Date", @@ -511,9 +501,9 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "Archived At", "Comments", "Attachment Count", - "Attachment Links", "Link", - "Subscriber Count", + "Estimate", + "Assignees", ] EXPORTER_MAPPER = { From 427276ffb12f33ccda89e674235555bccdb1f647 Mon Sep 17 00:00:00 2001 From: sangeethailango Date: Mon, 5 May 2025 16:57:08 +0530 Subject: [PATCH 05/12] chore: asset_url as attachment_links --- apiserver/plane/bgtasks/export_task.py | 72 ++++++++++++++++++++------ 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index 85c911a197b..1729ff6509e 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -19,13 +19,14 @@ from django.db.models.functions import Coalesce from django.db.models import Q from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models import OuterRef, Func, F +from django.db.models import OuterRef, Func, F, Count from django.db.models.functions import Concat +from django.db import models + # Module imports from plane.db.models import ExporterHistory, Issue, FileAsset from plane.utils.exception_logger import log_exception -# from plane.app.serializers import IssueAttachmentLiteSerializer def dateTimeConverter(time): @@ -201,10 +202,12 @@ def generate_table_row(issue): issue["comment_created_by_last_name"], ) ], - issue["attachment_count"] if issue["attachment_count"] else "", - issue["issue_link_url"] if issue["issue_link_url"] else "", issue["estimate_point__estimate__name"], + issue["issue_link_url"] if issue["issue_link_url"] else "", issue["assignee"] if issue["assignee"] else "", + issue["subscribers_count"] if issue["subscribers_count"] else "", + issue["attachment_count"] if issue["attachment_count"] else "", + issue["attachment_links"] if issue["attachment_links"] else "", ] @@ -402,15 +405,6 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s Value([]), ), ) - .annotate( - attachment_count=FileAsset.objects.filter( - issue_id=OuterRef("id"), - entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT, - ) - .order_by() - .annotate(count=Func(F("id"), function="Count")) - .values("count") - ) .annotate( issue_link_url=Coalesce( ArrayAgg( @@ -438,6 +432,46 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s Value([]), ) ) + .annotate( + subscribers_count=Count( + "issue_subscribers", + filter=Q(issue_subscribers__deleted_at__isnull=True), + distinct=True, + ) + ) + .annotate( + attachment_count=FileAsset.objects.filter( + issue_id=OuterRef("id"), + entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT, + ) + .order_by() + .annotate(count=Func(F("id"), function="Count")) + .values("count") + ) + .annotate( + attachment_links=Coalesce( + ArrayAgg( + Concat( + Value("/api/assets/v2/workspaces/"), + F("workspace_id"), + Value("/projects/"), + F("project_id"), + Value("/issues/"), + F("id"), + Value("/attachments/"), + F("assets__id"), + Value("/"), + output_field=models.CharField(), + ), + filter=Q( + assets__entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT, + assets__id__isnull=False, + ), + distinct=True, + ), + Value([]), + ) + ) .values( "id", "project__identifier", @@ -467,10 +501,12 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "comment_created_at", "comment_created_by_first_name", "comment_created_by_last_name", - "attachment_count", - "issue_link_url", "estimate_point__estimate__name", + "issue_link_url", "assignee", + "subscribers_count", + "attachment_count", + "attachment_links", ) ) .order_by("project__identifier", "sequence_id") @@ -500,10 +536,12 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "Completed At", "Archived At", "Comments", - "Attachment Count", - "Link", "Estimate", + "Link", "Assignees", + "Subscribers Count", + "Attachment Count", + "Attachment Links", ] EXPORTER_MAPPER = { From 114df85b46812c142f5fbccd33bdb2691be90df9 Mon Sep 17 00:00:00 2001 From: sangeethailango Date: Tue, 6 May 2025 13:42:14 +0530 Subject: [PATCH 06/12] chore: code refactor --- apiserver/plane/bgtasks/export_task.py | 311 +++++++++---------------- 1 file changed, 109 insertions(+), 202 deletions(-) diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index 1729ff6509e..d28051cbf2b 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -58,6 +58,7 @@ def create_xlsx_file(data): workbook = Workbook() sheet = workbook.active + print(data, "Data") for row in data: sheet.append(row) @@ -161,50 +162,28 @@ def upload_to_s3(zip_file, workspace_id, token_id, slug): def generate_table_row(issue): return [ - f"""{issue["project__identifier"]}-{issue["sequence_id"]}""", - issue["project__name"], + f"""{issue["project_identifier"]}-{issue["sequence_id"]}""", + issue["project_name"], issue["name"], - issue["description_stripped"], - issue["state__name"], + issue["description"], + issue["state_name"], dateConverter(issue["start_date"]), dateConverter(issue["target_date"]), issue["priority"], - ( - f"{issue['created_by__first_name']} {issue['created_by__last_name']}" - if issue["created_by__first_name"] and issue["created_by__last_name"] - else "" - ), - issue["labels__name"] if issue["labels__name"] else "", - issue["issue_cycle__cycle__name"], - dateConverter(issue["issue_cycle__cycle__start_date"]), - dateConverter(issue["issue_cycle__cycle__end_date"]), - issue["issue_module__module__name"], - dateConverter(issue["issue_module__module__start_date"]), - dateConverter(issue["issue_module__module__target_date"]), + issue["created_by"], + issue["labels"] if issue["labels"] else "", + issue.get("cycle_name", ""), + issue.get("cycle_start_date", ""), + issue.get("cycle_end_date", ""), + issue.get("module_name", ""), dateTimeConverter(issue["created_at"]), dateTimeConverter(issue["updated_at"]), dateTimeConverter(issue["completed_at"]), dateTimeConverter(issue["archived_at"]), - [ - { - "comment": comment, - "created_at": dateConverter(created_at), - "created_by": ( - f"{created_by_first_name} {created_by_last_name}" - if created_by_first_name and created_by_last_name - else "" - ), - } - for comment, created_at, created_by_first_name, created_by_last_name in zip( - issue["comment_stripped"], - issue["comment_created_at"], - issue["comment_created_by_first_name"], - issue["comment_created_by_last_name"], - ) - ], - issue["estimate_point__estimate__name"], - issue["issue_link_url"] if issue["issue_link_url"] else "", - issue["assignee"] if issue["assignee"] else "", + issue["comments"], + issue["estimate"] if issue["estimate"] else "", + issue["link"] if issue["link"] else "", + issue["assignees"] if issue["assignees"] else "", issue["subscribers_count"] if issue["subscribers_count"] else "", issue["attachment_count"] if issue["attachment_count"] else "", issue["attachment_links"] if issue["attachment_links"] else "", @@ -212,6 +191,7 @@ def generate_table_row(issue): def generate_json_row(issue): + print("generate_json_row", issue) return { "ID": f"""{issue["project__identifier"]}-{issue["sequence_id"]}""", "Project": issue["project__name"], @@ -236,8 +216,6 @@ def generate_json_row(issue): "Cycle Start Date": dateConverter(issue["issue_cycle__cycle__start_date"]), "Cycle End Date": dateConverter(issue["issue_cycle__cycle__end_date"]), "Module Name": issue["issue_module__module__name"], - "Module Start Date": dateConverter(issue["issue_module__module__start_date"]), - "Module Target Date": dateConverter(issue["issue_module__module__target_date"]), "Created At": dateTimeConverter(issue["created_at"]), "Updated At": dateTimeConverter(issue["updated_at"]), "Completed At": dateTimeConverter(issue["completed_at"]), @@ -328,7 +306,9 @@ def generate_csv(header, project_id, issues, files): def generate_json(header, project_id, issues, files): rows = [] for issue in issues: + print(issue, "Issue") row = generate_json_row(issue) + print(row, "Row") update_json_row(rows, row) json_file = create_json_file(rows) files.append((f"{project_id}.json", json_file)) @@ -351,169 +331,98 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s exporter_instance.save(update_fields=["status"]) workspace_issues = ( - ( - Issue.objects.filter( - workspace__id=workspace_id, - project_id__in=project_ids, - project__project_projectmember__member=exporter_instance.initiated_by_id, - project__project_projectmember__is_active=True, - project__archived_at__isnull=True, - ) - .select_related( - "project", - "workspace", - "state", - "parent", - "created_by", - "issue_link", - ) - .prefetch_related( - "labels", "issue_cycle__cycle", "issue_module__module" - ) - .annotate( - comment_stripped=Coalesce( - ArrayAgg( - "issue_comments__comment_stripped", - filter=Q(issue_comments__comment_stripped__isnull=False), - order_by=["-issue_comments__created_at"], - distinct=True, - ), - Value([]), - ), - comment_created_at=Coalesce( - ArrayAgg( - "issue_comments__created_at", - order_by=["-issue_comments__created_at"], - distinct=True, - ), - Value([]), - ), - comment_created_by_first_name=Coalesce( - ArrayAgg( - "issue_comments__created_by__first_name", - order_by=["-issue_comments__created_at"], - distinct=True, - ), - Value([]), - ), - comment_created_by_last_name=Coalesce( - ArrayAgg( - "issue_comments__created_by__last_name", - order_by=["-issue_comments__created_at"], - distinct=True, - ), - Value([]), - ), - ) - .annotate( - issue_link_url=Coalesce( - ArrayAgg( - "issue_link__url", - filter=Q(issue_link__url__isnull=False), - distinct=True, - ), - Value([]), - ) - ) - .annotate( - assignee=Coalesce( - ArrayAgg( - Concat( - F("assignees__first_name"), - Value(" "), - F("assignees__last_name"), - ), - filter=Q( - assignees__first_name__isnull=False, - assignees__last_name__isnull=False, - ), - distinct=True, - ), - Value([]), - ) - ) - .annotate( - subscribers_count=Count( - "issue_subscribers", - filter=Q(issue_subscribers__deleted_at__isnull=True), - distinct=True, - ) - ) - .annotate( - attachment_count=FileAsset.objects.filter( - issue_id=OuterRef("id"), - entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT, - ) - .order_by() - .annotate(count=Func(F("id"), function="Count")) - .values("count") - ) - .annotate( - attachment_links=Coalesce( - ArrayAgg( - Concat( - Value("/api/assets/v2/workspaces/"), - F("workspace_id"), - Value("/projects/"), - F("project_id"), - Value("/issues/"), - F("id"), - Value("/attachments/"), - F("assets__id"), - Value("/"), - output_field=models.CharField(), - ), - filter=Q( - assets__entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT, - assets__id__isnull=False, - ), - distinct=True, - ), - Value([]), - ) - ) - .values( - "id", - "project__identifier", - "project__name", - "project__id", - "sequence_id", - "name", - "description_stripped", - "priority", - "start_date", - "target_date", - "state__name", - "created_at", - "updated_at", - "completed_at", - "archived_at", - "issue_cycle__cycle__name", - "issue_cycle__cycle__start_date", - "issue_cycle__cycle__end_date", - "issue_module__module__name", - "issue_module__module__start_date", - "issue_module__module__target_date", - "created_by__first_name", - "created_by__last_name", - "labels__name", - "comment_stripped", - "comment_created_at", - "comment_created_by_first_name", - "comment_created_by_last_name", - "estimate_point__estimate__name", - "issue_link_url", - "assignee", - "subscribers_count", - "attachment_count", - "attachment_links", - ) + Issue.objects.filter( + workspace__id=workspace_id, + project_id__in=project_ids, + project__project_projectmember__member=exporter_instance.initiated_by_id, + project__project_projectmember__is_active=True, + project__archived_at__isnull=True, + ) + .select_related( + "project", + "workspace", + "state", + "parent", + "created_by", + "estimate_point", + ) + .prefetch_related( + "labels", + "issue_cycle__cycle", + "issue_module__module", + "issue_comments", + "assignees", + "issue_subscribers", + "issue_link", ) - .order_by("project__identifier", "sequence_id") - .distinct() ) - # CSV header + issues_data = [] + + for issue in workspace_issues: + issue_data = { + "id": issue.id, + "project_identifier": issue.project.identifier, + "project_name": issue.project.name, + "project_id": issue.project.id, + "sequence_id": issue.sequence_id, + "name": issue.name, + "description": issue.description_stripped, + "priority": issue.priority, + "start_date": issue.start_date, + "target_date": issue.target_date, + "state_name": issue.state.name if issue.state else None, + "created_at": issue.created_at, + "updated_at": issue.updated_at, + "completed_at": issue.completed_at, + "archived_at": issue.archived_at, + "module_name": [ + module.module.name for module in issue.issue_module.all() + ], + "created_by": f"{issue.created_by.first_name} {issue.created_by.last_name}", + "labels": [label.name for label in issue.labels.all().distinct()], + "comments": [ + { + "comment": comment.comment_stripped, + "created_at": dateConverter(comment.created_at), + "created_by": f"{comment.created_by.first_name} {comment.created_by.last_name}", + } + for comment in issue.issue_comments.all().distinct() + ], + "estimate": issue.estimate_point.estimate.name + if issue.estimate_point + else "", + "link": [link.url for link in issue.issue_link.all().distinct()], + "assignees": [ + f"{assignee.first_name} {assignee.last_name}" + for assignee in issue.assignees.all().distinct() + ], + "subscribers_count": issue.issue_subscribers.count(), + "attachment_count": FileAsset.objects.filter( + issue_id=issue.id, + entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT, + ).count(), + "attachment_links": [ + f"/api/assets/v2/workspaces/{issue.workspace.slug}/projects/{issue.project_id}/issues/{issue.id}/attachments/{asset.id}/" + for asset in FileAsset.objects.filter( + issue_id=issue.id, + entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT, + ) + ], + } + + # Get prefetched cycles and modules + cycles = list(issue.issue_cycle.all()) + + # Update cycle data + for cycle in cycles: + issue_data["cycle_name"] = cycle.cycle.name + issue_data["cycle_start_date"] = dateConverter(cycle.cycle.start_date) + issue_data["cycle_end_date"] = dateConverter(cycle.cycle.end_date) + + issues_data.append(issue_data) + + # CSV header header = [ "ID", "Project", @@ -529,8 +438,6 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "Cycle Start Date", "Cycle End Date", "Module Name", - "Module Start Date", - "Module Target Date", "Created At", "Updated At", "Completed At", @@ -553,7 +460,7 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s files = [] if multiple: for project_id in project_ids: - issues = workspace_issues.filter(project__id=project_id) + issues = issues_data.filter(project__id=project_id) exporter = EXPORTER_MAPPER.get(provider) if exporter is not None: exporter(header, project_id, issues, files) @@ -561,7 +468,7 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s else: exporter = EXPORTER_MAPPER.get(provider) if exporter is not None: - exporter(header, workspace_id, workspace_issues, files) + exporter(header, workspace_id, issues_data, files) zip_buffer = create_zip_file(files) upload_to_s3(zip_buffer, workspace_id, token_id, slug) From e25eea0f99c15aa632f28e44f62fecb783bb16be Mon Sep 17 00:00:00 2001 From: sangeethailango Date: Tue, 6 May 2025 14:44:39 +0530 Subject: [PATCH 07/12] fix: cannot export Excel --- apiserver/plane/bgtasks/export_task.py | 103 ++++++++++++------------- 1 file changed, 51 insertions(+), 52 deletions(-) diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index d28051cbf2b..edf498dd12b 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -58,7 +58,6 @@ def create_xlsx_file(data): workbook = Workbook() sheet = workbook.active - print(data, "Data") for row in data: sheet.append(row) @@ -171,72 +170,58 @@ def generate_table_row(issue): dateConverter(issue["target_date"]), issue["priority"], issue["created_by"], - issue["labels"] if issue["labels"] else "", + ", ".join(issue["labels"]) if issue["labels"] else "", issue.get("cycle_name", ""), issue.get("cycle_start_date", ""), issue.get("cycle_end_date", ""), - issue.get("module_name", ""), + ", ".join(issue.get("module_name", "")) if issue.get("module_name") else "", dateTimeConverter(issue["created_at"]), dateTimeConverter(issue["updated_at"]), dateTimeConverter(issue["completed_at"]), dateTimeConverter(issue["archived_at"]), - issue["comments"], + ", ".join( + [ + f"{comment['comment']} ({comment['created_at']} by {comment['created_by']})" + for comment in issue["comments"] + ] + ) + if issue["comments"] + else "", issue["estimate"] if issue["estimate"] else "", - issue["link"] if issue["link"] else "", - issue["assignees"] if issue["assignees"] else "", - issue["subscribers_count"] if issue["subscribers_count"] else "", + ", ".join(issue["link"]) if issue["link"] else "", + ", ".join(issue["assignees"]) if issue["assignees"] else "", issue["attachment_count"] if issue["attachment_count"] else "", - issue["attachment_links"] if issue["attachment_links"] else "", + ", ".join(issue["attachment_links"]) if issue["attachment_links"] else "", ] def generate_json_row(issue): - print("generate_json_row", issue) return { - "ID": f"""{issue["project__identifier"]}-{issue["sequence_id"]}""", - "Project": issue["project__name"], + "ID": f"""{issue["project_identifier"]}-{issue["sequence_id"]}""", + "Project": issue["project_name"], "Name": issue["name"], - "Description": issue["description_stripped"], - "State": issue["state__name"], + "Description": issue["description"], + "State": issue["state_name"], "Start Date": dateConverter(issue["start_date"]), "Target Date": dateConverter(issue["target_date"]), "Priority": issue["priority"], - "Created By": ( - f"{issue['created_by__first_name']} {issue['created_by__last_name']}" - if issue["created_by__first_name"] and issue["created_by__last_name"] - else "" - ), - "Assignee": ( - f"{issue['assignees__first_name']} {issue['assignees__last_name']}" - if issue["assignees__first_name"] and issue["assignees__last_name"] - else "" - ), - "Labels": issue["labels__name"] if issue["labels__name"] else "", - "Cycle Name": issue["issue_cycle__cycle__name"], - "Cycle Start Date": dateConverter(issue["issue_cycle__cycle__start_date"]), - "Cycle End Date": dateConverter(issue["issue_cycle__cycle__end_date"]), - "Module Name": issue["issue_module__module__name"], + "Created By": (f"{issue['created_by']}" if issue["created_by"] else ""), + "Assignee": issue["assignees"], + "Labels": issue["labels"], + "Cycle Name": issue["cycle_name"], + "Cycle Start Date": issue["cycle_start_date"], + "Cycle End Date": issue["cycle_end_date"], + "Module Name": issue["module_name"], "Created At": dateTimeConverter(issue["created_at"]), "Updated At": dateTimeConverter(issue["updated_at"]), "Completed At": dateTimeConverter(issue["completed_at"]), "Archived At": dateTimeConverter(issue["archived_at"]), - "Comments": [ - { - "comment": comment, - "created_at": dateConverter(created_at), - "created_by": ( - f"{created_by_first_name} {created_by_last_name}" - if created_by_first_name and created_by_last_name - else "" - ), - } - for comment, created_at, created_by_first_name, created_by_last_name in zip( - issue["comment_stripped"], - issue["comment_created_at"], - issue["comment_created_by_first_name"], - issue["comment_created_by_last_name"], - ) - ], + "Comments": issue["comments"], + "Estimate": issue["estimate"], + "Link": issue["link"], + "Subscribers Count": issue["subscribers_count"], + "Attachment Count": issue["attachment_count"], + "Attachment Links": issue["attachment_links"], } @@ -306,9 +291,7 @@ def generate_csv(header, project_id, issues, files): def generate_json(header, project_id, issues, files): rows = [] for issue in issues: - print(issue, "Issue") row = generate_json_row(issue) - print(row, "Row") update_json_row(rows, row) json_file = create_json_file(rows) files.append((f"{project_id}.json", json_file)) @@ -317,7 +300,9 @@ def generate_json(header, project_id, issues, files): def generate_xlsx(header, project_id, issues, files): rows = [header] for issue in issues: + print(issue, "Issue from xlsx") row = generate_table_row(issue) + print(row, "Row") update_table_row(rows, row) xlsx_file = create_xlsx_file(rows) files.append((f"{project_id}.xlsx", xlsx_file)) @@ -378,9 +363,15 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "archived_at": issue.archived_at, "module_name": [ module.module.name for module in issue.issue_module.all() - ], + ] + if issue.issue_module.exists() + else None, "created_by": f"{issue.created_by.first_name} {issue.created_by.last_name}", - "labels": [label.name for label in issue.labels.all().distinct()], + "labels": [ + label.name + for label in issue.labels.all().distinct() + if issue.labels.exists() + ], "comments": [ { "comment": comment.comment_stripped, @@ -388,15 +379,23 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "created_by": f"{comment.created_by.first_name} {comment.created_by.last_name}", } for comment in issue.issue_comments.all().distinct() - ], + ] + if issue.issue_comments.exists() + else None, "estimate": issue.estimate_point.estimate.name if issue.estimate_point else "", - "link": [link.url for link in issue.issue_link.all().distinct()], + "link": [ + link.url + for link in issue.issue_link.all().distinct() + if issue.issue_link.exists() + ], "assignees": [ f"{assignee.first_name} {assignee.last_name}" for assignee in issue.assignees.all().distinct() - ], + ] + if issue.assignees.exists() + else None, "subscribers_count": issue.issue_subscribers.count(), "attachment_count": FileAsset.objects.filter( issue_id=issue.id, From cbc3fc1f51ebfbf3afc9093fd6884b80a43cb2d0 Mon Sep 17 00:00:00 2001 From: sangeethailango Date: Tue, 6 May 2025 14:51:31 +0530 Subject: [PATCH 08/12] chore: remove print statements --- apiserver/plane/bgtasks/export_task.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index edf498dd12b..8a3469491a0 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -300,9 +300,8 @@ def generate_json(header, project_id, issues, files): def generate_xlsx(header, project_id, issues, files): rows = [header] for issue in issues: - print(issue, "Issue from xlsx") row = generate_table_row(issue) - print(row, "Row") + update_table_row(rows, row) xlsx_file = create_xlsx_file(rows) files.append((f"{project_id}.xlsx", xlsx_file)) From 66d7c061d52095cb5965b7f7e32d4c66f27c24c8 Mon Sep 17 00:00:00 2001 From: sangeethailango Date: Tue, 6 May 2025 15:50:42 +0530 Subject: [PATCH 09/12] fix: filtering in list --- apiserver/plane/bgtasks/export_task.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index 8a3469491a0..a5dafc9cdb2 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -458,7 +458,12 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s files = [] if multiple: for project_id in project_ids: - issues = issues_data.filter(project__id=project_id) + issues = [ + issue + for issue in issues_data + if str(issue["project_id"]) == str(project_id) + ] + exporter = EXPORTER_MAPPER.get(provider) if exporter is not None: exporter(header, project_id, issues, files) From a695a8baa0b797cf245f387c71d6fd471961bbc8 Mon Sep 17 00:00:00 2001 From: sangeethailango Date: Tue, 6 May 2025 17:15:42 +0530 Subject: [PATCH 10/12] chore: optimize attachment_count and attachment_link query --- apiserver/plane/bgtasks/export_task.py | 33 +++++++++++++------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index a5dafc9cdb2..6d12e3b3ca9 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -15,14 +15,9 @@ from django.conf import settings from django.utils import timezone from openpyxl import Workbook -from django.db.models import Value -from django.db.models.functions import Coalesce -from django.db.models import Q -from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models import OuterRef, Func, F, Count -from django.db.models.functions import Concat -from django.db import models +from django.db.models import F +from collections import defaultdict # Module imports from plane.db.models import ExporterHistory, Issue, FileAsset @@ -190,6 +185,7 @@ def generate_table_row(issue): issue["estimate"] if issue["estimate"] else "", ", ".join(issue["link"]) if issue["link"] else "", ", ".join(issue["assignees"]) if issue["assignees"] else "", + issue["subscribers_count"] if issue["subscribers_count"] else "", issue["attachment_count"] if issue["attachment_count"] else "", ", ".join(issue["attachment_links"]) if issue["attachment_links"] else "", ] @@ -283,6 +279,7 @@ def generate_csv(header, project_id, issues, files): rows = [header] for issue in issues: row = generate_table_row(issue) + update_table_row(rows, row) csv_file = create_csv_file(rows) files.append((f"{project_id}.csv", csv_file)) @@ -341,9 +338,19 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s ) ) + file_assets = FileAsset.objects.filter( + issue_id__in=workspace_issues.values_list("id", flat=True) + ).annotate(work_item_id=F("issue_id"), asset_id=F("id")) + + attachment_dict = defaultdict(list) + for asset in file_assets: + attachment_dict[asset.work_item_id].append(asset.asset_id) + issues_data = [] for issue in workspace_issues: + attachments = attachment_dict.get(issue.id, []) + issue_data = { "id": issue.id, "project_identifier": issue.project.identifier, @@ -396,16 +403,10 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s if issue.assignees.exists() else None, "subscribers_count": issue.issue_subscribers.count(), - "attachment_count": FileAsset.objects.filter( - issue_id=issue.id, - entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT, - ).count(), + "attachment_count": len(attachments), "attachment_links": [ - f"/api/assets/v2/workspaces/{issue.workspace.slug}/projects/{issue.project_id}/issues/{issue.id}/attachments/{asset.id}/" - for asset in FileAsset.objects.filter( - issue_id=issue.id, - entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT, - ) + f"/api/assets/v2/workspaces/{issue.workspace.slug}/projects/{issue.project_id}/issues/{issue.id}/attachments/{asset}/" + for asset in attachments ], } From bebb2961a4c54bc72fefcaf4868dfd6dd75653e7 Mon Sep 17 00:00:00 2001 From: sangeethailango Date: Tue, 6 May 2025 22:08:23 +0530 Subject: [PATCH 11/12] chore: optimize fetching issue details for multiple select --- apiserver/plane/bgtasks/export_task.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index 6d12e3b3ca9..26390e5e01c 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -458,12 +458,12 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s files = [] if multiple: + project_dict = defaultdict(list) + for issue in issues_data: + project_dict[str(issue["project_id"])].append(issue) + for project_id in project_ids: - issues = [ - issue - for issue in issues_data - if str(issue["project_id"]) == str(project_id) - ] + issues = project_dict.get(str(project_id), []) exporter = EXPORTER_MAPPER.get(provider) if exporter is not None: From 563561c038c76376ba191a047d4b896be9811ef4 Mon Sep 17 00:00:00 2001 From: sangeethailango Date: Wed, 7 May 2025 13:16:12 +0530 Subject: [PATCH 12/12] chore: use Prefetch to avoid duplicates --- apiserver/plane/bgtasks/export_task.py | 42 ++++++++++++-------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index 26390e5e01c..061167122e1 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -15,12 +15,12 @@ from django.conf import settings from django.utils import timezone from openpyxl import Workbook -from django.db.models import F +from django.db.models import F, Prefetch from collections import defaultdict # Module imports -from plane.db.models import ExporterHistory, Issue, FileAsset +from plane.db.models import ExporterHistory, Issue, FileAsset, Label, User from plane.utils.exception_logger import log_exception @@ -333,6 +333,16 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "issue_module__module", "issue_comments", "assignees", + Prefetch( + "assignees", + queryset=User.objects.only("first_name", "last_name").distinct(), + to_attr="assignee_details", + ), + Prefetch( + "labels", + queryset=Label.objects.only("name").distinct(), + to_attr="label_details", + ), "issue_subscribers", "issue_link", ) @@ -369,39 +379,25 @@ def issue_export_task(provider, workspace_id, project_ids, token_id, multiple, s "archived_at": issue.archived_at, "module_name": [ module.module.name for module in issue.issue_module.all() - ] - if issue.issue_module.exists() - else None, - "created_by": f"{issue.created_by.first_name} {issue.created_by.last_name}", - "labels": [ - label.name - for label in issue.labels.all().distinct() - if issue.labels.exists() ], + "created_by": f"{issue.created_by.first_name} {issue.created_by.last_name}", + "labels": [label.name for label in issue.label_details], "comments": [ { "comment": comment.comment_stripped, "created_at": dateConverter(comment.created_at), "created_by": f"{comment.created_by.first_name} {comment.created_by.last_name}", } - for comment in issue.issue_comments.all().distinct() - ] - if issue.issue_comments.exists() - else None, + for comment in issue.issue_comments.all() + ], "estimate": issue.estimate_point.estimate.name if issue.estimate_point else "", - "link": [ - link.url - for link in issue.issue_link.all().distinct() - if issue.issue_link.exists() - ], + "link": [link.url for link in issue.issue_link.all()], "assignees": [ f"{assignee.first_name} {assignee.last_name}" - for assignee in issue.assignees.all().distinct() - ] - if issue.assignees.exists() - else None, + for assignee in issue.assignee_details + ], "subscribers_count": issue.issue_subscribers.count(), "attachment_count": len(attachments), "attachment_links": [