From a682999ec32028c2d28b9653905480e1b70ffb0f Mon Sep 17 00:00:00 2001 From: NarayanBavisetti Date: Thu, 3 Aug 2023 13:41:55 +0530 Subject: [PATCH 1/2] feat: created issue export csv --- apiserver/plane/api/urls.py | 6 + apiserver/plane/api/views/__init__.py | 1 + apiserver/plane/api/views/issue.py | 28 ++++ .../plane/bgtasks/project_issue_export.py | 132 ++++++++++++++++++ .../templates/emails/exports/issues.html | 8 ++ 5 files changed, 175 insertions(+) create mode 100644 apiserver/plane/bgtasks/project_issue_export.py create mode 100644 apiserver/templates/emails/exports/issues.html diff --git a/apiserver/plane/api/urls.py b/apiserver/plane/api/urls.py index c8b5e7b5eff..2d1b2c9082f 100644 --- a/apiserver/plane/api/urls.py +++ b/apiserver/plane/api/urls.py @@ -86,6 +86,7 @@ IssueSubscriberViewSet, IssueReactionViewSet, CommentReactionViewSet, + ExportIssuesEndpoint, ## End Issues # States StateViewSet, @@ -808,6 +809,11 @@ IssueAttachmentEndpoint.as_view(), name="project-issue-attachments", ), + path( + "workspaces//export-issues/", + ExportIssuesEndpoint.as_view(), + name="export-issues", + ), ## End Issues ## Issue Activity path( diff --git a/apiserver/plane/api/views/__init__.py b/apiserver/plane/api/views/__init__.py index 75509a16cee..92b647a972d 100644 --- a/apiserver/plane/api/views/__init__.py +++ b/apiserver/plane/api/views/__init__.py @@ -75,6 +75,7 @@ IssueSubscriberViewSet, CommentReactionViewSet, IssueReactionViewSet, + ExportIssuesEndpoint ) from .auth_extended import ( diff --git a/apiserver/plane/api/views/issue.py b/apiserver/plane/api/views/issue.py index 9369ccf2b35..24e99625b26 100644 --- a/apiserver/plane/api/views/issue.py +++ b/apiserver/plane/api/views/issue.py @@ -74,6 +74,7 @@ from plane.bgtasks.issue_activites_task import issue_activity from plane.utils.grouper import group_results from plane.utils.issue_filters import issue_filters +from plane.bgtasks.project_issue_export import issue_export_task class IssueViewSet(BaseViewSet): @@ -1445,3 +1446,30 @@ def destroy(self, request, slug, project_id, comment_id, reaction_code): {"error": "Something went wrong please try again later"}, status=status.HTTP_400_BAD_REQUEST, ) + + + +class ExportIssuesEndpoint(BaseAPIView): + permission_classes = [ + WorkSpaceAdminPermission, + ] + + def post(self, request, slug): + try: + + issue_export_task.delay( + email=request.user.email, data=request.data, slug=slug + ) + + return Response( + { + "message": f"Once the export is ready it will be emailed to you at {str(request.user.email)}" + }, + status=status.HTTP_200_OK, + ) + except Exception as e: + capture_exception(e) + return Response( + {"error": "Something went wrong please try again later"}, + status=status.HTTP_400_BAD_REQUEST, + ) \ No newline at end of file diff --git a/apiserver/plane/bgtasks/project_issue_export.py b/apiserver/plane/bgtasks/project_issue_export.py new file mode 100644 index 00000000000..a8fbb0b7b71 --- /dev/null +++ b/apiserver/plane/bgtasks/project_issue_export.py @@ -0,0 +1,132 @@ + +# Python imports +import csv +import io + +# Django imports +from django.core.mail import EmailMultiAlternatives +from django.template.loader import render_to_string +from django.utils.html import strip_tags +from django.conf import settings + +# Third party imports +from celery import shared_task +from sentry_sdk import capture_exception + +# Module imports +from plane.db.models import Issue + +@shared_task +def issue_export_task(email, data, slug): + try: + project_ids = data.get("project_id") + + # If project_ids is empty, fetch all issues + if not project_ids: + issues = Issue.objects.filter(workspace__slug=slug) + else: + issues = Issue.objects.filter(workspace__slug=slug, project_id__in=project_ids) + + # CSV header + header = [ + "Issue ID", + "Name", + "Description", + "Priority", + "Start Date", + "Target Date", + "State", + "Project", + "Created At" + "Updated At" + "Completed At" + "Sort Order" + "Archived At" + "Cycle Name", + "Cycle Start Date", + "Cycle End Date", + "Module Name", + "Module Start Date", + "Module Target Date", + "Created By", + "Assignee", + "Labels" + ] + + # Prepare the CSV data + rows = [header] + + # Write data for each issue + for issue in issues: + created_by_fullname = f"{issue.created_by.first_name} {issue.created_by.last_name}" if issue.created_by else "" + assignees_names = ", ".join([f"{assignee.first_name} {assignee.last_name}" for assignee in issue.assignees.all()]) + labels_names = ", ".join([label.name for label in issue.labels.all()]) + + cycle_name, cycle_start_date, cycle_end_date = None, None, None + if hasattr(issue, 'issue_cycle'): + cycle_info = issue.issue_cycle + cycle_name = cycle_info.cycle.name + cycle_start_date = cycle_info.cycle.start_date + cycle_end_date = cycle_info.cycle.end_date + + module_name, module_start_date, module_target_date = None, None, None + if hasattr(issue, 'issue_module'): + module_info = issue.issue_module + module_name = module_info.module.name + module_start_date = module_info.module.start_date + module_target_date = module_info.module.target_date + + row = [ + str(issue.project.identifier) + "-" + str(issue.sequence_id), + issue.name, + issue.description_stripped, + issue.priority, + issue.start_date, + issue.target_date, + issue.state.name, + issue.project.name, + issue.created_at, + issue.updated_at, + issue.completed_at, + issue.sort_order, + issue.archived_at, + cycle_name, + cycle_start_date, + cycle_end_date, + module_name, + module_start_date, + module_target_date, + created_by_fullname, + assignees_names, + labels_names + ] + + rows.append(row) + + # Create CSV file in-memory + csv_buffer = io.StringIO() + writer = csv.writer(csv_buffer, delimiter=",", quoting=csv.QUOTE_ALL) + + # Write CSV data to the buffer + for row in rows: + writer.writerow(row) + + subject = "Your Issue Export is ready" + + html_content = render_to_string("emails/exports/issues.html", {}) + text_content = strip_tags(html_content) + + csv_buffer.seek(0) + msg = EmailMultiAlternatives( + subject, text_content, settings.EMAIL_FROM, [email] + ) + msg.attach(f"{slug}-issues.csv", csv_buffer.read(), "text/csv") + msg.send(fail_silently=False) + + except Exception as e: + # Print logs if in DEBUG mode + if settings.DEBUG: + print(e) + capture_exception(e) + return + diff --git a/apiserver/templates/emails/exports/issues.html b/apiserver/templates/emails/exports/issues.html new file mode 100644 index 00000000000..9f446f7870a --- /dev/null +++ b/apiserver/templates/emails/exports/issues.html @@ -0,0 +1,8 @@ + + + Hey there,
+ Your requested data export from Plane Issues is now ready. The information has been compiled into a CSV format for your convenience.
+ Please find the attachment and download the CSV file. This file can easily be imported into any spreadsheet program for further analysis.
+ If you require any assistance or have any questions, please do not hesitate to contact us.
+ Thank you + From c831dc1784f799dfcd62ac1a07ca1b12a3a31ad4 Mon Sep 17 00:00:00 2001 From: NarayanBavisetti Date: Mon, 7 Aug 2023 11:44:52 +0530 Subject: [PATCH 2/2] fix: optimized the queries --- apiserver/plane/api/views/issue.py | 2 +- .../plane/bgtasks/project_issue_export.py | 173 ++++++++++++------ .../templates/emails/exports/issues.html | 11 +- 3 files changed, 123 insertions(+), 63 deletions(-) diff --git a/apiserver/plane/api/views/issue.py b/apiserver/plane/api/views/issue.py index 24e99625b26..95e598dae35 100644 --- a/apiserver/plane/api/views/issue.py +++ b/apiserver/plane/api/views/issue.py @@ -1458,7 +1458,7 @@ def post(self, request, slug): try: issue_export_task.delay( - email=request.user.email, data=request.data, slug=slug + email=request.user.email, data=request.data, slug=slug ,exporter_name=request.user.first_name ) return Response( diff --git a/apiserver/plane/bgtasks/project_issue_export.py b/apiserver/plane/bgtasks/project_issue_export.py index a8fbb0b7b71..75088be9d00 100644 --- a/apiserver/plane/bgtasks/project_issue_export.py +++ b/apiserver/plane/bgtasks/project_issue_export.py @@ -1,4 +1,3 @@ - # Python imports import csv import io @@ -8,6 +7,7 @@ from django.template.loader import render_to_string from django.utils.html import strip_tags from django.conf import settings +from django.utils import timezone # Third party imports from celery import shared_task @@ -17,40 +17,70 @@ from plane.db.models import Issue @shared_task -def issue_export_task(email, data, slug): +def issue_export_task(email, data, slug, exporter_name): try: - project_ids = data.get("project_id") - # If project_ids is empty, fetch all issues - if not project_ids: - issues = Issue.objects.filter(workspace__slug=slug) - else: - issues = Issue.objects.filter(workspace__slug=slug, project_id__in=project_ids) + project_ids = data.get("project_id", []) + issues_filter = {"workspace__slug": slug} + + if project_ids: + issues_filter["project_id__in"] = project_ids + + issues = ( + Issue.objects.filter(**issues_filter) + .select_related("project", "workspace", "state", "parent", "created_by") + .prefetch_related( + "assignees", "labels", "issue_cycle__cycle", "issue_module__module" + ) + .values_list( + "project__identifier", + "sequence_id", + "name", + "description_stripped", + "priority", + "start_date", + "target_date", + "state__name", + "project__name", + "created_at", + "updated_at", + "completed_at", + "archived_at", + "issue_cycle__cycle__name", + "issue_cycle__cycle__start_date", + "issue_cycle__cycle__end_date", + "issue_module__module__name", + "issue_module__module__start_date", + "issue_module__module__target_date", + "created_by__first_name", + "created_by__last_name", + "assignees__first_name", + "assignees__last_name", + "labels__name", + ) + ) # CSV header header = [ "Issue ID", + "Project", "Name", "Description", - "Priority", - "Start Date", - "Target Date", "State", - "Project", - "Created At" - "Updated At" - "Completed At" - "Sort Order" - "Archived At" + "Priority", + "Created By", + "Assignee", + "Labels", "Cycle Name", "Cycle Start Date", "Cycle End Date", "Module Name", "Module Start Date", "Module Target Date", - "Created By", - "Assignee", - "Labels" + "Created At" + "Updated At" + "Completed At" + "Archived At" ] # Prepare the CSV data @@ -58,49 +88,75 @@ def issue_export_task(email, data, slug): # Write data for each issue for issue in issues: - created_by_fullname = f"{issue.created_by.first_name} {issue.created_by.last_name}" if issue.created_by else "" - assignees_names = ", ".join([f"{assignee.first_name} {assignee.last_name}" for assignee in issue.assignees.all()]) - labels_names = ", ".join([label.name for label in issue.labels.all()]) - - cycle_name, cycle_start_date, cycle_end_date = None, None, None - if hasattr(issue, 'issue_cycle'): - cycle_info = issue.issue_cycle - cycle_name = cycle_info.cycle.name - cycle_start_date = cycle_info.cycle.start_date - cycle_end_date = cycle_info.cycle.end_date - - module_name, module_start_date, module_target_date = None, None, None - if hasattr(issue, 'issue_module'): - module_info = issue.issue_module - module_name = module_info.module.name - module_start_date = module_info.module.start_date - module_target_date = module_info.module.target_date - - row = [ - str(issue.project.identifier) + "-" + str(issue.sequence_id), - issue.name, - issue.description_stripped, - issue.priority, - issue.start_date, - issue.target_date, - issue.state.name, - issue.project.name, - issue.created_at, - issue.updated_at, - issue.completed_at, - issue.sort_order, - issue.archived_at, + ( + project_identifier, + sequence_id, + name, + description, + priority, + start_date, + target_date, + state_name, + project_name, + created_at, + updated_at, + completed_at, + archived_at, cycle_name, cycle_start_date, cycle_end_date, module_name, module_start_date, module_target_date, + created_by_first_name, + created_by_last_name, + assignees_first_names, + assignees_last_names, + labels_names, + ) = issue + + created_by_fullname = ( + f"{created_by_first_name} {created_by_last_name}" + if created_by_first_name and created_by_last_name + else "" + ) + + assignees_names = "" + if assignees_first_names and assignees_last_names: + assignees_names = ", ".join( + [ + f"{assignees_first_name} {assignees_last_name}" + for assignees_first_name, assignees_last_name in zip( + assignees_first_names, assignees_last_names + ) + ] + ) + + labels_names = ", ".join(labels_names) if labels_names else "" + + row = [ + f"{project_identifier}-{sequence_id}", + project_name, + name, + description, + state_name, + priority, created_by_fullname, assignees_names, - labels_names + labels_names, + cycle_name, + cycle_start_date, + cycle_end_date, + module_name, + module_start_date, + module_target_date, + start_date, + target_date, + created_at, + updated_at, + completed_at, + archived_at, ] - rows.append(row) # Create CSV file in-memory @@ -113,14 +169,18 @@ def issue_export_task(email, data, slug): subject = "Your Issue Export is ready" - html_content = render_to_string("emails/exports/issues.html", {}) + context = { + "username": exporter_name, + } + + html_content = render_to_string("emails/exports/issues.html", context) text_content = strip_tags(html_content) csv_buffer.seek(0) msg = EmailMultiAlternatives( subject, text_content, settings.EMAIL_FROM, [email] ) - msg.attach(f"{slug}-issues.csv", csv_buffer.read(), "text/csv") + msg.attach(f"{slug}-issues-{timezone.now().date()}.csv", csv_buffer.read(), "text/csv") msg.send(fail_silently=False) except Exception as e: @@ -129,4 +189,3 @@ def issue_export_task(email, data, slug): print(e) capture_exception(e) return - diff --git a/apiserver/templates/emails/exports/issues.html b/apiserver/templates/emails/exports/issues.html index 9f446f7870a..a97432b9bd8 100644 --- a/apiserver/templates/emails/exports/issues.html +++ b/apiserver/templates/emails/exports/issues.html @@ -1,8 +1,9 @@ - Hey there,
- Your requested data export from Plane Issues is now ready. The information has been compiled into a CSV format for your convenience.
- Please find the attachment and download the CSV file. This file can easily be imported into any spreadsheet program for further analysis.
- If you require any assistance or have any questions, please do not hesitate to contact us.
- Thank you + Dear {{username}},
+ Your requested Issue's data has been successfully exported from Plane. The export includes all relevant information about issues you requested from your selected projects.
+ Please find the attachment and download the CSV file. If you have any questions or need further assistance, please don't hesitate to contact our support team at engineering@plane.so. We're here to help!
+ Thank you for using Plane. We hope this export will aid you in effectively managing your projects.
+ Regards, + Team Plane